diff --git a/.dependency-cruiser.js b/.dependency-cruiser.js index 8ca56a82e..b19b2eafc 100644 --- a/.dependency-cruiser.js +++ b/.dependency-cruiser.js @@ -52,6 +52,13 @@ module.exports = { from: { path: '^src/components/' }, to: { path: '^src/(screens|navigation)/' }, }, + { + name: 'core-not-to-pro-code', + severity: 'error', + comment: 'Video, voice, sync and tools are Pro and live in the private pro/ submodule. Core reaches them only through bootstrap/hookRegistry, slotRegistry and screenRegistry, so the open-source build compiles and runs with pro/ absent. Add a hook or slot; never import pro/ (or @offgrid/pro/*) from src/.', + from: { path: '^src/' }, + to: { path: '(^pro/|^@offgrid/pro/)' }, + }, // ── Dead code ───────────────────────────────────────────────────────────── { name: 'no-orphans', diff --git a/.gitignore b/.gitignore index 2d7905928..cff64d92b 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,7 @@ docs/wire-captures/ # Scratch renders derived from scripts/e2e/fixtures (regenerable, not versioned) /tmp/ + +# Generated native video runtime +.video-build/ +native/video/*.xcframework/ diff --git a/App.tsx b/App.tsx index f3ace64ec..1e95d785b 100644 --- a/App.tsx +++ b/App.tsx @@ -39,13 +39,21 @@ import { useDownloadStore } from './src/stores/downloadStore'; import { ErrorBoundary } from './src/components/ErrorBoundary'; import { DebugLogsScreen } from './src/components/DebugLogsScreen'; import { LoadingDots } from './src/components/LoadingDots'; +import { AnalyticsConsentPrompt } from './src/components/AnalyticsConsentPrompt'; +import { + startAnalytics, + trackScreen, +} from './src/services/analytics/analytics'; +import { featureSnapshot } from './src/services/analytics/featureSnapshot'; LogBox.ignoreAllLogs(); // Suppress all logs let stopStartupProbe: (() => void) | null = null; -// Dev-only: mirror logger output into the in-app Debug Logs viewer. The whole block -// is behind __DEV__, so release builds keep main's no-op logger (zero logging cost). -if (__DEV__) { +let stopAnalytics: (() => void) | null = null; +// Every build mirrors logger output into the in-app Debug Logs viewer and the size-capped log +// file, so Report a problem carries a trace from a store build too. Log lines hold metadata only, +// never chat content (docs/ANALYTICS.md, Report a problem). The console stays dev-only (logger.ts). +{ const fmt = (a: unknown): string => { if (a instanceof Error) return `${a.name}: ${a.message}`; if (typeof a === 'string') return a; @@ -68,7 +76,7 @@ if (__DEV__) { initDebugLogFile(); // Immediately after the sink exists, so the first sample lands before anything heavy runs. The app // was being killed by iOS at launch with the log going silent half a second in; this says where it - // stops and what memory was doing when it did. + // stops and what memory was doing when it did. Dev builds only (the probe is a no-op elsewhere). stopStartupProbe = startStartupMemoryProbe(); } @@ -373,6 +381,9 @@ function App() { logger.log('[BOOT] startup complete'); setIsInitializing(false); + // Settings are hydrated: analytics starts only now, and only if the person opted in. + stopAnalytics = startAnalytics(featureSnapshot); + // Reconcile downloaded Whisper models against disk at startup. presentModelIds // isn't persisted (the filesystem is the source of truth), so it rehydrates // empty — without this scan a freshly launched app shows an already-installed @@ -406,6 +417,8 @@ function App() { return () => { startupGeneration.current += 1; stopNetworkReconnectWatcher(); + stopAnalytics?.(); + stopAnalytics = null; }; }, [initializeApp]); @@ -447,7 +460,11 @@ function App() { {AppRoot ? : null} { + applyPendingProRedirect(); + trackScreen(appNavigationRef.getCurrentRoute()?.name); + }} + onStateChange={() => trackScreen(appNavigationRef.getCurrentRoute()?.name)} theme={{ dark: isDark, colors: { @@ -480,6 +497,7 @@ function App() { > + {__DEV__ && showSyncStatusBar ? : null} diff --git a/README.md b/README.md index 630813f84..c86620572 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,9 @@ ### The Swiss Army Knife of On-Device AI -**Chat. Generate images. Use tools. See. Listen. All on your phone or Mac. All offline. Zero data leaves your device.** +**Chat. Generate images. Use tools. See. Listen. All on your phone or Mac. All offline. Your chats and files never leave your device.** + +Anonymous usage data and crash reports only if you turn them on. See [what we share](docs/ANALYTICS.md). [![GitHub stars](https://img.shields.io/github/stars/off-grid-ai/OGAM?style=social)](https://github.com/off-grid-ai/OGAM) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) @@ -223,7 +225,7 @@ Built on the shoulders of giants: **Off Grid AI** — Your AI, your device, your data. -*No cloud. No data harvesting. Just AI that works anywhere.* +*No cloud. Your chats and files never leave your device. Just AI that works anywhere.* [Join the Community on Slack](https://join.slack.com/t/off-grid-mobile/shared_invite/zt-494gbuv5c-kj9tT7U0AS8Kq8UUO63_jw) diff --git a/__tests__/hardening/batch9-diagnostics-debuglog.test.ts b/__tests__/hardening/batch9-diagnostics-debuglog.test.ts index 5d831b032..3452e02d4 100644 --- a/__tests__/hardening/batch9-diagnostics-debuglog.test.ts +++ b/__tests__/hardening/batch9-diagnostics-debuglog.test.ts @@ -14,9 +14,9 @@ * (getDeviceTier thresholds + formatBytes are already COVERED-REAL in * hardware.test.ts and are NOT duplicated here.) * - * 2. debugLogFile.ts — the dev file sink's size-cap/rotation logic (flush appends, checks + * 2. debugLogFile.ts — the file sink's size-cap/rotation logic (flush appends, checks * stat, and truncates to the newest half once over MAX_BYTES), the flush-at-50-lines - * trigger, the disabled no-op, and __DEV__/idempotent init. No existing test covers it. + * trigger, the disabled no-op, idempotent init, and the dev-only wire file. No existing test covers it. * We stand up an in-memory RNFS so the REAL flush()/rotation runs end to end. */ @@ -109,7 +109,7 @@ describe('BATCH 9 — debugLogFile size-cap / rotation (real flush logic)', () = : ''; beforeEach(() => { - (global as any).__DEV__ = true; // the sink is __DEV__-gated + (global as any).__DEV__ = true; // the wire file is __DEV__-gated jest.resetModules(); // fresh module-level `enabled`/`buffer` state per test fileSystem = require('../harness/nativeFileSystem').defaultNativeFileSystemBoundary; fileSystem.reset(); @@ -182,13 +182,16 @@ describe('BATCH 9 — debugLogFile size-cap / rotation (real flush logic)', () = expect(mod.getDebugLogPath()).toBe('/mock/documents/offgrid-debug.log'); }); - it('initDebugLogFile is a no-op outside __DEV__', async () => { + it('a store build writes the log for Report a problem, but never the unbounded wire file', async () => { (global as any).__DEV__ = false; jest.resetModules(); + fileSystem = require('../harness/nativeFileSystem').defaultNativeFileSystemBoundary; const prodMod = require('../../src/utils/debugLogFile'); prodMod.initDebugLogFile(); prodMod.appendDebugLine('info', 'prod line'); + prodMod.appendDebugLine('log', '[WIRE-STT] {"textLength":3}'); await jest.runOnlyPendingTimersAsync(); - expect(await readLog()).toBe(''); + expect(await readLog()).toContain('prod line'); + expect(await fileSystem.exists(prodMod.getWireLogPath())).toBe(false); }); }); diff --git a/__tests__/harness/analyticsApp.ts b/__tests__/harness/analyticsApp.ts new file mode 100644 index 000000000..a786edd26 --- /dev/null +++ b/__tests__/harness/analyticsApp.ts @@ -0,0 +1,145 @@ +/** + * The real App on the production navigation stack, for opt-in analytics journeys. + * + * Device boundaries only: the llama.rn engine, an in-memory disk holding one downloaded model, the + * RAM sensor, the OS app-state events, and the PostHog endpoint at `fetch`. A fresh install opens on + * Onboarding, so every journey passes Auto Setup the way a new user does. + */ +import { GB, installNativeBoundary, requireRTL, type NativeBoundary } from './nativeBoundary'; +import { installPostHogEndpoint, type PostHogEndpoint } from './postHogEndpoint'; +import { createDownloadedModel } from '../utils/factories'; + +type DevGlobal = { __DEV__?: boolean }; + +export interface AnalyticsApp { + boundary: NativeBoundary; + endpoint: PostHogEndpoint; + rtl: ReturnType; + view: ReturnType['render']>; +} + +const originalDev = (global as DevGlobal).__DEV__; + +/** Put `__DEV__` back after a test that ran as a release build. */ +export function restoreBuildFlavor(): void { + (global as DevGlobal).__DEV__ = originalDev; +} + +/** + * Launch the App. `release: true` runs it as a store build (`__DEV__` false), the only build that + * may start the analytics SDK. `relaunch: true` keeps the disk and storage of the previous launch. + */ +export async function launchApp(options: { + release: boolean; + relaunch?: boolean; + /** Installed before this version: onboarding was finished on an earlier launch. */ + returningUser?: boolean; + platform?: 'ios' | 'android'; + /** The native background-download module, for journeys that download a model. */ + download?: boolean; + /** More device boundaries, placed after the native boundary and before the App loads. */ + beforeApp?: (boundary: NativeBoundary) => void; +}): Promise { + (global as DevGlobal).__DEV__ = !options.release; + const boundary = installNativeBoundary({ + llama: true, + fs: true, + download: options.download, + ram: { platform: options.platform ?? 'android', totalBytes: 12 * GB, availBytes: 8 * GB }, + }); + const endpoint = installPostHogEndpoint(); + const docs = boundary.fs!.DocumentDirectoryPath; + const model = { + ...createDownloadedModel({ + id: 'm', + name: 'Test Model', + fileName: 'small.gguf', + filePath: `${docs}/models/small.gguf`, + engine: 'llama', + fileSize: 500 * 1024 * 1024, + }), + // Where a Hugging Face download records it came from (a public repo). + origin: { repoId: 'unsloth/Qwen3-0.6B-GGUF', revision: 'main', path: 'small.gguf' }, + }; + // The disk a download leaves behind: the file and its record. Nothing else is pre-placed. + boundary.fs!.seedFile(model.filePath!, model.fileSize); + const storage = require('@react-native-async-storage/async-storage'); + if (!options.relaunch) { + await storage.clear(); + await storage.setItem('@local_llm/downloaded_models', JSON.stringify([model])); + if (options.returningUser) { + // What an earlier version left on disk: onboarding done, the setup tour dismissed. + await storage.setItem( + 'local-llm-app-storage', + JSON.stringify({ state: { hasCompletedOnboarding: true, checklistDismissed: true }, version: 0 }), + ); + } + } + options.beforeApp?.(boundary); + const g = globalThis as unknown as { window?: Record }; + if (!g.window) g.window = { dispatchEvent: () => true, addEventListener: () => {}, removeEventListener: () => {} }; + const React = require('react'); + const rtl = requireRTL(); + const App = require('../../App').default; + const view = rtl.render(React.createElement(App)); + await rtl.waitFor(() => expect(view.queryByTestId('app-loading')).toBeNull(), { timeout: 10_000 }); + return { boundary, endpoint, rtl, view }; +} + +/** A new user: Onboarding → Auto Setup (ticking the share row or not) → Skip for Now → Home. */ +export async function passOnboarding(app: AnalyticsApp, share: boolean): Promise { + const { view, rtl } = app; + rtl.fireEvent.press(await view.findByTestId('onboarding-skip')); + const skip = await view.findByTestId('auto-setup-skip', {}, { timeout: 10_000 }); + const row = view.getByTestId('auto-setup-share-analytics'); + expect(row.props.accessibilityState).toEqual({ checked: false }); + if (share) { + rtl.fireEvent.press(row); + await rtl.waitFor(() => + expect(view.getByTestId('auto-setup-share-analytics').props.accessibilityState).toEqual({ checked: true }), + ); + } + rtl.fireEvent.press(skip); + await view.findByTestId('home-screen'); +} + +export async function openNewChat(app: AnalyticsApp): Promise { + const { view, rtl } = app; + rtl.fireEvent.press(view.getByTestId('browse-models-button')); + rtl.fireEvent.press(await view.findByTestId('text-model-row-m')); + await rtl.waitFor(() => expect(view.queryByTestId('text-model-row-m')).toBeNull()); + rtl.fireEvent.press(view.getByTestId('new-chat-button')); + await view.findByTestId('chat-input'); +} + +/** Type and send a message; the model's reply (scripted at the engine leaf) renders. */ +export async function sendMessage(app: AnalyticsApp, prompt: string, reply: string): Promise { + const { view, rtl, boundary } = app; + boundary.llama!.scriptCompletion({ text: reply }); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), prompt); + rtl.fireEvent.press(view.getByTestId('send-button')); + await view.findByText(reply, {}, { timeout: 10_000 }); + await rtl.waitFor(() => expect(view.queryByTestId('stop-button')).toBeNull()); +} + +/** The person leaves the app: the OS moves it to the background. */ +export async function sendToBackground(app: AnalyticsApp): Promise { + await app.rtl.act(async () => { + app.boundary.emitAppStateChange('background'); + }); +} + +export async function bringToForeground(app: AnalyticsApp): Promise { + await app.rtl.act(async () => { + app.boundary.emitAppStateChange('active'); + }); +} + +/** From a chat: back to the tabs, then the Settings tab. */ +export async function openSettings(app: AnalyticsApp): Promise { + const { view, rtl } = app; + const back = view.queryAllByLabelText('Back'); + if (back.length > 0) rtl.fireEvent.press(back[0]); + rtl.fireEvent.press(await view.findByTestId('settings-tab')); + await view.findByTestId('settings-analytics'); +} diff --git a/__tests__/harness/chatHarness.ts b/__tests__/harness/chatHarness.ts index 4cf6c30db..f418ed301 100644 --- a/__tests__/harness/chatHarness.ts +++ b/__tests__/harness/chatHarness.ts @@ -45,6 +45,8 @@ export interface ChatHarnessOptions { audio?: boolean; /** Install the driveable whisper.rn STT fake (for chat-mode voice input flows). */ whisper?: boolean; + /** (llama) A llama.rn audio projector starts, as Qwen3-ASR's does (speech-to-text on llama.rn). */ + llamaAudio?: boolean; /** Install the stateful background-download native fake (drive DownloadProgress/Complete/Error * events through the real backgroundDownloadService — e.g. an in-flight STT model download). */ download?: boolean; @@ -218,6 +220,7 @@ export async function setupChatScreen(opts: ChatHarnessOptions) { llamaChatTemplate: opts.chatTemplate, llamaModelInfo: opts.llamaModelInfo, llamaVision: opts.engine === 'llama' && opts.vision, + llamaAudio: opts.engine === 'llama' && opts.llamaAudio, deviceHardware: opts.deviceHardware, socModel: opts.socModel, fs: true, diff --git a/__tests__/harness/desktopModelPulls.ts b/__tests__/harness/desktopModelPulls.ts new file mode 100644 index 000000000..a1b830b6a --- /dev/null +++ b/__tests__/harness/desktopModelPulls.ts @@ -0,0 +1,137 @@ +/** + * Off Grid AI Desktop's model manager, at the network boundary (fetch), layered over another Desktop + * fake (serveDesktopVideoJobs) that keeps the video job routes. + * + * Wire shapes are Desktop's own (desktop/src/main/model-server.ts and models-manager.ts): + * GET /v1/models/catalog is { kinds, memoryGb, models } (memoryGb absent on an older Desktop); + * GET /v1/models/installed is { installed }; POST /v1/models/pull { id } answers 202 and starts one + * download; GET /v1/models/pull/status?id= is the last progress { status, percent, totalBytes } or + * { status: 'idle' }; POST /v1/models/activate { id, kind } succeeds for an installed model. + * POST /v1/stories/storyboard { sentence } is the storyboard Desktop writes with no text model loaded + * (desktop/pro/main/video/studio/gateway.ts): the shared template, marked `source: 'template'`. + * + * A download really runs: each status read moves it on, until it completes and its model is installed. + * `hold()` keeps it where it is, so a test can see it in flight, and `release()` lets it go on. + */ +import { CATALOG, templateStoryboard } from '@offgrid/models'; +import { DESKTOP_VIDEO_ENDPOINT, DESKTOP_VIDEO_MODEL } from './desktopVideoServer'; + +interface Progress { + status: 'queued' | 'downloading' | 'completed' | 'failed'; + percent: number; + totalBytes: number; +} + +export interface DesktopModelPulls { + /** Every POST /v1/models/pull body Desktop received. */ + pulls: Array<{ id: string }>; + /** What Desktop activated, in order. */ + activations: string[]; + hold: () => void; + release: () => void; + /** A model installed on the computer itself (its own Models screen), after the phone last asked. */ + install: (id: string) => void; + restore: () => void; +} + +const PERCENT_PER_READ = 50; + +export function serveDesktopModelPulls(options: { + /** The computer's memory as its catalog reports it; undefined is an older Desktop. */ + memoryGb?: number; + /** Catalog ids already on the computer. */ + installed?: string[]; + /** A download already running on the computer when the phone first asks. */ + downloading?: { id: string; percent: number }; +}): DesktopModelPulls { + const inner = globalThis.fetch; + const installed = new Set([DESKTOP_VIDEO_MODEL.id, ...(options.installed ?? [])]); + const downloads = new Map(); + let held = false; + let activeVideo: string | null = DESKTOP_VIDEO_MODEL.id; + const videoEntries = CATALOG.filter(model => model.kind === 'video'); + const sizeOf = (id: string) => + videoEntries.find(model => model.id === id)?.files.reduce((sum, file) => sum + (file.sizeBytes ?? 0), 0) ?? 0; + if (options.downloading) { + downloads.set(options.downloading.id, { + status: 'downloading', + percent: options.downloading.percent, + totalBytes: sizeOf(options.downloading.id), + }); + } + const state: DesktopModelPulls = { + pulls: [], + activations: [], + hold: () => { + held = true; + }, + release: () => { + held = false; + }, + install: (id: string) => { + installed.add(id); + }, + restore: () => { + globalThis.fetch = inner; + }, + }; + const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); + const advance = (id: string): Progress | undefined => { + const progress = downloads.get(id); + if (!progress || held || progress.status === 'completed' || progress.status === 'failed') return progress; + const percent = Math.min(100, progress.percent + PERCENT_PER_READ); + const next: Progress = { ...progress, status: percent >= 100 ? 'completed' : 'downloading', percent }; + if (next.status === 'completed') installed.add(id); + downloads.set(id, next); + return next; + }; + + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const href = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + const url = new URL(href); + if (url.origin !== DESKTOP_VIDEO_ENDPOINT) return inner(input, init); + const method = (init?.method ?? 'GET').toUpperCase(); + const route = `${method} ${url.pathname}`; + if (route === 'GET /v1/models/catalog') { + return json({ + kinds: ['text', 'vision', 'image', 'video', 'speech', 'transcription'], + ...(options.memoryGb === undefined ? {} : { memoryGb: options.memoryGb }), + models: [ + { ...DESKTOP_VIDEO_MODEL, kind: 'video', org: 'Wan-AI', tags: [], files: [] }, + ...videoEntries.map(model => ({ id: model.id, name: model.name, kind: 'video', files: model.files })), + ], + }); + } + if (route === 'GET /v1/models/installed') return json({ installed: [...installed] }); + if (route === 'GET /v1/models/active') { + return json({ text: null, image: null, video: activeVideo, speech: null, transcription: null }); + } + if (route === 'POST /v1/models/activate') { + const body = JSON.parse(String(init?.body ?? '{}')) as { id?: string; kind?: string }; + const known = body.kind === 'video' && !!body.id && installed.has(body.id); + if (known) { + activeVideo = body.id!; + state.activations.push(body.id!); + } + return json(known ? { success: true } : { success: false, error: 'Model is not installed.' }, known ? 200 : 400); + } + if (route === 'POST /v1/models/pull') { + const { id } = JSON.parse(String(init?.body ?? '{}')) as { id: string }; + state.pulls.push({ id }); + if (!downloads.has(id) || downloads.get(id)!.status === 'failed') + downloads.set(id, { status: 'queued', percent: downloads.get(id)?.percent ?? 0, totalBytes: sizeOf(id) }); + return json({ status: 'started', id, poll: `/v1/models/pull/status?id=${encodeURIComponent(id)}` }, 202); + } + if (route === 'POST /v1/stories/storyboard') { + const { sentence } = JSON.parse(String(init?.body ?? '{}')) as { sentence: string }; + return json({ ...templateStoryboard(sentence.trim()), source: 'template' }); + } + if (route === 'GET /v1/models/pull/status') { + const progress = advance(url.searchParams.get('id') ?? ''); + return json(progress ?? { status: 'idle' }); + } + return inner(input, init); + }) as typeof globalThis.fetch; + return state; +} diff --git a/__tests__/harness/desktopSearchServer.ts b/__tests__/harness/desktopSearchServer.ts new file mode 100644 index 000000000..1c58099a0 --- /dev/null +++ b/__tests__/harness/desktopSearchServer.ts @@ -0,0 +1,82 @@ +/** + * Off Grid AI Desktop's search, at the network boundary (`fetch`). + * + * Wire shape is Desktop's own (desktop/pro/main/search-gateway.ts over the shared + * @offgrid/rag remote-search contract): `POST /v1/search {query, limit}` answers + * `{object: 'list', data: [{id, title, snippet, kind, source, timestamp, deepLinkId}]}`, best + * first. Like the real gateway it refuses a request without the right key with 401, answers a + * route it does not serve with 404, and, when the computer is off, never answers at all (the + * request fails as React Native's fetch fails: `TypeError: Network request failed`). + */ + +export const DESKTOP_SEARCH_ENDPOINT = 'http://192.168.1.62:7878'; // NOSONAR - private LAN test fixture + +export interface DesktopRecord { + id: string; + title: string; + snippet: string; + kind: string; + source: string; + timestamp: number; + deepLinkId?: string; +} + +export interface DesktopSearch { + /** Every query the computer was asked. */ + queries: string[]; + /** The computer is switched off (true) or back on (false). */ + setOffline: (offline: boolean) => void; + /** Add a record the computer has seen (e.g. a chat synced from this phone). */ + remember: (record: DesktopRecord) => void; + restore: () => void; +} + +export function serveDesktopSearch(options: { apiKey: string; records: DesktopRecord[] }): DesktopSearch { + const original = globalThis.fetch; + const records = [...options.records]; + let offline = false; + const state: DesktopSearch = { + queries: [], + setOffline: value => { + offline = value; + }, + remember: record => { + records.unshift(record); + }, + restore: () => { + globalThis.fetch = original; + }, + }; + const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); + + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const href = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + const url = new URL(href); + if (url.origin !== DESKTOP_SEARCH_ENDPOINT) return original(input, init); + if (init?.signal?.aborted) throw Object.assign(new Error('Aborted'), { name: 'AbortError' }); + if (offline) throw new TypeError('Network request failed'); + const authorization = new Headers(init?.headers).get('Authorization'); + if (authorization !== `Bearer ${options.apiKey}`) { + return json({ error: { message: 'A valid API key is required.', type: 'authentication_error' } }, 401); + } + const method = (init?.method ?? 'GET').toUpperCase(); + // The rest of what a phone's connection check asks Desktop: no models in this fixture. + if (method === 'GET' && url.pathname === '/v1/models') return json({ object: 'list', data: [] }); + if (method === 'GET' && url.pathname === '/v1/models/catalog') return json({ kinds: ['text'], models: [] }); + if (method === 'GET' && url.pathname === '/v1/models/installed') return json({ installed: [] }); + if (method === 'GET' && url.pathname === '/v1/models/active') return json({ text: null }); + if (method !== 'POST' || url.pathname !== '/v1/search') { + return json({ error: { message: 'Not found', type: 'invalid_request_error' } }, 404); + } + const body = JSON.parse(String(init?.body ?? '{}')) as { query?: string; limit?: number }; + const query = (body.query ?? '').toLowerCase(); + state.queries.push(body.query ?? ''); + const data = records + .filter(record => `${record.title} ${record.snippet}`.toLowerCase().includes(query)) + .slice(0, body.limit ?? 20) + .map(record => ({ ...record, deepLinkId: record.deepLinkId ?? record.id })); + return json({ object: 'list', data }); + }) as typeof globalThis.fetch; + return state; +} diff --git a/__tests__/harness/desktopStoryJobs.ts b/__tests__/harness/desktopStoryJobs.ts new file mode 100644 index 000000000..fe1e18cb3 --- /dev/null +++ b/__tests__/harness/desktopStoryJobs.ts @@ -0,0 +1,136 @@ +/** + * Off Grid AI Desktop rendering a whole film, at the network boundary (fetch + the RNFS download), + * layered over the other Desktop fakes (serveDesktopVideoJobs / serveDesktopModelPulls). + * + * Wire shapes are Desktop's own (desktop/pro/main/video/studio/gateway.ts, storyGatewayRoute): + * POST /v1/stories answers 202 { request_id: story.id, object: 'story', status: 'queued', poll_url }; + * GET /v1/stories/{id} is { request_id, object: 'story', status, story, progress? , error? , content_url? } + * where status is the shared VideoStoryStatus ('queued' | 'rendering' | 'done' | 'failed' | 'cancelled'), + * `story.shots[].status` the shared VideoShotStatus ('pending' … 'final'), `progress` Desktop's + * StudioProgress ({ phase, shot, fraction }) and `error` { message, type }; + * POST /v1/stories/{id}/cancel answers { cancelled }; GET /v1/stories/{id}/content is the MP4. + * + * A film really runs: each poll moves it one shot on, until every shot is final and the film is done. + * `hold()` keeps it where it is, so a test can see it in flight; `release()` lets it go on. + * `failNext(message)` makes the next film fail on the computer, the way Desktop reports a failure. + */ +import type { NativeFileSystemBoundary } from './nativeFileSystem'; +import { DESKTOP_VIDEO_ENDPOINT } from './desktopVideoServer'; + +interface Film { + story: { id: string; shots: Array> } & Record; + done: number; + status: 'queued' | 'rendering' | 'done' | 'failed' | 'cancelled'; + error?: string; +} + +export interface DesktopStoryJobs { + /** Every story body the phone sent (POST /v1/stories). */ + submissions: Array>; + /** Every film the phone asked Desktop to stop. */ + cancels: string[]; + hold: () => void; + release: () => void; + failNext: (message: string) => void; + restore: () => void; + /** After a relaunch: film downloads are served to the new launch's disk too. Films carry on. */ + useDisk: (fs: NativeFileSystemBoundary) => void; + /** When the phone last asked this fake anything (ms since epoch), so a test can end once it is quiet. */ + lastRequestAt: () => number; +} + +export function serveDesktopStoryJobs(disk: NativeFileSystemBoundary): DesktopStoryJobs { + const inner = globalThis.fetch; + // Every launch's disk: a launch the test closed still finishes what it was downloading, as a + // killed process would simply never ask; serving it keeps that stale launch from failing loudly. + const disks = [disk]; + let lastRequest = 0; + const films = new Map(); + let held = false; + let failure: string | null = null; + const state: DesktopStoryJobs = { + submissions: [], + cancels: [], + hold: () => { + held = true; + }, + release: () => { + held = false; + }, + failNext: message => { + failure = message; + }, + restore: () => { + globalThis.fetch = inner; + }, + useDisk: next => { + disks.push(next); + }, + lastRequestAt: () => lastRequest, + }; + const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); + const errBody = (message: string, type = 'invalid_request_error') => ({ error: { message, type } }); + const advance = (film: Film) => { + if (held || film.status === 'done' || film.status === 'failed' || film.status === 'cancelled') return; + if (failure) { + film.status = 'failed'; + film.error = failure; + failure = null; + return; + } + film.status = 'rendering'; + film.done = Math.min(film.story.shots.length, film.done + 1); + if (film.done === film.story.shots.length) film.status = 'done'; + }; + const view = (film: Film) => ({ + ...film.story, + status: film.status, + shots: film.story.shots.map((shot, index) => ({ ...shot, status: index < film.done ? 'final' : 'pending' })), + }); + + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const href = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + const url = new URL(href); + if (url.origin !== DESKTOP_VIDEO_ENDPOINT) return inner(input, init); + lastRequest = Date.now(); + const method = (init?.method ?? 'GET').toUpperCase(); + if (url.pathname === '/v1/stories' && method === 'POST') { + const body = JSON.parse(String(init?.body ?? '{}')) as { story: Film['story'] }; + state.submissions.push(body as unknown as Record); + const id = body.story.id; + if (!films.has(id) || films.get(id)!.status !== 'rendering') films.set(id, { story: body.story, done: 0, status: 'queued' }); + return json({ request_id: id, object: 'story', status: 'queued', poll_url: `/v1/stories/${id}` }, 202); + } + const match = /^\/v1\/stories\/([A-Za-z0-9_-]{8,100})(?:\/(content|captions|cancel))?$/.exec(url.pathname); + if (!match) return inner(input, init); + const film = films.get(match[1]); + if (!film) return json(errBody('Story not found.', 'not_found'), 404); + const action = match[2]; + if (!action && method === 'GET') { + advance(film); + const contentUrl = `/v1/stories/${film.story.id}/content`; + if (film.status === 'done') for (const fs of disks) fs.serveRemoteFile(`${DESKTOP_VIDEO_ENDPOINT}${contentUrl}`, { body: 'film-mp4-bytes' }); + const running = film.status === 'queued' || film.status === 'rendering'; + return json({ + request_id: film.story.id, + object: 'story', + status: film.status, + story: view(film), + ...(running + ? { progress: { phase: 'clips', shot: film.done, fraction: film.done / film.story.shots.length } } + : {}), + ...(film.error ? { error: { message: film.error, type: 'server_error' } } : {}), + ...(film.status === 'done' ? { content_url: contentUrl } : {}), + }); + } + if (action === 'cancel' && method === 'POST') { + state.cancels.push(film.story.id); + const cancelled = film.status === 'queued' || film.status === 'rendering'; + if (cancelled) film.status = 'cancelled'; + return json({ cancelled }); + } + return inner(input, init); + }) as typeof globalThis.fetch; + return state; +} diff --git a/__tests__/harness/desktopVideoServer.ts b/__tests__/harness/desktopVideoServer.ts new file mode 100644 index 000000000..4e2a7591e --- /dev/null +++ b/__tests__/harness/desktopVideoServer.ts @@ -0,0 +1,166 @@ +/** + * Off Grid AI Desktop as the video server, at the network boundary (fetch + the RNFS download). + * + * Wire shapes are Desktop's own (desktop/src/main/model-server.ts, startGatewayVideoJob and + * gatewayVideoJob): POST /v1/videos answers { request_id, status, poll_url }; GET /v1/videos/{id} + * is the poll { request_id, status, stage, progress, result, error }; a completed result is the + * shared VideoGenerationResultContract plus `url: /v1/videos/{id}/content`, which the phone + * downloads. The model switcher is /v1/models/catalog, /installed, /active and /activate. + */ +import type { NativeFileSystemBoundary } from './nativeFileSystem'; + +export const DESKTOP_VIDEO_ENDPOINT = 'http://192.168.1.62:7878'; // NOSONAR - private LAN test fixture +export const DESKTOP_VIDEO_MODEL = { id: 'wan2.1-1.3b', name: 'Wan 2.1 1.3B' }; + +export interface DesktopVideoJobs { + /** Every video submission Desktop received: the JSON body the phone sent. */ + submissions: Array>; + /** Every job the phone asked Desktop to stop (POST /v1/videos/:id/cancel). */ + cancels: string[]; + /** From now on a running job carries Desktop's live preview (its poll's `preview`, and GET .../preview). */ + showPreviews: () => void; + /** Every job running now ends failed with this reason, as Desktop reports a failed job. */ + failRunning: (message: string) => void; + /** Keep every job running (polls answer `running`) until release(). */ + hold: () => void; + release: () => void; + restore: () => void; + /** After a relaunch: content downloads are served to the new launch's disk too. Jobs carry on. */ + useDisk: (fs: NativeFileSystemBoundary) => void; + /** When the phone last asked this fake anything (ms since epoch), so a test can end once it is quiet. */ + lastRequestAt: () => number; +} + +export function serveDesktopVideoJobs(disk: NativeFileSystemBoundary): DesktopVideoJobs { + const original = globalThis.fetch; + // Every launch's disk: a launch the test closed still finishes what it was downloading, as a + // killed process would simply never ask; serving it keeps that stale launch from failing loudly. + const disks = [disk]; + let lastRequest = 0; + const jobs = new Map; polls: number; cancelled?: boolean; failed?: string }>(); + let activeVideo: string | null = null; + let held = false; + let previews = false; + const state: DesktopVideoJobs = { + submissions: [], + cancels: [], + showPreviews: () => { + previews = true; + }, + failRunning: message => { + for (const job of jobs.values()) if (!job.cancelled) job.failed = message; + }, + hold: () => { + held = true; + }, + release: () => { + held = false; + }, + restore: () => { + globalThis.fetch = original; + }, + useDisk: next => { + disks.push(next); + }, + lastRequestAt: () => lastRequest, + }; + const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); + + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const href = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + const url = new URL(href); + if (url.origin !== DESKTOP_VIDEO_ENDPOINT) return original(input, init); + lastRequest = Date.now(); + const method = (init?.method ?? 'GET').toUpperCase(); + const route = `${method} ${url.pathname}`; + if (route === 'GET /v1/models') return json({ object: 'list', data: [] }); + if (route === 'GET /v1/models/catalog') { + return json({ + kinds: ['text', 'vision', 'image', 'video', 'speech', 'transcription'], + models: [{ ...DESKTOP_VIDEO_MODEL, kind: 'video', org: 'Wan-AI', tags: [], files: [] }], + }); + } + if (route === 'GET /v1/models/installed') return json({ installed: [DESKTOP_VIDEO_MODEL.id] }); + if (route === 'GET /v1/models/active') { + return json({ text: null, image: null, video: activeVideo, speech: null, transcription: null }); + } + if (route === 'POST /v1/models/activate') { + const body = JSON.parse(String(init?.body ?? '{}')) as { id?: string; kind?: string }; + const known = body.kind === 'video' && body.id === DESKTOP_VIDEO_MODEL.id; + if (known) activeVideo = DESKTOP_VIDEO_MODEL.id; + return json({ success: known }); + } + if (route === 'POST /v1/videos') { + const body = JSON.parse(String(init?.body ?? '{}')) as Record; + state.submissions.push(body); + const id = String(body.client_job_id); + // The same client_job_id recovers the job Desktop already has; it never starts a second clip. + if (!jobs.has(id)) jobs.set(id, { body, polls: 0 }); + return json({ request_id: id, object: 'request', kind: 'video', status: 'queued', poll_url: `/v1/videos/${id}` }, 202); + } + const cancel = url.pathname.match(/^\/v1\/videos\/([^/]+)\/cancel$/); + if (cancel && method === 'POST') { + const id = decodeURIComponent(cancel[1]); + const job = jobs.get(id); + if (!job) return json({ error: { message: 'Video job was not found.', type: 'not_found' } }, 404); + job.cancelled = true; + state.cancels.push(id); + return json({ request_id: id, status: 'running', poll_url: `/v1/videos/${id}` }); + } + const preview = url.pathname.match(/^\/v1\/videos\/([^/]+)\/preview$/); + if (preview && method === 'GET') { + // Desktop streams the running job's first decoded frame as a PNG (gateway.ts, resource 'preview'). + const job = jobs.get(decodeURIComponent(preview[1])); + if (!job || !previews || job.cancelled || job.failed) + return json({ error: { message: 'Video preview is not available.', type: 'invalid_request_error' } }, 404); + return new Response(new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), { + status: 200, + headers: { 'Content-Type': 'image/png', 'Cache-Control': 'no-store' }, + }); + } + const poll = url.pathname.match(/^\/v1\/videos\/([^/]+)$/); + if (poll && method === 'GET') { + const id = decodeURIComponent(poll[1]); + const job = jobs.get(id); + if (!job) return json({ error: { message: 'Video job was not found.', type: 'not_found' } }, 404); + job.polls += 1; + const base = { request_id: id, poll_url: `/v1/videos/${id}` }; + // Desktop records a stop as a failed request whose error type is `cancelled` (pro/main/video/gateway.ts). + if (job.failed) { + return json({ ...base, status: 'failed', error: { message: job.failed, type: 'server_error' } }); + } + if (job.cancelled) { + return json({ ...base, status: 'failed', error: { message: 'Video generation stopped.', type: 'cancelled' } }); + } + if (job.polls === 1 || held) { + return json({ + ...base, + status: 'running', + stage: 'generating', + ...(previews ? { preview: { url: `/v1/videos/${id}/preview`, width: 320, height: 192 } } : {}), + progress: { step: 1, total: 4 }, + }); + } + const contentUrl = `${DESKTOP_VIDEO_ENDPOINT}/v1/videos/${id}/content`; + for (const fs of disks) fs.serveRemoteFile(contentUrl, { body: 'mp4-bytes' }); + // Desktop returns its job output without the local path (VideoGenerationOutputContract). + const { prompt, negativePrompt, width, height, frames, fps, steps, guidance, seed } = job.body; + return json({ + ...base, + status: 'completed', + stage: 'encoding', + result: { + prompt, negativePrompt, width, height, frames, fps, steps, guidance, seed, + model: DESKTOP_VIDEO_MODEL.id, + durationSeconds: Number(frames) / Number(fps), + syncId: `desktop-video-${id}`, + durationMs: 4200, + url: `/v1/videos/${id}/content`, + }, + }); + } + return json({ error: { message: 'Unknown endpoint', type: 'not_found' } }, 404); + }) as typeof globalThis.fetch; + return state; +} diff --git a/__tests__/harness/lanProbe.ts b/__tests__/harness/lanProbe.ts index a9583e13c..33ebf9910 100644 --- a/__tests__/harness/lanProbe.ts +++ b/__tests__/harness/lanProbe.ts @@ -19,6 +19,15 @@ interface LiveHost { paths: string[]; /** The body it returns, device-shaped for an OpenAI-compatible server. */ body: unknown; + /** + * Off Grid AI Desktop with API keys on: every API path answers 401, while /health stays open and + * names the gateway, as Desktop's gateway does. + */ + locked?: boolean; + /** The gateway key a locked Desktop accepts: with `Authorization: Bearer ` its API answers. */ + apiKey?: string; + /** The Sync device id the Desktop names on /health (`computer_id`), as Desktop's gateway does. */ + computerId?: string; } export interface LanProbeHandle { @@ -47,7 +56,10 @@ export function installLanProbe(live: Record): LanProbeHandle const original = global.fetch; const requested: string[] = []; - const fake = (input: unknown, init?: { signal?: AbortSignal }): Promise => { + const fake = ( + input: unknown, + init?: { signal?: AbortSignal; headers?: Record }, + ): Promise => { const url = String(input); requested.push(url); @@ -60,11 +72,23 @@ export function installLanProbe(live: Record): LanProbeHandle const path = match?.[2] ?? '/'; const host = live[authority]; - if (!host || !host.paths.some(allowed => path.startsWith(allowed))) { + if (!host || !(host.paths.some(allowed => path.startsWith(allowed)) || (host.locked && path.startsWith('/health')))) { // A closed port refuses at once. This is what the great majority of a /24 does. return Promise.reject(new Error('Network request failed')); } + const keyed = !!host.apiKey && init?.headers?.Authorization === `Bearer ${host.apiKey}`; + if (host.locked && !keyed && path.startsWith('/v1/')) { + return Promise.resolve({ ok: false, status: 401, json: async () => ({ error: 'unauthorized' }) } as unknown as Response); + } + if (host.locked && path.startsWith('/health')) { + const health = { + name: 'Off Grid AI — local model gateway', + ...(host.computerId ? { computer_id: host.computerId } : {}), + }; + return Promise.resolve({ ok: true, status: 200, json: async () => health, text: async () => JSON.stringify(health) } as unknown as Response); + } + return Promise.resolve({ ok: true, status: 200, diff --git a/__tests__/harness/nativeBoundary.ts b/__tests__/harness/nativeBoundary.ts index 6b7fef7ee..e9790a573 100644 --- a/__tests__/harness/nativeBoundary.ts +++ b/__tests__/harness/nativeBoundary.ts @@ -419,6 +419,7 @@ function makeLlamaFake( modelInfo?: Record, fs?: NativeFileSystemBoundary, vision = false, + audio = false, ): LlamaFake { const calls: LlamaFake['calls'] = { completion: [], @@ -464,6 +465,13 @@ function makeLlamaFake( // Sticky Metal backend error (ggml-metal-context has_error): set when the app goes to background with a // completion in flight on Metal; cleared only by a fresh initLlama (a new native context). let backendFault = false; + // llama.cpp's own log (llama.rn toggleNativeLog / addNativeLogListener). While a lazy tool grammar + // waits for its trigger, llama.cpp logs every sampled token, with the token's text, at info level. + let nativeLogOn = false; + const nativeLogListeners: Array<(level: string, text: string) => void> = []; + const nativeLog = (level: string, text: string) => { + if (nativeLogOn) for (const listener of nativeLogListeners) listener(level, text); + }; let completionInFlight = false; // The device shape of a completion that the backend fault ended: what streamed so far, no EOS, no stop // word, no cap, NOT interrupted (rn-completion.cpp just sets has_next_token=false on the failed decode). @@ -570,6 +578,11 @@ function makeLlamaFake( if (backendFault) break; // the GPU refused the next decode (app went to background) acc += c; streamed = acc; + // A real token is about a word: log the word this character ends. + const word = /(\S+)$/.exec(acc.slice(0, -1))?.[1]; + if (/\s/.test(c) && word) { + nativeLog('info', `Grammar still awaiting trigger after token ${acc.length} (\` ${word}\`)`); + } onToken({ token: c, content: acc }); if ( scripted.pauseAfter && @@ -691,7 +704,25 @@ function makeLlamaFake( setTimeout(() => onRelease?.(), 50); }), tokenize: jest.fn().mockResolvedValue({ tokens: [1, 2, 3] }), - initMultimodal: jest.fn().mockResolvedValue(vision), + initMultimodal: jest.fn().mockResolvedValue(vision || audio), + // llama.rn renders the chat with the model's template; each media part becomes its marker and + // its path joins media_paths, in order. + getFormattedChat: jest.fn(async (messages: Array<{ role: string; content?: unknown }>) => { + const mediaPaths: string[] = []; + const turns = messages.map(message => { + const parts = Array.isArray(message.content) ? message.content : [{ type: 'text', text: message.content }]; + const body = parts.map((part: { type: string; text?: string; input_audio?: { url?: string }; image_url?: { url?: string } }) => { + if (part.type === 'input_audio' || part.type === 'image_url') { + mediaPaths.push(String(part.input_audio?.url ?? part.image_url?.url ?? '').replace(/^file:\/\//, '')); + return '<__media__>'; + } + return part.text ?? ''; + }).join(''); + return `<|im_start|>${message.role}\n${body}<|im_end|>\n`; + }).join(''); + const prompt = `${turns}<|im_start|>assistant\n`; + return { type: 'jinja', prompt, has_media: mediaPaths.length > 0, media_paths: mediaPaths }; + }), // The post-init multimodal probe. A scripted hold parks the caller here — the real device's // window between context init and capability detection — until releaseMultimodalHold(). getMultimodalSupport: jest.fn(async () => { @@ -703,7 +734,7 @@ function makeLlamaFake( }); mmHoldEngaged = false; } - return { vision, audio: false }; + return { vision, audio }; }), // Embedding boundary (embedding-model contexts, initLlama({embedding:true})): return a device-shaped // 384-dim vector derived from the text so RAG cosine ranking is real. Matches all-MiniLM-L6-v2 (384). @@ -736,6 +767,13 @@ function makeLlamaFake( const module: Record = { loadLlamaModelInfo: jest.fn(async () => modelInfo ?? {}), + toggleNativeLog: jest.fn(async (enabled: boolean) => { + nativeLogOn = enabled; + }), + addNativeLogListener: jest.fn((listener: (level: string, text: string) => void) => { + nativeLogListeners.push(listener); + return { remove: () => nativeLogListeners.splice(nativeLogListeners.indexOf(listener), 1) }; + }), // Faithful to llama.rn/llama.cpp: the native loader reports gpu=true (+ the offload device // list) when it actually offloaded layers (n_gpu_layers > 0), and gpu=false for a pure-CPU // init. Echo that from the requested load params so the REAL captureGpuInfo → GenerationMeta @@ -959,6 +997,9 @@ export interface DownloadRow { bytesDownloaded?: number; totalBytes?: number; reason?: string; + modelKey?: string; + combinedTotalBytes?: number; + createdAt?: number; } export interface DownloadFake { @@ -1183,6 +1224,8 @@ export interface RamProfile { totalBytes: number; /** Truly-free RAM right now, in bytes (os_proc_available). */ availBytes: number; + /** iOS only: pages the whole phone can still hand out (free + inactive + purgeable). Unset: not reported. */ + systemAvailBytes?: number; } export const GB = 1024 * 1024 * 1024; @@ -1216,6 +1259,8 @@ export interface InstallOpts { llamaModelInfo?: Record; /** The projector starts: initMultimodal resolves true and the context reports vision support. */ llamaVision?: boolean; + /** An audio projector starts: initMultimodal resolves true and the context reports audio support. */ + llamaAudio?: boolean; /** Seed a stateful background-download native module (boundary.download). */ download?: boolean; /** Replace the global whisper.rn stub with a driveable STT context (boundary.whisper). */ @@ -1276,7 +1321,8 @@ export function installNativeBoundary(opts: InstallOpts = {}): NativeBoundary { // context FREES memory (footprint drops, available rises), so the post-unload reclaim barrier // (memoryBudget.awaitMemoryReclaim) observes the drop and resolves — instead of timing out on a // frozen snapshot. A released llama context frees roughly a heavy model's worth (~3GB). - const memState = { + const memState: { availBytes: number; footprintBytes: number; systemAvailBytes?: number } = { + systemAvailBytes: ram.systemAvailBytes, availBytes: ram.availBytes, footprintBytes: ram.totalBytes - ram.availBytes, }; @@ -1284,6 +1330,7 @@ export function installNativeBoundary(opts: InstallOpts = {}): NativeBoundary { const freed = Math.min(memState.footprintBytes, 3 * GB); memState.footprintBytes -= freed; memState.availBytes += freed; + if (memState.systemAvailBytes !== undefined) memState.systemAvailBytes += freed; }; const litert = makeLiteRTFake(handle); @@ -1298,7 +1345,7 @@ export function installNativeBoundary(opts: InstallOpts = {}): NativeBoundary { // Scriptable llama.rn: override the global stub so completion output is under test control. const llamaFake = opts.llama - ? makeLlamaFake(freeModelMemory, opts.llamaChatTemplate, opts.llamaModelInfo, fsFake, opts.llamaVision) + ? makeLlamaFake(freeModelMemory, opts.llamaChatTemplate, opts.llamaModelInfo, fsFake, opts.llamaVision, opts.llamaAudio) : undefined; if (llamaFake) jest.doMock('llama.rn', () => llamaFake.module); @@ -1337,6 +1384,7 @@ export function installNativeBoundary(opts: InstallOpts = {}): NativeBoundary { // must be able to SEE the footprint drop, exactly as it does on device. getMemoryInfo: jest.fn(async () => ({ processAvailableBytes: memState.availBytes, + ...(memState.systemAvailBytes === undefined ? {} : { systemAvailableBytes: memState.systemAvailBytes }), footprintBytes: memState.footprintBytes, })), }; @@ -1406,6 +1454,7 @@ export function installNativeBoundary(opts: InstallOpts = {}): NativeBoundary { const setRam = (profile: RamProfile) => { memState.availBytes = profile.availBytes; + memState.systemAvailBytes = profile.systemAvailBytes; memState.footprintBytes = profile.totalBytes - profile.availBytes; (DeviceInfo.getTotalMemory as jest.Mock).mockResolvedValue( profile.totalBytes, diff --git a/__tests__/harness/nativeVideoEngine.ts b/__tests__/harness/nativeVideoEngine.ts new file mode 100644 index 000000000..6d8e183e3 --- /dev/null +++ b/__tests__/harness/nativeVideoEngine.ts @@ -0,0 +1,238 @@ +/** + * The phone's own video engine and film composer, at the native-module boundary. + * + * VideoGenerationModule (android/app/src/main/java/ai/offgridmobile/video/VideoGenerationModule.kt): + * `generate(input)` emits `VideoGenerationProgress` { stage, step, total, backend? } — preparing, + * conditioning (with the backend), each sampling step, decoding, encoding — writes the MP4 at + * `input.outputPath` and resolves { path }. `cancel()` stops it: the promise rejects with code + * VIDEO_CANCELLED and the file is discarded. A second generate while one runs rejects VIDEO_BUSY. + * `getVideoStatus(path)` reports the last job; getConstants() says it can start from a picture. + * + * OffgridFilm (pro/native/film/android/.../FilmModule.kt): `compose(input)` writes the film at + * `input.outputPath` and resolves { path, durationSeconds }; `lastFrame(video, out)` writes a PNG and + * resolves its path; the session calls keep a long render alive and resolve null. + * + * `hold()` parks the next clip after its first sampling step, so a test can see it in flight; + * `release()` lets it finish. `failNext(message)` makes the next clip fail as the engine reports it. + */ +import type { NativeBoundary } from './nativeBoundary'; + +export interface NativeVideoEngine { + /** Every generate input the engine received. */ + generations: Array>; + /** Every film the composer joined. */ + films: Array<{ clips: Array<{ path: string; caption?: string }>; outputPath: string }>; + hold: () => void; + release: () => void; + failNext: (message: string) => void; + /** From now on the engine also hands back its first decoded frame, as sd.cpp's preview callback does. */ + showPreviews: () => void; +} + +/** What the native side keeps across a JS reload: the job it is running, and its status. */ +interface EngineCore { + fs: NonNullable; + events: NativeBoundary['litertEvents']; + holdNext: boolean; + parked: (() => void) | null; + failure: string | null; + previews: boolean; + running: { reject: (error: Error) => void; output: string } | null; + status: Record | null; + /** The JS context that asked for the running job is still there to hear how it ends. */ + jsAttached: boolean; +} + +const cores = new WeakMap(); + +/** + * Install both modules. Call from launchApp's `beforeApp`, before anything requires Pro. + * + * `carryOver`: the app's JS was started again while the native side kept running (Android keeps the + * video worker alive under its foreground service). The engine keeps its job and status; the old JS + * context is gone, so the old promise is never settled, and the new launch finds the job by status. + */ +export function installNativeVideoEngine(boundary: NativeBoundary, carryOver?: NativeVideoEngine): NativeVideoEngine { + const RN = require('react-native'); + const previous = carryOver ? cores.get(carryOver) : undefined; + const core: EngineCore = previous ?? { + fs: boundary.fs!, + events: boundary.litertEvents, + holdNext: false, + parked: null, + failure: null, + previews: false, + running: null, + status: null, + jsAttached: true, + }; + core.fs = boundary.fs!; + core.events = boundary.litertEvents; + if (previous) core.jsAttached = false; + const engine: NativeVideoEngine = { + generations: carryOver?.generations ?? [], + films: carryOver?.films ?? [], + hold: () => { + core.holdNext = true; + }, + release: () => { + core.holdNext = false; + core.parked?.(); + core.parked = null; + }, + failNext: message => { + core.failure = message; + }, + showPreviews: () => { + core.previews = true; + }, + }; + cores.set(engine, core); + const coded = (code: string, message: string) => Object.assign(new Error(message), { code }); + const progress = (stage: string, step: number, total: number, backend?: string) => { + core.status = { ...core.status, stage, step, total, ...(backend ? { backend } : {}) }; + core.events.emit('VideoGenerationProgress', { stage, step, total, ...(backend ? { backend } : {}) }); + }; + + RN.NativeModules.VideoGenerationModule = { + getConstants: () => ({ supportsInitImage: true, supportsUpscale: false }), + addListener: () => undefined, + removeListeners: () => undefined, + getVideoStatus: async (outputPath: string) => (core.status && core.status.path === outputPath ? core.status : null), + cancel: async () => { + core.parked?.(); + core.parked = null; + core.running?.reject(coded('VIDEO_CANCELLED', 'Video generation stopped.')); + return null; + }, + generate: (input: Record) => + new Promise<{ path: string }>((resolve, reject) => { + if (core.running) { + reject(coded('VIDEO_BUSY', 'Video generation is already running.')); + return; + } + engine.generations.push(input); + const output = String(input.outputPath); + let settled = false; + const finish = (error?: Error) => { + if (settled) return; + settled = true; + core.running = null; + const heard = core.jsAttached; + if (error) { + core.status = { ...core.status, phase: 'failed', code: (error as Error & { code?: string }).code, error: error.message }; + if (heard) reject(error); + } else { + core.status = { ...core.status, phase: 'succeeded' }; + if (heard) resolve({ path: output }); + } + }; + core.jsAttached = true; + core.running = { reject: error => finish(error), output }; + core.status = { path: output, phase: 'running', stage: 'preparing', step: 0, total: 0 }; + const steps = Number(input.steps) || 4; + (async () => { + progress('preparing', 0, steps); + progress('conditioning', 0, 0, 'cpu'); + progress('generating', 0, steps); + progress('generating', 1, steps); + if (core.previews) { + // The native preview: a PNG beside the clip, announced on the encoding stage (VideoGenerationModule.preview). + const preview = { path: `${output}.preview.png`, width: Number(input.width) || 320, height: Number(input.height) || 192 }; + core.fs.seedFile(preview.path, 2048); + core.status = { ...core.status, stage: 'encoding', step: 0, total: 0, preview }; + core.events.emit('VideoGenerationProgress', { stage: 'encoding', step: 0, total: 0, preview }); + } + if (core.holdNext) { + core.holdNext = false; + await new Promise(go => { + core.parked = go; + }); + } + if (settled) return; + if (core.failure) { + const message = core.failure; + core.failure = null; + finish(coded('VIDEO_FAILED', message)); + return; + } + for (let step = 2; step <= steps; step += 1) progress('generating', step, steps); + progress('decoding', 1, 1); + progress('encoding', 0, 0); + // The clip is written where the app asked, on the disk of the launch that is running now. + core.fs.seedFile(output, 64 * 1024); + finish(); + })().catch(error => finish(error as Error)); + }), + }; + + RN.NativeModules.OffgridFilm = { + addListener: () => undefined, + removeListeners: () => undefined, + beginSession: async () => null, + updateSession: async () => null, + endSession: async () => null, + cancel: async () => null, + notifyReady: async () => null, + askToNotify: async () => true, + compose: async (input: { clips: Array<{ path: string; caption?: string }>; outputPath: string; crossfadeSeconds: number }) => { + engine.films.push(input); + core.fs.seedFile(input.outputPath, 256 * 1024); + return { path: input.outputPath, durationSeconds: input.clips.length * 5 - (input.clips.length - 1) * input.crossfadeSeconds }; + }, + lastFrame: async (_videoPath: string, outputPath: string) => { + core.fs.seedFile(outputPath, 4096); + return outputPath; + }, + }; + return engine; +} + +/** + * The phone's background downloads, finished on demand. The native module moves a finished file to + * where the app asked (moveCompletedDownload); the stand-in in nativeBoundary only answers the path, + * so this writes the file there at the size the download reported, as the device does. + */ +export interface PhoneDownloads { + /** Finish every file downloading now, one DownloadComplete each. Returns how many finished. */ + finishRunning: () => Promise; +} + +export function phoneDownloads(boundary: NativeBoundary, act: (work: () => Promise) => Promise): PhoneDownloads { + const download = boundary.download!; + const fs = boundary.fs!; + const sizes = new Map(); + const finished = new Set(); + const start = download.module.startDownload.getMockImplementation()!; + download.module.startDownload.mockImplementation(async (params: { totalBytes?: number }) => { + const row = await start(params); + sizes.set(row.downloadId, Number(params.totalBytes) || 1024); + return row; + }); + download.module.moveCompletedDownload.mockImplementation(async (id: string, target: string) => { + fs.seedFile(target, sizes.get(id) ?? 1024); + return target; + }); + return { + finishRunning: async () => { + const rows = download.active().filter(row => row.status === 'running' && !finished.has(row.downloadId)); + for (const row of rows) { + finished.add(row.downloadId); + const total = sizes.get(row.downloadId) ?? 1024; + await act(async () => { + download.events.emit('DownloadComplete', { + downloadId: row.downloadId, + fileName: row.fileName, + modelId: row.modelId, + bytesDownloaded: total, + totalBytes: total, + status: 'completed', + localUri: `/downloads/${row.fileName}`, + }); + await new Promise(resolve => setTimeout(resolve, 0)); + }); + } + return rows.length; + }, + }; +} diff --git a/__tests__/harness/postHogEndpoint.ts b/__tests__/harness/postHogEndpoint.ts new file mode 100644 index 000000000..b537e1d70 --- /dev/null +++ b/__tests__/harness/postHogEndpoint.ts @@ -0,0 +1,75 @@ +/** + * Fake PostHog Cloud EU endpoint — the network boundary for opt-in analytics. + * + * Installed at `fetch` (the SDK's only transport), it records every request to a PostHog host and + * answers like the real ingest API (`{ status: 1 }`). Every other URL still goes to the previous + * fetch (jest.setup's "network disabled" guard), so no test ever reaches the real host. Events are + * decoded exactly as the server would read them, including gzip bodies. + */ +import { gunzipSync } from 'node:zlib'; + +export interface PostHogEvent { + event: string; + distinct_id?: string; + properties: Record; +} + +export interface PostHogEndpoint { + /** Every URL the app requested on a PostHog host, in order. */ + readonly urls: string[]; + /** Every event delivered in a batch, in order. */ + readonly events: PostHogEvent[]; + /** The raw request bodies, decoded to text. */ + readonly bodies: string[]; + named(event: string): PostHogEvent[]; + restore(): void; +} + +const POSTHOG_HOST = /^https:\/\/[a-z0-9.-]*posthog\.com\//i; + +async function bodyBytes(body: unknown): Promise { + if (body === undefined || body === null) return Buffer.alloc(0); + if (typeof body === 'string') return Buffer.from(body, 'utf8'); + if (body instanceof Uint8Array) return Buffer.from(body); + if (body instanceof ArrayBuffer) return Buffer.from(new Uint8Array(body)); + const blob = body as { arrayBuffer?: () => Promise }; + if (typeof blob.arrayBuffer === 'function') return Buffer.from(new Uint8Array(await blob.arrayBuffer())); + return Buffer.from(String(body), 'utf8'); +} + +async function decode(body: unknown): Promise { + const bytes = await bodyBytes(body); + const gzipped = bytes.length > 2 && bytes[0] === 0x1f && bytes[1] === 0x8b; + return (gzipped ? gunzipSync(bytes) : bytes).toString('utf8'); +} + +export function installPostHogEndpoint(): PostHogEndpoint { + const urls: string[] = []; + const events: PostHogEvent[] = []; + const bodies: string[] = []; + const previous = globalThis.fetch; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + if (!POSTHOG_HOST.test(url)) return previous(input, init); + urls.push(url); + const text = await decode(init?.body); + bodies.push(text); + if (text) { + const payload = JSON.parse(text) as { batch?: PostHogEvent[] } & PostHogEvent; + for (const event of payload.batch ?? [payload]) events.push(event); + } + return new Response(JSON.stringify({ status: 1 }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof fetch; + return { + urls, + events, + bodies, + named: name => events.filter(event => event.event === name), + restore: () => { + globalThis.fetch = previous; + }, + }; +} diff --git a/__tests__/harness/videoStudioApp.ts b/__tests__/harness/videoStudioApp.ts new file mode 100644 index 000000000..302133598 --- /dev/null +++ b/__tests__/harness/videoStudioApp.ts @@ -0,0 +1,178 @@ +/** + * The real App with Pro, paired with Off Grid AI Desktop as its video computer, for Studio journeys. + * + * Device boundaries only (see analyticsApp): the llama.rn engine, the disk, the RAM sensor, and + * Desktop's gateway at `fetch` (its video jobs, its model manager and its film jobs). Nothing in the app is seeded: + * the computer's video model is picked on Home, and a clip is made the way a person makes one — + * video mode in a chat, then Send. + * + * Callers unmock '@react-navigation/native' so the studio screens open on the real navigator. + */ +import { launchApp, passOnboarding, restoreBuildFlavor, type AnalyticsApp } from './analyticsApp'; +import type { NativeBoundary } from './nativeBoundary'; +import { + DESKTOP_VIDEO_ENDPOINT, + DESKTOP_VIDEO_MODEL, + serveDesktopVideoJobs, + type DesktopVideoJobs, +} from './desktopVideoServer'; +import { serveDesktopModelPulls, type DesktopModelPulls } from './desktopModelPulls'; +import { serveDesktopStoryJobs, type DesktopStoryJobs } from './desktopStoryJobs'; + +export interface StudioApp extends AnalyticsApp { + jobs: DesktopVideoJobs; + models: DesktopModelPulls; + stories: DesktopStoryJobs; + /** The OS dialog on screen now (Alert.alert), or null. */ + dialog: () => OsDialog | null; + /** Tap a button on the OS dialog on screen, as a person would. */ + answer: (button: string) => Promise; +} + +export interface OsDialog { + title: string; + message?: string; + buttons: Array<{ text?: string; onPress?: () => void }>; +} + +let current: StudioApp | undefined; +/** PostHog endpoints of earlier launches in this test, put back last (they sit under the Desktop fakes). */ +let earlierEndpoints: Array<{ restore: () => void }> = []; + +/** The OS dialog is a device surface: remember what it offers so the test can answer it like a person. */ +function osDialog(rtl: AnalyticsApp['rtl']): Pick { + let dialog: OsDialog | null = null; + require('react-native').Alert.alert = (title: string, message?: string, buttons?: OsDialog['buttons']) => { + dialog = { title, message, buttons: buttons ?? [{ text: 'OK' }] }; + }; + const answer = async (button: string) => { + const shown = dialog; + if (!shown) throw new Error(`No dialog is on screen to answer "${button}".`); + const choice = shown.buttons.find(item => item.text === button); + if (!choice) throw new Error(`The dialog "${shown.title}" has no "${button}" button.`); + dialog = null; + await rtl.act(async () => { + choice.onPress?.(); + }); + }; + return { dialog: () => dialog, answer }; +} + +/** Home with Pro and the paired computer's video model picked. */ +export async function launchStudioApp( + computer: Parameters[0] = { memoryGb: 34 }, + options: { + imageModel?: boolean; + /** The phone's native video engine and film composer (a phone build that has them). */ + phoneEngine?: boolean; + } = {}, +): Promise { + const app = await launchApp({ + release: false, + beforeApp: boundary => { + if (options.imageModel) placeImageModel(boundary); + if (options.phoneEngine) require('./nativeVideoEngine').installNativeVideoEngine(boundary); + }, + }); + const { view, rtl, boundary } = app; + const jobs = serveDesktopVideoJobs(boundary.fs!); + const models = serveDesktopModelPulls(computer); + const stories = serveDesktopStoryJobs(boundary.fs!); + current = { ...app, jobs, models, stories, ...osDialog(rtl) }; + await passOnboarding(app, false); + await require('./proHarness').installPro(); + // What pairing leaves behind; the person picks the computer's video model on Home. + const { remoteServerManager } = require('../../src/services/remoteServerManager'); + const server = await remoteServerManager.addServer({ + name: 'Studio Mac', + endpoint: DESKTOP_VIDEO_ENDPOINT, + providerType: 'openai-compatible', + modelManagement: 'offgrid-desktop-v1', + modelCatalog: { video: [DESKTOP_VIDEO_MODEL] }, + }); + rtl.fireEvent.press(await view.findByTestId('model-summary-video-open')); + rtl.fireEvent.press( + await view.findByTestId(`remote-video-model-${server.id}:${DESKTOP_VIDEO_MODEL.id}`, {}, { timeout: 5000 }), + ); + return current; +} + +/** + * A downloaded image model, as a finished download leaves it: its files on disk (MNN, the Android + * runtime) and its record in storage. It is picked in the chat with the image-mode toggle. + */ +function placeImageModel(boundary: NativeBoundary): void { + const directory = `${boundary.fs!.DocumentDirectoryPath}/image_models/sd`; + for (const name of [ + 'pos_emb.bin', 'token_emb.bin', 'tokenizer.json', 'unet.mnn', 'unet.mnn.weight', + 'vae_decoder.mnn', 'vae_decoder.mnn.weight', 'clip_v2.mnn', 'clip_v2.mnn.weight', + ]) + boundary.fs!.seedFile(`${directory}/${name}`, 8 * 1024 * 1024); + const { createONNXImageModel } = require('../utils/factories'); + const model = createONNXImageModel({ id: 'sd', name: 'SD', modelPath: directory, backend: 'mnn' }); + const storage = require('@react-native-async-storage/async-storage'); + storage.setItem('@local_llm/downloaded_image_models', JSON.stringify([model])); +} + +/** From Home: a new chat, video mode, a description, Send — and the clip card it leaves. */ +export async function makeClipInChat(app: StudioApp, prompt: string): Promise { + const { view, rtl } = app; + rtl.fireEvent.press(await view.findByTestId('new-chat-button', {}, { timeout: 5000 })); + await view.findByTestId('chat-input'); + rtl.fireEvent.press(view.getByTestId('quick-settings-button')); + rtl.fireEvent.press(await view.findByTestId('quick-video-mode')); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), prompt); + rtl.fireEvent.press(view.getByTestId('send-button')); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); +} + +/** + * The app is closed and opened again: the phone's storage stays (its disk is a new one), and the + * paired computer keeps every job it was running. Opens on Home. + */ +export async function relaunchStudioApp(): Promise { + const before = current!; + before.view.unmount(); + earlierEndpoints.push(before.endpoint); + const app = await launchApp({ release: false, relaunch: true }); + before.jobs.useDisk(app.boundary.fs!); + before.stories.useDisk(app.boundary.fs!); + await require('./proHarness').installPro(); + current = { ...app, jobs: before.jobs, models: before.models, stories: before.stories, ...osDialog(app.rtl) }; + await app.view.findByTestId('home-screen', {}, { timeout: 10_000 }); + return current; +} + +/** How long the paired computer must go unasked before a test may put its fakes away (polls run every 1-3 s). */ +const QUIET_MS = 4000; + +/** + * afterEach: let every clip and film still on its way settle, then unmount and put every boundary + * back. Held jobs are released first, so nothing a test left running polls a restored `fetch` (the + * real network) after it ends — including a launch the test closed with relaunchStudioApp. + */ +export async function closeStudioApp(): Promise { + if (!current) return; + const app = current; + app.jobs.release(); + app.stories.release(); + app.models.release(); + const { videoSeam } = require('../../src/bootstrap/videoSeam'); + await app.rtl.waitFor(() => expect(videoSeam.hasPending() || videoSeam.isBusy()).toBe(false), { timeout: 30_000 }); + await app.rtl.waitFor( + () => expect(Date.now() - Math.max(app.jobs.lastRequestAt(), app.stories.lastRequestAt())).toBeGreaterThan(QUIET_MS), + { timeout: 30_000, interval: 250 }, + ); + // What ending Pro does: stop every render and the charger watch, so no timer outlives the test. + require('../../pro/video/studio/filmRenderer').stopAllRenders(); + app.view.unmount(); + app.endpoint.restore(); + app.stories.restore(); + app.models.restore(); + app.jobs.restore(); + for (const endpoint of earlierEndpoints.reverse()) endpoint.restore(); + earlierEndpoints = []; + require('../../src/stores/remoteServerStore').useRemoteServerStore.getState().clearAllServers(); + current = undefined; + restoreBuildFlavor(); +} diff --git a/__tests__/integration/analytics/analyticsOwner.test.ts b/__tests__/integration/analytics/analyticsOwner.test.ts new file mode 100644 index 000000000..ceab61b37 --- /dev/null +++ b/__tests__/integration/analytics/analyticsOwner.test.ts @@ -0,0 +1,405 @@ +/** + * The analytics owner over its real stores and the real SDK, with only device boundaries faked + * (PostHog at `fetch`, the JS runtime's global error handler, device-info, the disk). Each case + * reports a fact the way its seam does and checks what reaches PostHog — the network is the + * terminal surface for these events (docs/ANALYTICS.md). + */ +import { GB, installNativeBoundary, requireRTL, type NativeBoundary } from '../../harness/nativeBoundary'; +import { installPostHogEndpoint, type PostHogEndpoint } from '../../harness/postHogEndpoint'; +import { restoreBuildFlavor } from '../../harness/analyticsApp'; + +type Analytics = typeof import('../../../src/services/analytics/analytics'); + +interface Rig { + boundary: NativeBoundary; + endpoint: PostHogEndpoint; + analytics: Analytics; + stop: () => void; + flush: () => Promise; + errorHandler: () => (error: unknown, isFatal?: boolean) => void; +} + +let rig: Rig | undefined; + +afterEach(async () => { + rig?.stop(); + await new Promise(resolve => setTimeout(resolve, 50)); + rig?.endpoint.restore(); + rig = undefined; + restoreBuildFlavor(); +}); + +/** The JS runtime's global error handler (Hermes ErrorUtils), which the SDK hooks for crashes. */ +function installErrorUtils() { + let handler: (error: unknown, isFatal?: boolean) => void = () => {}; + (global as unknown as { ErrorUtils: unknown }).ErrorUtils = { + setGlobalHandler: (next: typeof handler) => { + handler = next; + }, + getGlobalHandler: () => handler, + }; + return () => handler; +} + +async function setup( + consent: { usage?: boolean; crashes?: boolean } | null, + device: { platform?: 'ios' | 'android'; deviceHardware?: string; socModel?: string } = {}, +): Promise { + (global as unknown as { __DEV__: boolean }).__DEV__ = false; + const boundary = installNativeBoundary({ + fs: true, + ram: { platform: device.platform ?? 'android', totalBytes: 8 * GB, availBytes: 4 * GB }, + deviceHardware: device.deviceHardware, + socModel: device.socModel, + }); + const errorHandler = installErrorUtils(); + const endpoint = installPostHogEndpoint(); + // The App starts analytics only once its settings are read from disk. + await require('../../../src/stores/appStore').useAppStore.persist.rehydrate(); + const analytics: Analytics = require('../../../src/services/analytics/analytics'); + const { featureSnapshot } = require('../../../src/services/analytics/featureSnapshot'); + const stop = analytics.startAnalytics(featureSnapshot); + if (consent) await analytics.setAnalyticsConsent(consent, 'settings'); + const flush = async () => { + boundary.emitAppStateChange('background'); + await new Promise(resolve => setTimeout(resolve, 30)); + }; + rig = { boundary, endpoint, analytics, stop, flush, errorHandler }; + return rig; +} + +const waitFor = async (check: () => void) => { + await requireRTL().waitFor(check); +}; + +it('with Crash reports only, usage events are dropped and a crash arrives scrubbed', async () => { + const { analytics, endpoint, flush, errorHandler } = await setup({ crashes: true }); + analytics.featureUsed('tools'); + const error = new TypeError('Cannot open "/Users/me/Diary.txt": my secret words'); + error.stack = 'TypeError: Cannot open\n at openFile (/Users/me/app/index.bundle:10:3)'; + errorHandler()(error, false); + await flush(); + await waitFor(() => expect(endpoint.named('$exception')).toHaveLength(1)); + expect(endpoint.named('feature_used')).toEqual([]); + expect(endpoint.named('analytics_opted_in')).toEqual([]); + const crash = endpoint.named('$exception')[0].properties; + const sent = JSON.stringify(crash); + expect(sent).not.toMatch(/secret|Diary|\/Users\/me/); + expect(crash.platform).toBe('android'); + const [exception] = crash.$exception_list as Array<{ type: string; value: string }>; + expect(exception).toMatchObject({ type: 'TypeError', value: 'Cannot open' }); +}); + +it('turning Crash reports off keeps usage flowing on a rebuilt client with the same ID', async () => { + const { analytics, endpoint, flush } = await setup({ usage: true, crashes: true }); + analytics.track('chat_created'); + await flush(); + await waitFor(() => expect(endpoint.named('chat_created')).toHaveLength(1)); + await analytics.setAnalyticsConsent({ crashes: false }, 'settings'); + analytics.track('project_created'); + await flush(); + await waitFor(() => expect(endpoint.named('project_created')).toHaveLength(1)); + expect(endpoint.named('project_created')[0].distinct_id).toBe(endpoint.named('chat_created')[0].distinct_id); +}); + +it('reports every action, flow and Pro funnel step with only the spec properties', async () => { + const { analytics, endpoint, flush } = await setup({ usage: true }); + analytics.featureUsed('web_search'); + analytics.documentAdded('pdf', 3 * GB); + analytics.track('problem_reported'); + analytics.flowStep('model_transfer', 'device_picked'); + analytics.flowFinished('model_transfer', 'done'); + analytics.flowStarted('connector_add', 'opened'); + analytics.flowStep('connector_add', 'opened'); + analytics.flowStep('connector_add', 'connector_picked', { connector: 'custom' }); + analytics.flowFinished('connector_add', 'failed', Object.assign(new Error('x'), { code: 'oauth_denied' })); + analytics.flowStarted('sync_pairing'); + analytics.flowFinished('sync_pairing', 'cancelled'); + analytics.proGateShown('sync'); + analytics.proPaywallViewed(); + analytics.proPurchased(); + analytics.proPaywallViewed(); + analytics.proPurchased(); + await flush(); + await waitFor(() => expect(endpoint.named('flow_finished')).toHaveLength(4)); + expect(endpoint.named('feature_used')[0].properties.feature).toBe('web_search'); + expect(endpoint.named('document_added')[0].properties).toMatchObject({ file_type: 'pdf', size_bucket: '2-4GB' }); + expect(endpoint.named('problem_reported')).toHaveLength(1); + const finished = endpoint.named('flow_finished').map(e => e.properties); + expect(finished.map(f => [f.flow, f.outcome, f.failed_step, f.error_code])).toEqual([ + ['connector_add', 'failed', 'connector_picked', 'oauth_denied'], + ['sync_pairing', 'cancelled', undefined, undefined], + ['pro_upgrade', 'done', undefined, undefined], + ['pro_upgrade', 'done', undefined, undefined], + ]); + const steps = endpoint.named('flow_step').map(e => [e.properties.flow, e.properties.step, e.properties.feature ?? e.properties.connector]); + expect(steps).toEqual([ + ['connector_add', 'opened', undefined], + ['connector_add', 'connector_picked', 'custom'], + ['pro_upgrade', 'gate_shown', 'sync'], + ['pro_upgrade', 'paywall_viewed', undefined], + ['pro_upgrade', 'purchased', undefined], + ['pro_upgrade', 'paywall_viewed', undefined], + ['pro_upgrade', 'purchased', undefined], + ]); +}); + +it('downloads report start, finish speed, failure, and the image, voice and video setup funnels', async () => { + const { analytics, endpoint, flush } = await setup({ usage: true }); + const { useDownloadStore } = require('../../../src/stores/downloadStore'); + const entry = (modelKey: string, modelType: string, downloadId: string) => ({ + modelKey, downloadId, modelId: modelType === 'image' ? 'anythingv5_cpu' : 'ggml-org/SmolVLM-GGUF', + fileName: 'f', quantization: modelType === 'text' ? 'Q4_K_M' : '', modelType, status: 'pending', + bytesDownloaded: 0, totalBytes: 1.5 * GB, combinedTotalBytes: 0, progress: 0, createdAt: 0, + }); + analytics.flowStarted('image_setup', 'opened'); + useDownloadStore.getState().add(entry('image:a', 'image', 'd1')); + useDownloadStore.getState().setCompleted('d1'); + useDownloadStore.getState().add(entry('text:b', 'text', 'd2')); + useDownloadStore.getState().setStatus('d2', 'failed', { message: 'disk full at /data/x', code: 'disk_full' }); + useDownloadStore.getState().retryEntry('text:b', 'd3'); + useDownloadStore.getState().setStatus('d3', 'completed'); + analytics.downloadEnded('never-started', 'done'); + useDownloadStore.getState().setCompleted('unknown-id'); + analytics.flowStarted('voice_setup', 'opened'); + analytics.downloadStarted('tts:kokoro', { model: 'kokoro', modelType: 'tts' }); + analytics.downloadEnded('tts:kokoro', 'done'); + analytics.flowStarted('video_setup', 'opened'); + analytics.downloadStarted('video:wan', { model: 'Comfy-Org/Wan_2.1_ComfyUI_repackaged', modelType: 'video' }); + analytics.downloadEnded('video:wan', 'done'); + await new Promise(resolve => setTimeout(resolve, 20)); + await flush(); + await waitFor(() => expect(endpoint.named('model_download_finished')).toHaveLength(4)); + expect(endpoint.named('model_download_started')).toHaveLength(5); + const failed = endpoint.named('model_download_failed')[0].properties; + expect(failed).toMatchObject({ model: 'ggml-org/SmolVLM-GGUF', quant: 'Q4_K_M', size_bucket: '1-2GB', network: 'unknown', error_code: 'disk_full' }); + expect(JSON.stringify(failed)).not.toContain('/data/x'); + const done = endpoint.named('model_download_finished')[0].properties; + expect(done.model).toBe('anythingv5_cpu'); + expect(typeof done.free_storage_gb).toBe('number'); + expect(endpoint.named('model_download_finished')[2].properties.mb_per_sec).toBeUndefined(); + const funnels = endpoint.named('flow_finished').map(e => e.properties.flow); + expect(funnels).toEqual(['image_setup', 'voice_setup', 'video_setup']); +}); + +it('reports the spec properties when the device cannot say its chip, memory or battery', async () => { + const { analytics, endpoint, flush } = await setup(null); + // Before opting in a download is not tracked at all, so its end later sends nothing. + analytics.downloadStarted('text:early', { model: 'custom', modelType: 'text' }); + const DeviceInfo = require('react-native-device-info'); + DeviceInfo.getTotalMemory.mockRejectedValueOnce(new Error('unavailable')); + DeviceInfo.getPowerState.mockRejectedValue(new Error('unavailable')); + DeviceInfo.getHardware.mockRejectedValueOnce(new Error('unavailable')); + await analytics.setAnalyticsConsent({ usage: true }, 'settings'); + analytics.downloadEnded('text:early', 'done'); + await expect(analytics.trackModelLoad({ model: 'custom', engine: 'llama.cpp', backend: 'cpu' }, async () => 'ok')).resolves.toBe('ok'); + analytics.generationStarted('image', { model: 'custom', engine: 'onnx', backend: 'npu' })('stopped'); + await new Promise(resolve => setTimeout(resolve, 20)); + await flush(); + await waitFor(() => expect(endpoint.named('generation_finished')).toHaveLength(1)); + await waitFor(() => expect(endpoint.named('model_loaded')).toHaveLength(1)); + expect(endpoint.named('model_loaded')[0].properties.soc).toBe('unknown'); + const generation = endpoint.named('generation_finished')[0].properties; + expect(generation).toMatchObject({ modality: 'image', outcome: 'stopped' }); + expect(generation.battery_pct).toBeUndefined(); + expect(generation.charging).toBeUndefined(); + expect(endpoint.named('model_download_finished')).toEqual([]); + DeviceInfo.getPowerState.mockResolvedValue({ batteryLevel: 0.64, batteryState: 'unplugged', lowPowerMode: false }); +}); + +it('a cold launch or a system dialog (inactive, then active) is not a new app open', async () => { + const { endpoint, flush, boundary } = await setup({ usage: true }); + boundary.emitAppStateChange('inactive'); + boundary.emitAppStateChange('active'); + boundary.emitAppStateChange('inactive'); + boundary.emitAppStateChange('active'); + await flush(); + await waitFor(() => expect(endpoint.named('app_backgrounded')).toHaveLength(1)); + expect(endpoint.named('app_opened')).toHaveLength(1); + boundary.emitAppStateChange('active'); + await flush(); + await waitFor(() => expect(endpoint.named('app_backgrounded')).toHaveLength(2)); + expect(endpoint.named('app_opened')).toHaveLength(2); +}); + +it('sends a daily features snapshot once per day and survives unreadable storage', async () => { + const { analytics, endpoint, flush, boundary } = await setup({ usage: true }); + const AsyncStorage = require('@react-native-async-storage/async-storage'); + boundary.emitAppStateChange('active'); + boundary.emitAppStateChange('background'); + boundary.emitAppStateChange('active'); + analytics.trackScreen('HomeTab'); + boundary.emitAppStateChange('background'); + boundary.emitAppStateChange('background'); + boundary.emitAppStateChange('active'); + analytics.trackScreen('HomeTab'); + analytics.trackScreen(undefined); + analytics.trackScreen('Settings'); + AsyncStorage.getItem.mockRejectedValueOnce(new Error('unreadable')); + AsyncStorage.setItem.mockRejectedValueOnce(new Error('unwritable')); + await AsyncStorage.removeItem('offgrid.analytics.snapshotDay'); + boundary.emitAppStateChange('background'); + boundary.emitAppStateChange('active'); + await new Promise(resolve => setTimeout(resolve, 20)); + await flush(); + await waitFor(() => expect(endpoint.named('$screen')).toHaveLength(2)); + const snapshots = endpoint.named('features_snapshot'); + expect(snapshots.length).toBeGreaterThanOrEqual(1); + expect(snapshots[0].properties).toMatchObject({ voice_input: false, image_generation: false, vision: false, web_search: true, tools: true, remote_server: false }); + const [home, settings] = endpoint.named('$screen').map(e => e.properties); + expect(home).toMatchObject({ screen: 'HomeTab' }); + expect(home.prev_screen).toBeUndefined(); + expect(settings).toMatchObject({ screen: 'Settings', prev_screen: 'HomeTab' }); + expect(typeof settings.prev_ms).toBe('number'); +}); + +it('withdrawing Usage before anything is queued still lets a crash arrive', async () => { + // Both were turned on in an earlier launch, so this launch queues no opt-in: the queue is empty. + const AsyncStorage = require('@react-native-async-storage/async-storage'); + await AsyncStorage.setItem( + 'local-llm-app-storage', + JSON.stringify({ state: { settings: { analyticsUsage: true, analyticsCrashes: true } }, version: 0 }), + ); + const { analytics, endpoint, flush, errorHandler } = await setup(null); + await analytics.setAnalyticsConsent({ usage: false }, 'settings'); + errorHandler()(new TypeError('Cannot read the file'), false); + await flush(); + await waitFor(() => expect(endpoint.named('$exception')).toHaveLength(1)); + expect(endpoint.named('feature_used')).toEqual([]); +}); + +it('withdrawing Usage drops a queued entry that names no event, and a crash still arrives', async () => { + // The SDK's queue from the last launch, on disk: its queue item type allows an entry with no message. + const AsyncStorage = require('@react-native-async-storage/async-storage'); + await AsyncStorage.setItem('.posthog-rn.json', JSON.stringify({ version: 'v1', content: { queue: [{}] } })); + const { analytics, endpoint, flush, errorHandler } = await setup({ usage: true, crashes: true }); + await analytics.setAnalyticsConsent({ usage: false }, 'settings'); + errorHandler()(new TypeError('Cannot read the file'), false); + await flush(); + await waitFor(() => expect(endpoint.named('$exception')).toHaveLength(1)); + expect(endpoint.named('chat_created')).toEqual([]); +}); + +it('a corrupt unclean-exit marker from the last launch is ignored and cleared', async () => { + const AsyncStorage = require('@react-native-async-storage/async-storage'); + await AsyncStorage.setItem('offgrid.analytics.uncleanExit', '{not json'); + const { endpoint, flush } = await setup({ usage: true, crashes: true }); + await new Promise(resolve => setTimeout(resolve, 20)); + await flush(); + await waitFor(() => expect(endpoint.named('analytics_opted_in')).toHaveLength(1)); + expect(endpoint.named('unclean_exit')).toEqual([]); + expect(await AsyncStorage.getItem('offgrid.analytics.uncleanExit')).toBeNull(); +}); + +it('every event says tablet, Pro and the chip as the device reports them', async () => { + const { analytics, endpoint, flush } = await setup(null, { platform: 'ios' }); + const DeviceInfo = require('react-native-device-info'); + DeviceInfo.isTablet.mockReturnValue(true); + const { useAppStore } = require('../../../src/stores/appStore'); + // A Pro change before opting in has no client to tell. + useAppStore.getState().setHasRegisteredPro(true); + await analytics.setAnalyticsConsent({ usage: true }, 'settings'); + useAppStore.getState().setHasRegisteredPro(false); + useAppStore.getState().setHasRegisteredPro(true); + await analytics.trackModelLoad({ model: 'custom', engine: 'llama.cpp', backend: 'gpu' }, async () => undefined); + await new Promise(resolve => setTimeout(resolve, 20)); + await flush(); + await waitFor(() => expect(endpoint.named('model_loaded')).toHaveLength(1)); + DeviceInfo.isTablet.mockReturnValue(false); + const loaded = endpoint.named('model_loaded')[0].properties; + expect(loaded).toMatchObject({ device_class: 'tablet', pro: true, platform: 'ios' }); + expect(String(loaded.soc)).toMatch(/^A1\d/); +}); + +it('a Snapdragon with an NPU names its generation, and a load failure is reported and rethrown', async () => { + const { analytics, endpoint, flush } = await setup({ usage: true }, { deviceHardware: 'qcom', socModel: 'SM8650' }); + const failure = Object.assign(new Error('out of memory at /data/x'), { code: 'oom' }); + await expect( + analytics.trackModelLoad({ model: 'custom', engine: 'llama.cpp', backend: 'npu' }, async () => { + throw failure; + }), + ).rejects.toBe(failure); + await new Promise(resolve => setTimeout(resolve, 20)); + await flush(); + await waitFor(() => expect(endpoint.named('model_load_failed')).toHaveLength(1)); + const failed = endpoint.named('model_load_failed')[0].properties; + expect(failed.soc).toBe('qualcomm-8gen2'); + expect(failed.error_code).toBe('oom'); +}); + +it('each generation ends once; voice generations finish without an unclean-exit marker', async () => { + const { analytics, endpoint, flush } = await setup({ usage: true }); + const AsyncStorage = require('@react-native-async-storage/async-storage'); + const text = analytics.generationStarted('text', { model: 'custom', engine: 'llama.cpp', backend: 'cpu' }); + let finishLoad: () => void = () => {}; + const load = analytics.trackModelLoad({ model: 'custom', engine: 'llama.cpp', backend: 'cpu' }, () => + new Promise(resolve => { + finishLoad = resolve; + }), + ); + // The reply ends while a newer load holds the marker: the load's marker stays. + text('failed', { error: Object.assign(new Error('x'), { code: 'context_full' }) }); + text('done'); + await new Promise(resolve => setTimeout(resolve, 10)); + expect(JSON.parse(await AsyncStorage.getItem('offgrid.analytics.uncleanExit')).stage).toBe('load'); + finishLoad(); + await load; + const stt = analytics.generationStarted('stt', { model: 'base.en', engine: 'whisper', backend: 'cpu' }); + expect(await AsyncStorage.getItem('offgrid.analytics.uncleanExit')).toBeNull(); + stt('done'); + await new Promise(resolve => setTimeout(resolve, 20)); + await flush(); + await waitFor(() => expect(endpoint.named('generation_finished')).toHaveLength(2)); + const [first, voice] = endpoint.named('generation_finished').map(e => e.properties); + expect(first).toMatchObject({ modality: 'text', outcome: 'failed', error_code: 'context_full' }); + expect(voice).toMatchObject({ modality: 'stt', outcome: 'done' }); + expect(voice.error_code).toBeUndefined(); +}); + +it('time on a screen stops while the app is in the background', async () => { + const { analytics, endpoint, flush, boundary } = await setup({ usage: true }); + boundary.emitAppStateChange('background'); + boundary.emitAppStateChange('active'); + analytics.trackScreen('HomeTab'); + boundary.emitAppStateChange('background'); + analytics.trackScreen('Chat'); + await flush(); + await waitFor(() => expect(endpoint.named('$screen')).toHaveLength(2)); + expect(endpoint.named('$screen')[1].properties).toMatchObject({ screen: 'Chat', prev_screen: 'HomeTab' }); +}); + +it('a failed model download asks someone who has not opted in, and sends nothing', async () => { + const { endpoint, flush } = await setup(null); + const React = require('react'); + const { render, act } = requireRTL(); + const { AnalyticsConsentPrompt } = require('../../../src/components/AnalyticsConsentPrompt'); + const prompt = render(React.createElement(AnalyticsConsentPrompt)); + expect(prompt.queryByText('Help us fix this?')).toBeNull(); + const { useDownloadStore } = require('../../../src/stores/downloadStore'); + await act(async () => { + useDownloadStore.getState().add({ + modelKey: 'text:b', downloadId: 'd9', modelId: 'ggml-org/SmolVLM-GGUF', fileName: 'f', quantization: 'Q4_K_M', + modelType: 'text', status: 'pending', bytesDownloaded: 0, totalBytes: GB, combinedTotalBytes: 0, progress: 0, createdAt: 0, + }); + useDownloadStore.getState().setStatus('d9', 'failed', { message: 'network lost', code: 'network' }); + }); + expect(await prompt.findByText('Help us fix this?')).toBeTruthy(); + await flush(); + expect(endpoint.named('model_download_failed')).toHaveLength(0); + prompt.unmount(); +}); + +it('withdrawing Usage drops usage events still queued, and a crash still arrives on the same ID', async () => { + const { analytics, endpoint, flush, errorHandler } = await setup({ usage: true, crashes: true }); + analytics.track('project_created'); + await flush(); + await waitFor(() => expect(endpoint.named('project_created')).toHaveLength(1)); + // Taken while Usage was on, still queued when it is turned off. + analytics.track('chat_created'); + await analytics.setAnalyticsConsent({ usage: false }, 'settings'); + errorHandler()(new TypeError('Cannot read the file'), false); + await flush(); + await waitFor(() => expect(endpoint.named('$exception')).toHaveLength(1)); + expect(endpoint.named('chat_created')).toEqual([]); + expect(endpoint.named('$exception')[0].distinct_id).toBe(endpoint.named('project_created')[0].distinct_id); +}); diff --git a/__tests__/integration/analytics/consentPrompts.rendered.test.tsx b/__tests__/integration/analytics/consentPrompts.rendered.test.tsx new file mode 100644 index 000000000..d644d2c4c --- /dev/null +++ b/__tests__/integration/analytics/consentPrompts.rendered.test.tsx @@ -0,0 +1,125 @@ +/** + * Consent prompts outside Auto Setup (docs/ANALYTICS.md, Consent). + * + * SPEC (OGAM user's view): + * - Someone who installed before this version and was never asked sees "Help us fix what breaks" + * once, after their first successful reply. "Share" turns analytics on (source + * `first_chat_prompt`). It never shows again. + * - When a model fails to load and the person has not opted in, "Help us fix this?" asks once; + * after "Not now" it does not ask again for 30 days. "Share" turns analytics on (source + * `failure_prompt`). + * - "What we share" lists what is and is not sent. + */ +import { + launchApp, + openNewChat, + restoreBuildFlavor, + sendMessage, + sendToBackground, + type AnalyticsApp, +} from '../../harness/analyticsApp'; + +jest.unmock('@react-navigation/native'); + +let app: AnalyticsApp | undefined; + +afterEach(async () => { + await require('../../../src/services/generationService').generationService.stopGeneration(); + app?.view.unmount(); + await new Promise(resolve => setTimeout(resolve, 50)); + app?.endpoint.restore(); + app = undefined; + restoreBuildFlavor(); +}); + +async function newChatFromHome(current: AnalyticsApp): Promise { + current.rtl.fireEvent.press(current.view.getByLabelText('Back')); + current.rtl.fireEvent.press(await current.view.findByTestId('new-chat-button')); + await current.view.findByTestId('chat-input'); +} + +async function failToLoad(current: AnalyticsApp): Promise { + const { view, rtl, boundary } = current; + boundary.llama!.scriptInitFailure(true); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), 'hello'); + rtl.fireEvent.press(view.getByTestId('send-button')); + await rtl.waitFor(() => expect(boundary.llama!.calls.initLlama.length).toBeGreaterThan(0)); +} + +it('an existing user is asked once after their first reply, and Share opts them in', async () => { + app = await launchApp({ release: true, returningUser: true }); + const current = app; + const { view, rtl, endpoint } = current; + await view.findByTestId('home-screen'); + await openNewChat(current); + expect(view.queryByText('Help us fix what breaks')).toBeNull(); + await sendMessage(current, 'first', 'First reply.'); + expect(await view.findByText('Help us fix what breaks')).toBeTruthy(); + expect(view.getByText('Send anonymous crash reports and usage. Your chats stay on this device.')).toBeTruthy(); + rtl.fireEvent.press(view.getByLabelText('Share')); + await rtl.waitFor(() => expect(view.queryByText('Help us fix what breaks')).toBeNull()); + + await newChatFromHome(current); + await sendMessage(current, 'second', 'Second reply.'); + await sendToBackground(current); + await rtl.waitFor(() => expect(endpoint.named('generation_finished')).toHaveLength(1)); + expect(view.queryByText('Help us fix what breaks')).toBeNull(); + expect(endpoint.named('analytics_opted_in').map(e => e.properties.source)).toEqual(['first_chat_prompt']); +}); + +it('a failed model load asks once; Not now keeps analytics off and the next failure does not ask', async () => { + app = await launchApp({ release: true, returningUser: true }); + const current = app; + const { view, rtl, endpoint } = current; + await view.findByTestId('home-screen'); + await openNewChat(current); + expect(view.queryByText('Help us fix this?')).toBeNull(); + await failToLoad(current); + expect(await view.findByText('Help us fix this?')).toBeTruthy(); + rtl.fireEvent.press(view.getByLabelText('Not now')); + await rtl.waitFor(() => expect(view.queryByText('Help us fix this?')).toBeNull()); + + // The failed load deselected the model: pick it again from Home and send again. + const loadsBefore = current.boundary.llama!.calls.initLlama.length; + rtl.fireEvent.press(view.getByLabelText('Back')); + await openNewChat(current); + await failToLoad(current); + await rtl.waitFor(() => expect(current.boundary.llama!.calls.initLlama.length).toBeGreaterThan(loadsBefore)); + expect(view.queryByText('Help us fix this?')).toBeNull(); + await sendToBackground(current); + await new Promise(resolve => setTimeout(resolve, 200)); + expect(endpoint.urls).toEqual([]); +}); + +it('Share on the failure prompt opts in with source failure_prompt', async () => { + app = await launchApp({ release: true, returningUser: true }); + const current = app; + const { view, rtl, endpoint } = current; + await view.findByTestId('home-screen'); + await openNewChat(current); + await failToLoad(current); + rtl.fireEvent.press(await view.findByLabelText('Share')); + await rtl.waitFor(() => expect(view.queryByText('Help us fix this?')).toBeNull()); + await sendToBackground(current); + await rtl.waitFor(() => expect(endpoint.named('analytics_opted_in')).toHaveLength(1)); + expect(endpoint.named('analytics_opted_in')[0].properties.source).toBe('failure_prompt'); +}); + +it('What we share, from Auto Setup, lists what is and is not sent', async () => { + app = await launchApp({ release: false }); + const { view, rtl } = app; + rtl.fireEvent.press(await view.findByTestId('onboarding-skip')); + rtl.fireEvent.press(await view.findByTestId('auto-setup-what-we-share', {}, { timeout: 10_000 })); + expect(await view.findByTestId('analytics-we-share')).toBeTruthy(); + expect(view.getByText('Your chats, prompts, replies, transcripts or reasoning.')).toBeTruthy(); + expect(view.getByTestId('analytics-we-never-share')).toBeTruthy(); + rtl.fireEvent.press(view.getByLabelText('Back')); + await rtl.waitFor(() => expect(view.queryByTestId('analytics-we-share')).toBeNull()); + expect(view.getByTestId('auto-setup-skip')).toBeTruthy(); + await passOnboardingFromSetup(app); +}); + +async function passOnboardingFromSetup(current: AnalyticsApp): Promise { + current.rtl.fireEvent.press(current.view.getByTestId('auto-setup-skip')); + await current.view.findByTestId('home-screen'); +} diff --git a/__tests__/integration/analytics/consentRulesAndFacts.test.ts b/__tests__/integration/analytics/consentRulesAndFacts.test.ts new file mode 100644 index 000000000..0cd5b955c --- /dev/null +++ b/__tests__/integration/analytics/consentRulesAndFacts.test.ts @@ -0,0 +1,114 @@ +/** + * The consent-prompt rules, the features snapshot and the model facts, over the real stores + * (docs/ANALYTICS.md: Consent, `features_snapshot`, Shared value rules). The rendered prompts are + * covered by consentPrompts.rendered.test.tsx; these are the edges the journeys do not reach. + */ +import { Platform } from 'react-native'; +import { useAppStore } from '../../../src/stores/appStore'; +import { useRemoteServerStore } from '../../../src/stores/remoteServerStore'; +import { registerHook, HOOKS, _clearHooksForTesting } from '../../../src/bootstrap/hookRegistry'; +import { + closeConsentPrompt, + maybePromptAfterFailure, + maybePromptAfterFirstChat, + useConsentPromptStore, +} from '../../../src/services/analytics/consentPrompt'; +import { featureSnapshot } from '../../../src/services/analytics/featureSnapshot'; +import { imageModelFacts, textModelFacts, videoModelFacts, REMOTE_MODEL_FACTS } from '../../../src/services/analytics/modelFacts'; +import { createDownloadedModel } from '../../utils/factories'; + +const DAY = 24 * 60 * 60 * 1000; + +beforeEach(() => { + closeConsentPrompt(); + useAppStore.getState().resetSettings(); + useAppStore.getState().updateSettings({ analyticsAsked: false, analyticsLastPromptAt: null }); + _clearHooksForTesting(); +}); + +describe('consent prompt rules', () => { + it('asks after a failure once, then waits 30 days', () => { + expect(useConsentPromptStore.getState().kind).toBeNull(); + maybePromptAfterFailure(); + expect(useConsentPromptStore.getState().kind).toBe('failure'); + closeConsentPrompt(); + maybePromptAfterFailure(); + expect(useConsentPromptStore.getState().kind).toBeNull(); + useAppStore.getState().updateSettings({ analyticsLastPromptAt: Date.now() - 31 * DAY }); + maybePromptAfterFailure(); + expect(useConsentPromptStore.getState().kind).toBe('failure'); + }); + + it('one prompt at a time, and never for someone already opted in', () => { + maybePromptAfterFailure(); + maybePromptAfterFirstChat(); + expect(useConsentPromptStore.getState().kind).toBe('failure'); + closeConsentPrompt(); + useAppStore.getState().updateSettings({ analyticsCrashes: true, analyticsLastPromptAt: null, analyticsAsked: false }); + maybePromptAfterFailure(); + maybePromptAfterFirstChat(); + expect(useConsentPromptStore.getState().kind).toBeNull(); + }); + + it('the first-chat prompt is once ever', () => { + maybePromptAfterFirstChat(); + expect(useConsentPromptStore.getState().kind).toBe('first_chat'); + closeConsentPrompt(); + maybePromptAfterFirstChat(); + expect(useConsentPromptStore.getState().kind).toBeNull(); + }); +}); + +describe('features snapshot', () => { + it('reports each feature as switched on or off, Pro features included', () => { + useAppStore.getState().setDownloadedModels([]); + useAppStore.getState().setActiveModelId(null); + useAppStore.getState().updateSettings({ enabledTools: [] }); + expect(featureSnapshot()).toEqual({ + voice_input: false, image_generation: false, video_generation: false, vision: false, web_search: false, tools: false, + remote_server: false, + }); + + const llamaVision = createDownloadedModel({ id: 'v', engine: 'llama', isVisionModel: true }); + const litert = { ...createDownloadedModel({ id: 'l', engine: 'litert' }), liteRTVision: true }; + useAppStore.getState().setDownloadedModels([llamaVision, litert]); + useAppStore.getState().setActiveModelId('v'); + useAppStore.getState().setActiveImageModelId('img'); + useAppStore.getState().setActiveVideoModelId('Comfy-Org/Wan_2.1_ComfyUI_repackaged'); + useAppStore.getState().updateSettings({ enabledTools: ['web_search'] }); + useRemoteServerStore.getState().addServer({ name: 'Mac', endpoint: 'http://192.168.1.2:1234', providerType: 'openai-compatible' }); + registerHook(HOOKS.analyticsProFeatures, () => ({ tts: true, sync: false })); + expect(featureSnapshot()).toMatchObject({ + image_generation: true, video_generation: true, vision: true, web_search: true, tools: true, remote_server: true, tts: true, sync: false, + }); + useAppStore.getState().setActiveModelId('l'); + expect(featureSnapshot().vision).toBe(true); + }); +}); + +describe('model facts', () => { + it('names the route without any file name', () => { + expect(textModelFacts(true)).toEqual(REMOTE_MODEL_FACTS); + useAppStore.getState().setActiveModelId('missing'); + expect(textModelFacts(false)).toMatchObject({ model: 'custom', engine: 'llama.cpp' }); + useAppStore.getState().setDownloadedModels([ + { ...createDownloadedModel({ id: 'local_import/Family Recipes.gguf', engine: 'litert', quantization: '' }) }, + ]); + useAppStore.getState().setActiveModelId('local_import/Family Recipes.gguf'); + useAppStore.getState().updateSettings({ liteRTBackend: 'npu' }); + expect(textModelFacts(false)).toEqual({ model: 'custom', quant: undefined, engine: 'litert', backend: 'npu' }); + + expect(imageModelFacts({ id: 'sd_qnn', backend: 'qnn' } as never)).toEqual({ model: 'sd_qnn', engine: 'onnx', backend: 'npu' }); + expect(imageModelFacts({ id: 'sd_coreml', backend: 'coreml' } as never)).toEqual({ model: 'sd_coreml', engine: 'coreml', backend: 'gpu' }); + expect(imageModelFacts({ id: 'sd-flux', backend: 'sd' } as never)).toEqual({ model: 'sd-flux', engine: 'sd.cpp', backend: 'gpu' }); + expect(videoModelFacts('Comfy-Org/Wan_2.1_ComfyUI_repackaged')).toEqual({ + model: 'Comfy-Org/Wan_2.1_ComfyUI_repackaged', engine: 'sd.cpp', backend: 'gpu', + }); + const os = Platform.OS; + Object.defineProperty(Platform, 'OS', { value: 'ios', configurable: true }); + expect(imageModelFacts(undefined)).toEqual({ model: 'custom', engine: 'coreml', backend: 'cpu' }); + Object.defineProperty(Platform, 'OS', { value: 'android', configurable: true }); + expect(imageModelFacts(undefined).engine).toBe('onnx'); + Object.defineProperty(Platform, 'OS', { value: os, configurable: true }); + }); +}); diff --git a/__tests__/integration/analytics/optInFromAutoSetup.rendered.test.tsx b/__tests__/integration/analytics/optInFromAutoSetup.rendered.test.tsx new file mode 100644 index 000000000..55c144d5f --- /dev/null +++ b/__tests__/integration/analytics/optInFromAutoSetup.rendered.test.tsx @@ -0,0 +1,86 @@ +/** + * Opt-in analytics from Auto Setup (docs/ANALYTICS.md, Rules + Consent). + * + * SPEC (OGAM user's view): analytics is off by default. A person who passes Auto Setup without + * ticking "Share anonymous usage and crash reports" and then chats sends nothing to PostHog. A + * person who ticks it sends `analytics_opted_in` (source `setup`) and a `generation_finished` for + * the reply, and neither their prompt nor the reply ever leaves the device. + * + * The real App runs as a store build. Only device boundaries are faked; the PostHog endpoint is + * faked at `fetch`, and what reaches it is the terminal artifact (the network is the surface). + */ +import { + launchApp, + openNewChat, + passOnboarding, + restoreBuildFlavor, + sendMessage, + sendToBackground, + type AnalyticsApp, +} from '../../harness/analyticsApp'; + +jest.unmock('@react-navigation/native'); + +const PROMPT = 'plan my sister birthday at the lake house'; +const REPLY = 'Book the lake house for Saturday and invite the cousins.'; + +let app: AnalyticsApp | undefined; + +afterEach(async () => { + await require('../../../src/services/generationService').generationService.stopGeneration(); + app?.view.unmount(); + // Let the client's last flush land on the fake endpoint before the endpoint goes away. + await new Promise(resolve => setTimeout(resolve, 50)); + app?.endpoint.restore(); + app = undefined; + restoreBuildFlavor(); +}); + +it('a default install sends nothing to PostHog through Auto Setup and a chat', async () => { + app = await launchApp({ release: true }); + await passOnboarding(app, false); + await openNewChat(app); + await sendMessage(app, PROMPT, REPLY); + await sendToBackground(app); + + expect(app.view.getByText(REPLY)).toBeTruthy(); + // Give a would-be flush every chance to happen before asserting that none did. + await new Promise(resolve => setTimeout(resolve, 300)); + expect(app.endpoint.urls).toEqual([]); +}); + +it('ticking the Auto Setup row sends opt-in and the reply outcome, never the chat', async () => { + app = await launchApp({ release: true }); + await passOnboarding(app, true); + // Opting in alone does not chat; nothing about a reply exists yet. + expect(app.endpoint.named('generation_finished')).toEqual([]); + await openNewChat(app); + await sendMessage(app, PROMPT, REPLY); + await sendToBackground(app); + + const endpoint = app.endpoint; + await app.rtl.waitFor(() => expect(endpoint.named('generation_finished')).toHaveLength(1)); + expect(endpoint.named('analytics_opted_in').map(e => e.properties.source)).toEqual(['setup']); + + const finished = endpoint.named('generation_finished')[0].properties; + expect(finished).toMatchObject({ modality: 'text', engine: 'llama.cpp', outcome: 'done', platform: 'android' }); + // The public repo, never the file name. + expect(finished.model).toBe('unsloth/Qwen3-0.6B-GGUF'); + expect(endpoint.named('message_sent')[0].properties).toMatchObject({ has_image: false, has_document: false }); + + // Only the events docs/ANALYTICS.md lists: none of the SDK's own lifecycle events, whose + // "Application Opened" carries the deep link the app was opened with. + const listed = new Set([ + 'app_opened', 'app_backgrounded', '$screen', 'analytics_opted_in', 'model_loaded', + 'generation_finished', 'chat_created', 'message_sent', 'features_snapshot', + 'flow_started', 'flow_step', 'flow_finished', + ]); + expect(endpoint.events.map(e => e.event).filter(name => !listed.has(name))).toEqual([]); + // Only event batches: no flags, no remote config, no surveys. + expect(endpoint.urls.every(url => url === 'https://eu.i.posthog.com/batch/')).toBe(true); + // Never the chat, never a file name. + const sent = endpoint.bodies.join('\n'); + expect(sent).not.toContain('lake house'); + expect(sent).not.toContain('cousins'); + expect(sent).not.toContain('small.gguf'); +}); diff --git a/__tests__/integration/analytics/optOutInSettings.rendered.test.tsx b/__tests__/integration/analytics/optOutInSettings.rendered.test.tsx new file mode 100644 index 000000000..55b8888f0 --- /dev/null +++ b/__tests__/integration/analytics/optOutInSettings.rendered.test.tsx @@ -0,0 +1,92 @@ +/** + * Opting out in Settings (docs/ANALYTICS.md, Rules). + * + * SPEC (OGAM user's view): turning off both Usage data and Crash reports stops every send, drops + * what is queued and resets the anonymous ID. Turning analytics back on later starts a new, + * unrelated anonymous ID. + * + * The real App runs as a store build over device boundaries; PostHog is faked at `fetch`. + */ +import { + bringToForeground, + launchApp, + openNewChat, + openSettings, + passOnboarding, + restoreBuildFlavor, + sendMessage, + sendToBackground, + type AnalyticsApp, +} from '../../harness/analyticsApp'; + +jest.unmock('@react-navigation/native'); + +let app: AnalyticsApp | undefined; + +afterEach(async () => { + await require('../../../src/services/generationService').generationService.stopGeneration(); + app?.view.unmount(); + // Let the client's last flush land on the fake endpoint before the endpoint goes away. + await new Promise(resolve => setTimeout(resolve, 50)); + app?.endpoint.restore(); + app = undefined; + restoreBuildFlavor(); +}); + +/** The switch's rendered state: Android's native switch draws `on`, iOS's draws `value`. */ +function isOn(element: { props: Record }): boolean { + return Boolean(element.props.on ?? element.props.value); +} + +async function chatFromHome(current: AnalyticsApp, prompt: string, reply: string): Promise { + current.rtl.fireEvent.press(current.view.getByTestId('home-tab')); + current.rtl.fireEvent.press(await current.view.findByTestId('new-chat-button')); + await current.view.findByTestId('chat-input'); + await sendMessage(current, prompt, reply); +} + +it('turning both toggles off stops sending and a later opt-in uses a new anonymous ID', async () => { + app = await launchApp({ release: true }); + const current = app; + const { view, rtl, endpoint } = current; + await passOnboarding(current, true); + await openNewChat(current); + await sendMessage(current, 'first question', 'First answer.'); + await sendToBackground(current); + await rtl.waitFor(() => expect(endpoint.named('generation_finished')).toHaveLength(1)); + const firstId = endpoint.named('generation_finished')[0].distinct_id; + expect(firstId).toBeTruthy(); + + await bringToForeground(current); + await openSettings(current); + const usage = view.getByLabelText('Usage data'); + const crashes = view.getByLabelText('Crash reports'); + expect(isOn(usage)).toBe(true); + expect(isOn(crashes)).toBe(true); + rtl.fireEvent(usage, 'valueChange', false); + rtl.fireEvent(crashes, 'valueChange', false); + await rtl.waitFor(() => { + expect(isOn(view.getByLabelText('Usage data'))).toBe(false); + expect(isOn(view.getByLabelText('Crash reports'))).toBe(false); + }); + + const sentBeforeOptOut = endpoint.urls.length; + await chatFromHome(current, 'second question', 'Second answer.'); + await sendToBackground(current); + await new Promise(resolve => setTimeout(resolve, 300)); + expect(view.getByText('Second answer.')).toBeTruthy(); + expect(endpoint.urls).toHaveLength(sentBeforeOptOut); + + // Back on: a fresh anonymous ID, unrelated to the first one. + await bringToForeground(current); + await openSettings(current); + rtl.fireEvent(view.getByLabelText('Usage data'), 'valueChange', true); + await rtl.waitFor(() => expect(isOn(view.getByLabelText('Usage data'))).toBe(true)); + await chatFromHome(current, 'third question', 'Third answer.'); + await sendToBackground(current); + await rtl.waitFor(() => expect(endpoint.named('generation_finished')).toHaveLength(2)); + expect(endpoint.named('analytics_opted_in').map(e => e.properties.source)).toEqual(['setup', 'settings']); + const secondId = endpoint.named('generation_finished')[1].distinct_id; + expect(secondId).toBeTruthy(); + expect(secondId).not.toBe(firstId); +}); diff --git a/__tests__/integration/analytics/reportProblemLogHasNoChat.rendered.test.tsx b/__tests__/integration/analytics/reportProblemLogHasNoChat.rendered.test.tsx new file mode 100644 index 000000000..100b50e49 --- /dev/null +++ b/__tests__/integration/analytics/reportProblemLogHasNoChat.rendered.test.tsx @@ -0,0 +1,129 @@ +/** + * Report a problem sends a debug log with no chat content (docs/ANALYTICS.md, Report a problem). + * + * SPEC (OGAM user's view): after chatting, Settings → Report a problem shows the debug log, and + * "Send to support" hands it to the share sheet. Neither the log the person sees nor what is shared + * holds their prompt or the model's reply. It works with analytics off. + * + * The real App runs as a store build: the debug log is on in every build, so this is the log a + * person actually sends. The share sheet is the OS boundary: it is replaced by a recorder of what + * the app handed it. + */ +import { + launchApp, + openNewChat, + openSettings, + passOnboarding, + restoreBuildFlavor, + sendMessage, + type AnalyticsApp, +} from '../../harness/analyticsApp'; + +jest.unmock('@react-navigation/native'); + +const PROMPT = 'my passport number is in the drawer by the window'; +const REPLY = 'Keep the passport drawer locked when guests visit.'; + +let app: AnalyticsApp | undefined; + +afterEach(async () => { + await require('../../../src/services/generationService').generationService.stopGeneration(); + app?.view.unmount(); + // Let the client's last flush land on the fake endpoint before the endpoint goes away. + await new Promise(resolve => setTimeout(resolve, 50)); + app?.endpoint.restore(); + app = undefined; + restoreBuildFlavor(); +}); + +it('the shared debug log holds the chat trace but none of the prompt or reply', async () => { + app = await launchApp({ release: true }); + const shared: Array<{ message?: string; url?: string }> = []; + require('react-native').Share.share = async (content: { message?: string; url?: string }) => { + shared.push(content); + return { action: 'sharedAction' }; + }; + const { view, rtl } = app; + await passOnboarding(app, false); + await openNewChat(app); + await sendMessage(app, PROMPT, REPLY); + + await openSettings(app); + rtl.fireEvent.press(view.getByTestId('settings-report-problem')); + const send = await view.findByTestId('debug-logs-send-report'); + // The person sees the log first, and it is not empty. + expect(view.queryByText('No logs yet')).toBeNull(); + expect(view.queryByText(/passport/)).toBeNull(); + + expect(shared).toHaveLength(0); + rtl.fireEvent.press(send); + await rtl.waitFor(() => expect(shared).toHaveLength(1)); + + const message = shared[0].message ?? ''; + expect(message).toContain('support@getoffgridai.co'); + // The trace of the chat is there: the generation ran and was logged. + expect(message).toMatch(/\[(GEN|LLM|ToolLoop|GenService)/); + expect(message).not.toMatch(/passport|drawer|window|guests/i); +}); + +it('on iOS the log goes as an attached file, and closing the viewer sends nothing', async () => { + app = await launchApp({ release: true, platform: 'ios' }); + const shared: Array<{ message?: string; url?: string }> = []; + require('react-native').Share.share = async (content: { message?: string; url?: string }) => { + shared.push(content); + return { action: 'sharedAction' }; + }; + const { view, rtl } = app; + await passOnboarding(app, false); + await openSettings(app); + rtl.fireEvent.press(view.getByTestId('settings-report-problem')); + rtl.fireEvent.press(await view.findByTestId('debug-logs-close')); + await rtl.waitFor(() => expect(view.queryByTestId('debug-logs-send-report')).toBeNull()); + expect(shared).toHaveLength(0); + + rtl.fireEvent.press(view.getByTestId('settings-report-problem')); + rtl.fireEvent.press(await view.findByTestId('debug-logs-send-report')); + await rtl.waitFor(() => expect(shared).toHaveLength(1)); + expect(shared[0].url).toMatch(/^file:\/\/.*offgrid-debug\.log$/); + expect(shared[0].message).toContain('support@getoffgridai.co'); + expect(shared[0].message).not.toContain('---'); +}); + +it('an unreadable log still opens the share sheet with the support address', async () => { + app = await launchApp({ release: true }); + const shared: Array<{ message?: string; url?: string }> = []; + require('react-native').Share.share = async (content: { message?: string; url?: string }) => { + shared.push(content); + throw new Error('share sheet dismissed'); + }; + const fs = require('react-native-fs'); + fs.readFile = async () => { + throw new Error('EACCES'); + }; + const { view, rtl } = app; + await passOnboarding(app, false); + await openSettings(app); + rtl.fireEvent.press(view.getByTestId('settings-report-problem')); + rtl.fireEvent.press(await view.findByTestId('debug-logs-send-report')); + await rtl.waitFor(() => expect(shared).toHaveLength(1)); + expect(shared[0].url).toBeUndefined(); + expect(shared[0].message).toContain('support@getoffgridai.co'); +}); + +it('with no log on disk yet, the share sheet still gets the support address', async () => { + app = await launchApp({ release: true }); + const shared: Array<{ message?: string; url?: string }> = []; + require('react-native').Share.share = async (content: { message?: string; url?: string }) => { + shared.push(content); + return { action: 'sharedAction' }; + }; + require('react-native-fs').exists = async () => false; + const { view, rtl } = app; + await passOnboarding(app, false); + await openSettings(app); + rtl.fireEvent.press(view.getByTestId('settings-report-problem')); + rtl.fireEvent.press(await view.findByTestId('debug-logs-send-report')); + await rtl.waitFor(() => expect(shared).toHaveLength(1)); + expect(shared[0].message).toContain('support@getoffgridai.co'); + expect(shared[0].message).not.toContain('---'); +}); diff --git a/__tests__/integration/analytics/uncleanExit.rendered.test.tsx b/__tests__/integration/analytics/uncleanExit.rendered.test.tsx new file mode 100644 index 000000000..d6ebd2e24 --- /dev/null +++ b/__tests__/integration/analytics/uncleanExit.rendered.test.tsx @@ -0,0 +1,82 @@ +/** + * `unclean_exit` (docs/ANALYTICS.md, Crashes). + * + * SPEC (OGAM user's view, opted in): if the app dies in the foreground while a model is loading, + * the next launch of the same version reports one `unclean_exit` with stage `load`. If it was in + * the background when it died (the OS reclaiming memory), nothing is reported. + * + * The death is the device boundary: the native load is held open (it never returns), the app is + * torn down without any clean shutdown, and a new launch reads the same disk and storage. + */ +import { + launchApp, + passOnboarding, + restoreBuildFlavor, + sendToBackground, + type AnalyticsApp, +} from '../../harness/analyticsApp'; + +jest.unmock('@react-navigation/native'); + +const launched: AnalyticsApp[] = []; + +afterEach(async () => { + for (const app of launched.reverse()) { + app.view.unmount(); + // Let the client's last flush land on the fake endpoint before the endpoint goes away. + await new Promise(resolve => setTimeout(resolve, 50)); + app.endpoint.restore(); + } + launched.length = 0; + restoreBuildFlavor(); +}); + +/** Opt in at Auto Setup, then start a chat whose model load never finishes. */ +async function dieWhileLoading(inBackground: boolean): Promise { + const app = await launchApp({ release: true }); + launched.push(app); + const { view, rtl, boundary } = app; + await passOnboarding(app, true); + boundary.llama!.scriptMultimodalHold(); + rtl.fireEvent.press(view.getByTestId('browse-models-button')); + rtl.fireEvent.press(await view.findByTestId('text-model-row-m')); + rtl.fireEvent.press(await view.findByTestId('new-chat-button')); + rtl.fireEvent.changeText(await view.findByTestId('chat-input'), 'hello'); + rtl.fireEvent.press(view.getByTestId('send-button')); + await rtl.waitFor(() => expect(boundary.llama!.multimodalHoldActive()).toBe(true)); + expect(view.getByText(/Loading Test Model/)).toBeTruthy(); + if (inBackground) await sendToBackground(app); + // The process dies here: no unmount clean-up runs before the next launch reads the disk. +} + +async function relaunch(): Promise { + const next = await launchApp({ release: true, relaunch: true }); + launched.push(next); + await next.view.findByTestId('home-screen'); + await sendToBackground(next); + // This launch's own first event: the earlier, dead launch never sends `app_opened` with a cold start. + await next.rtl.waitFor(() => + expect(next.endpoint.named('app_opened').some(e => typeof e.properties.cold_start_ms === 'number')).toBe(true), + ); + return next; +} + +it('a foreground death during a model load is reported once on the next launch', async () => { + await dieWhileLoading(false); + const next = await relaunch(); + const reports = next.endpoint.named('unclean_exit'); + expect(reports).toHaveLength(1); + expect(reports[0].properties).toMatchObject({ + stage: 'load', + model: 'unsloth/Qwen3-0.6B-GGUF', + engine: 'llama.cpp', + backend: 'cpu', + }); + expect(typeof reports[0].properties.elapsed_ms).toBe('number'); +}); + +it('a death in the background is not reported', async () => { + await dieWhileLoading(true); + const next = await relaunch(); + expect(next.endpoint.named('unclean_exit')).toEqual([]); +}); diff --git a/__tests__/integration/app/bootNotBlockedByDownloadDb.rendered.test.tsx b/__tests__/integration/app/bootNotBlockedByDownloadDb.rendered.test.tsx index fb6ab488c..2ad2bca7a 100644 --- a/__tests__/integration/app/bootNotBlockedByDownloadDb.rendered.test.tsx +++ b/__tests__/integration/app/bootNotBlockedByDownloadDb.rendered.test.tsx @@ -12,6 +12,7 @@ * artifact: the boot loader ('app-loading') CLEARS anyway. RED on HEAD: the loader stays * forever because initializeApp awaits hydrateDownloadStore/reattach before first paint. */ +import { proIsPresent } from '../../pro/helpers/requirePro'; import { installNativeBoundary, requireRTL, @@ -65,10 +66,12 @@ describe('app boot is not blocked by the download DB (rendered)', () => { releaseDownloadDb([]); await pendingDownloadDb; }); + // The core boot reads the download DB twice; with Pro checked out, Pro's video downloads + // reconcile at launch too (pro 6fb3b12e), one read more. await rtl.waitFor(() => { expect( NativeModules.DownloadManagerModule.getActiveDownloads, - ).toHaveBeenCalledTimes(2); + ).toHaveBeenCalledTimes(proIsPresent() ? 3 : 2); }); view.unmount(); diff --git a/__tests__/integration/audio/qwen3AsrTranscription.rendered.test.tsx b/__tests__/integration/audio/qwen3AsrTranscription.rendered.test.tsx new file mode 100644 index 000000000..7da5462f2 --- /dev/null +++ b/__tests__/integration/audio/qwen3AsrTranscription.rendered.test.tsx @@ -0,0 +1,252 @@ +/** + * Qwen3-ASR 0.6B as a speech-to-text model beside Whisper. + * + * From the person's side: + * - On a phone the memory guard admits, Models > Transcription lists Qwen3-ASR 0.6B. Downloading it + * fetches its weights and audio projector (checksummed) and makes it the speech model. + * - On a phone the guard refuses, or with less than the 3 GB its catalog entry measured, it is not + * listed; the Whisper models still are. + * - With it selected, dictation in a chat puts what they said in the composer. + * - When Qwen3-ASR fails, the composer says so in one line, and a downloaded Whisper model + * transcribes the recording instead. + * + * Real TranscriptionModelsTab, ChatScreen, language select, stores, residency, download service and + * transcription service. Fakes only at the device boundary: llama.rn (the Qwen3-ASR context), whisper.rn, + * the recorder, the file system, the background downloader, the RAM sensor, and the engine's reported + * llama.cpp build (llama.rn BuildInfo). + */ +import { setupChatScreen } from '../../harness/chatHarness'; +import { installNativeBoundary, requireRTL, GB } from '../../harness/nativeBoundary'; + +jest.mock('@react-navigation/native', () => ({ + useNavigation: () => ({ navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {} }), + useRoute: () => require('../../harness/chatHarness').routeHolder, + useFocusEffect: () => {}, + useIsFocused: () => true, +})); + +const CARD = 'transcription-model-card-qwen3-asr'; +const WEIGHTS = 'Qwen3-ASR-0.6B-Q8_0.gguf'; +const PROJECTOR = 'mmproj-Qwen3-ASR-0.6B-Q8_0.gguf'; + +/** The llama.cpp build this app ships, as llama.rn reports it about itself. */ +function engineReportsShippedBuild() { + const src: string = require('fs').readFileSync(require.resolve('llama.rn/src/version.ts'), 'utf8'); + const number = /BUILD_NUMBER = '(\d+)'/.exec(src)![1]; + require('llama.rn').BuildInfo = { number, commit: 'test' }; +} + +/** What the app does at launch: read the device once (the RAM it reads is the boundary's). */ +async function launchReadsDevice() { + const { hardwareService } = require('../../../src/services/hardware'); + await hardwareService.getDeviceInfo(); +} + +/** The catalog's Qwen3-ASR files, as the downloader is asked for them. */ +function catalogFile(name: string): { sizeBytes: number; sha256: string } { + const { CATALOG } = require('@offgrid/models'); + const entry = CATALOG.find((e: { id: string }) => e.id === 'ggml-org/Qwen3-ASR-0.6B-GGUF'); + return entry.files.find((f: { name: string }) => f.name === name); +} + +describe('Qwen3-ASR on the Transcription tab', () => { + it('is listed where the memory guard admits it, and downloads both files checksummed', async () => { + const boundary = installNativeBoundary({ download: true, fs: true }); + engineReportsShippedBuild(); + await launchReadsDevice(); + const React = require('react'); + const { render, fireEvent, waitFor, act } = requireRTL(); + const { registerCoreDownloadProviders } = require('../../../src/services/modelDownloadService/registerProviders'); + const { TranscriptionModelsTab } = require('../../../src/screens/ModelsScreen/TranscriptionModelsTab'); + registerCoreDownloadProviders(); + + const screen = render(React.createElement(TranscriptionModelsTab, { showRemoteModels: false })); + expect(screen.getByText('Qwen3-ASR 0.6B')).toBeTruthy(); + expect(screen.getByTestId(`${CARD}-download`)).toBeTruthy(); + expect(screen.queryByTestId(`${CARD}-delete`)).toBeNull(); + + fireEvent.press(screen.getByTestId(`${CARD}-download`)); + + // The projector comes first, then the weights; each finishes on disk where the app reads it. + const docs = boundary.fs!.DocumentDirectoryPath; + for (const name of [PROJECTOR, WEIGHTS]) { + const row = await waitFor(() => { + const started = boundary.download!.active().filter((r: { fileName?: string }) => r.fileName === name); + expect(started).toHaveLength(1); + return started[0] as { downloadId: string; fileName: string; modelId: string; sha256?: string }; + }); + const file = catalogFile(name); + expect(row.sha256).toBe(file.sha256); + await act(async () => { + boundary.fs!.seedFile(`${docs}/whisper-models/${name}`, file.sizeBytes); + boundary.download!.events.emit('DownloadComplete', { + downloadId: row.downloadId, fileName: name, modelId: row.modelId, + bytesDownloaded: file.sizeBytes, totalBytes: file.sizeBytes, status: 'completed', + localUri: `${docs}/whisper-models/${name}`, + }); + }); + } + + // Downloaded and in use: the card offers removal, not download, and reads as the selected model. + await waitFor(() => { expect(screen.getByTestId(`${CARD}-delete`)).toBeTruthy(); }, { timeout: 4000 }); + expect(screen.queryByTestId(`${CARD}-download`)).toBeNull(); + expect(screen.getByTestId(CARD).props.accessibilityState.selected).toBe(true); + }); + + it('is not listed on a phone whose memory the guard refuses', async () => { + installNativeBoundary({ download: true, fs: true, ram: { platform: 'android', totalBytes: 1 * GB, availBytes: 0.5 * GB } }); + engineReportsShippedBuild(); + await launchReadsDevice(); + const React = require('react'); + const { render } = requireRTL(); + const { TranscriptionModelsTab } = require('../../../src/screens/ModelsScreen/TranscriptionModelsTab'); + + const screen = render(React.createElement(TranscriptionModelsTab, { showRemoteModels: false })); + expect(screen.getByTestId('transcription-model-card-0')).toBeTruthy(); + expect(screen.queryByText('Qwen3-ASR 0.6B')).toBeNull(); + expect(screen.queryByTestId(CARD)).toBeNull(); + }); + + it('is not listed on a 2 GB phone: Qwen3-ASR 0.6B needs the 3 GB its catalog entry measured', async () => { + installNativeBoundary({ download: true, fs: true, ram: { platform: 'android', totalBytes: 2 * GB, availBytes: 1 * GB } }); + engineReportsShippedBuild(); + await launchReadsDevice(); + const React = require('react'); + const { render } = requireRTL(); + const { TranscriptionModelsTab } = require('../../../src/screens/ModelsScreen/TranscriptionModelsTab'); + + const screen = render(React.createElement(TranscriptionModelsTab, { showRemoteModels: false })); + expect(screen.getByTestId('transcription-model-card-0')).toBeTruthy(); + expect(screen.queryByText('Qwen3-ASR 0.6B')).toBeNull(); + }); +}); + +type Harness = Awaited>; + +/** The person has Qwen3-ASR on disk (a finished download) and picks it on the Transcription tab. */ +async function selectQwenAsr(h: Harness, opts: { whisperToo?: boolean } = {}) { + const React = require('react'); + const { TranscriptionModelsTab } = require('../../../src/screens/ModelsScreen/TranscriptionModelsTab'); + const docs = h.boundary.fs!.DocumentDirectoryPath; + for (const name of [WEIGHTS, PROJECTOR]) { + h.boundary.fs!.seedFile(`${docs}/whisper-models/${name}`, catalogFile(name).sizeBytes); + } + if (opts.whisperToo) h.boundary.fs!.seedFile(`${docs}/whisper-models/ggml-tiny.en.bin`, 75 * 1024 * 1024); + const tab = h.rtl.render(React.createElement(TranscriptionModelsTab, { showRemoteModels: false })); + const card = await h.rtl.waitFor(() => { + expect(tab.getByTestId(`${CARD}-delete`)).toBeTruthy(); + return tab.getByTestId(CARD); + }, { timeout: 4000 }); + expect(card.props.accessibilityState.selected).toBe(false); + h.rtl.fireEvent.press(card); + await h.rtl.waitFor(() => { + expect(tab.getByTestId(CARD).props.accessibilityState.selected).toBe(true); + }, { timeout: 4000 }); + tab.unmount(); +} + +/** Pick the transcription language the way the chat settings offer it. */ +async function chooseLanguage(h: Harness, code: string) { + const React = require('react'); + const { TranscriptionLanguageSelect } = require('../../../src/components/TranscriptionLanguageSelect'); + const select = h.rtl.render(React.createElement(TranscriptionLanguageSelect)); + h.rtl.fireEvent.press(select.getByTestId('transcription-language-select')); + h.rtl.fireEvent.press(await h.rtl.waitFor(() => select.getByTestId(`transcription-language-select-${code}`))); + select.unmount(); +} + +/** Hold the mic, speak, let go: one chat-mode dictation. */ +async function dictate(h: Harness) { + await h.tapMic(); + await h.settle(300); + await h.releaseMic(); + await h.settle(1200); +} + +describe('dictation with Qwen3-ASR selected', () => { + it.each([ + // A pinned language prefills `language English`, so the model writes only the words. + { language: 'en', modelSays: 'hello world' }, + // Auto-detect: the model names the language, then the words. + { language: 'auto', modelSays: 'language Englishhello world' }, + ])('puts what was said in the composer (language $language)', async ({ language, modelSays }) => { + const h = await setupChatScreen({ engine: 'llama', platform: 'android', whisper: true, llamaAudio: true }); + engineReportsShippedBuild(); + await selectQwenAsr(h); + await chooseLanguage(h, language); + h.render(); + const view = h.view!; + const input = await h.rtl.waitFor(() => view.getByTestId('chat-input')); + expect(input.props.value ?? '').toBe(''); + + h.boundary.llama!.scriptCompletion({ text: modelSays }); + await dictate(h); + + await h.rtl.waitFor(() => { + expect(view.getByTestId('chat-input').props.value ?? '').toBe('hello world'); + }, { timeout: 4000 }); + expect(view.queryByTestId('transcription-notice')).toBeNull(); + }); + + it('offers Qwen3-ASR\'s own languages, and a pinned one reaches the model by the name it writes', async () => { + const h = await setupChatScreen({ engine: 'llama', platform: 'android', whisper: true, llamaAudio: true }); + engineReportsShippedBuild(); + await selectQwenAsr(h); + // Filipino and Cantonese are Qwen3-ASR languages the Whisper list does not offer. + const React = require('react'); + const { TranscriptionLanguageSelect } = require('../../../src/components/TranscriptionLanguageSelect'); + const select = h.rtl.render(React.createElement(TranscriptionLanguageSelect)); + h.rtl.fireEvent.press(select.getByTestId('transcription-language-select')); + await h.rtl.waitFor(() => expect(select.getByTestId('transcription-language-select-yue')).toBeTruthy()); + select.unmount(); + await chooseLanguage(h, 'fil'); + h.render(); + const view = h.view!; + + h.boundary.llama!.scriptCompletion({ text: 'magandang umaga' }); + await dictate(h); + + await h.rtl.waitFor(() => { + expect(view.getByTestId('chat-input').props.value ?? '').toBe('magandang umaga'); + }, { timeout: 4000 }); + const prompts = h.boundary.llama!.calls.completion.map(([params]) => String((params as { prompt?: string }).prompt ?? '')); + expect(prompts.some(prompt => prompt.endsWith('language Filipino'))).toBe(true); + }); + + it('says so in one line and lets Whisper transcribe when Qwen3-ASR fails', async () => { + const h = await setupChatScreen({ engine: 'llama', platform: 'android', whisper: true, llamaAudio: true }); + engineReportsShippedBuild(); + await selectQwenAsr(h, { whisperToo: true }); + h.render(); + const view = h.view!; + expect(view.queryByTestId('transcription-notice')).toBeNull(); + + h.boundary.llama!.scriptCompletion({ throwMessage: 'Failed to evaluate audio chunk' }); + h.boundary.whisper!.setFileTranscript('take a note'); + await dictate(h); + + await h.rtl.waitFor(() => { + expect(view.getByTestId('chat-input').props.value ?? '').toBe('take a note'); + }, { timeout: 4000 }); + expect(view.getByTestId('transcription-notice').props.children).toBe( + "Qwen3-ASR couldn't transcribe that, so Whisper did.", + ); + }); + + it('says so in one line when Qwen3-ASR fails and no Whisper model is there', async () => { + const h = await setupChatScreen({ engine: 'llama', platform: 'android', whisper: true, llamaAudio: true }); + engineReportsShippedBuild(); + await selectQwenAsr(h); + h.render(); + const view = h.view!; + expect(view.queryByTestId('transcription-notice')).toBeNull(); + + h.boundary.llama!.scriptCompletion({ throwMessage: 'Failed to evaluate audio chunk' }); + await dictate(h); + + await h.rtl.waitFor(() => { + expect(view.getByText(/Qwen3-ASR couldn't transcribe that\. Try again, or download a Whisper model\./)).toBeTruthy(); + }, { timeout: 4000 }); + expect(view.getByTestId('chat-input').props.value ?? '').toBe(''); + }); +}); diff --git a/__tests__/integration/chat/kvSessionOnChatSwitch.test.tsx b/__tests__/integration/chat/kvSessionOnChatSwitch.test.tsx index 0721bb61b..0fcdb0db0 100644 --- a/__tests__/integration/chat/kvSessionOnChatSwitch.test.tsx +++ b/__tests__/integration/chat/kvSessionOnChatSwitch.test.tsx @@ -94,7 +94,9 @@ describe('chat switch keeps each chat warm', () => { llamaModelInfo: { 'general.architecture': 'qwen35', 'qwen35.nextn_predict_layers': 1 }, }); const { TextGenerationSection } = require('../../../src/components/GenerationSettingsModal/TextGenerationSection'); - const section = h.rtl.render(h.React.createElement(TextGenerationSection, {})); + // The sheet shows the shared General controls (backend, loading mode) above the Text section. + const { GeneralGenerationControls } = require('../../../src/components/settings/GeneralGenerationControls'); + const section = h.rtl.render(h.React.createElement(h.React.Fragment, null, h.React.createElement(GeneralGenerationControls), h.React.createElement(TextGenerationSection, {}))); h.rtl.fireEvent.press(section.getByTestId('modal-text-advanced-toggle')); h.rtl.fireEvent.press(section.getByTestId('speculative-on-button')); await h.rtl.waitFor(() => expect(section.queryByTestId('speculative-draft-stepper-value')).not.toBeNull()); @@ -118,11 +120,13 @@ describe('chat switch keeps each chat warm', () => { }); // Choose the GPU the way the user does, before the first load. const { TextGenerationSection } = require('../../../src/components/GenerationSettingsModal/TextGenerationSection'); - const section = h.rtl.render(h.React.createElement(TextGenerationSection, {})); + // The sheet shows the shared General controls (backend, loading mode) above the Text section. + const { GeneralGenerationControls } = require('../../../src/components/settings/GeneralGenerationControls'); + const section = h.rtl.render(h.React.createElement(h.React.Fragment, null, h.React.createElement(GeneralGenerationControls), h.React.createElement(TextGenerationSection, {}))); h.rtl.fireEvent.press(section.getByTestId('modal-text-advanced-toggle')); h.rtl.fireEvent.press(section.getByTestId('backend-opencl-button')); await h.rtl.waitFor(() => { - expect(section.queryByText('GPU Layers (OpenCL)')).not.toBeNull(); + expect(section.queryByText('Text GPU Layers (OpenCL)')).not.toBeNull(); }); section.unmount(); diff --git a/__tests__/integration/chats/chatDeleteRemovesVideos.rendered.test.tsx b/__tests__/integration/chats/chatDeleteRemovesVideos.rendered.test.tsx new file mode 100644 index 000000000..8136a01c5 --- /dev/null +++ b/__tests__/integration/chats/chatDeleteRemovesVideos.rendered.test.tsx @@ -0,0 +1,78 @@ +/** + * Deleting a chat from the Chats list removes the videos made in it, from the disk and the Gallery. + * A video file the device refuses to remove keeps its record, so the person can delete it again. + * + * Real ChatsListScreen and the real chat and app stores. The device filesystem is the in-memory native + * boundary; it refuses one unlink, as a locked file does on device. + */ +import { installNativeBoundary, requireRTL } from '../../harness/nativeBoundary'; + +jest.mock('@react-navigation/native', () => ({ + useNavigation: () => ({ navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {} }), + useRoute: () => ({ params: {} }), + useFocusEffect: () => {}, + useIsFocused: () => true, +})); + +it('removes the chat\'s videos, file first, and keeps a video whose file stays', async () => { + const boundary = installNativeBoundary({ fs: true }); + const fs = boundary.fs!; + const AsyncStorage = require('@react-native-async-storage/async-storage'); + await AsyncStorage.clear(); + const updatedAt = '2026-10-27T12:00:00.000Z'; + await AsyncStorage.setItem( + 'local-llm-chat-storage', + JSON.stringify({ + state: { + conversations: [{ + id: 'chat-fox', title: 'Fox clips', modelId: 'remote-model', + messages: [{ id: 'm1', role: 'user', content: 'a fox in snow', timestamp: Date.parse(updatedAt) }], + createdAt: updatedAt, updatedAt, + }], + activeConversationId: null, + }, + version: 2, + }), + ); + const React = require('react'); + const rtl = requireRTL(); + const { useChatStore } = require('../../../src/stores/chatStore'); + const { useAppStore } = require('../../../src/stores/appStore'); + await useChatStore.persist.rehydrate(); + + const saveVideo = (id: string, conversationId: string) => { + const videoPath = `${fs.DocumentDirectoryPath}/generated-videos/${id}.mp4`; + fs.seedFile(videoPath, 4096); + useAppStore.getState().addGeneratedVideo({ + id, prompt: 'a fox in snow', videoPath, width: 480, height: 832, fps: 16, frames: 33, durationSeconds: 2, + steps: 20, guidance: 5, seed: 1, modelId: 'wan', createdAt: updatedAt, conversationId, + }); + return videoPath; + }; + const freeVideo = saveVideo('vid-free', 'chat-fox'); + const lockedVideo = saveVideo('vid-locked', 'chat-fox'); + const otherChatVideo = saveVideo('vid-other', 'chat-other'); + const unlink = fs.module.unlink.getMockImplementation()!; + fs.module.unlink.mockImplementation(async (path: string) => { + if (path === lockedVideo) throw new Error('EACCES: permission denied'); + return unlink(path); + }); + + const { ChatsListScreen } = require('../../../src/screens/ChatsListScreen'); + const chats = rtl.render(React.createElement(ChatsListScreen)); + rtl.fireEvent.press(chats.getByLabelText('Delete Fox clips')); + await rtl.act(() => new Promise(resolve => setTimeout(resolve, 350))); + await rtl.act(async () => { rtl.fireEvent.press(chats.getByText('Delete')); }); + + await rtl.waitFor(() => expect(chats.queryByText('Fox clips')).toBeNull()); + await rtl.waitFor(async () => expect(await fs.exists(freeVideo)).toBe(false)); + expect(await fs.exists(lockedVideo)).toBe(true); + expect(await fs.exists(otherChatVideo)).toBe(true); + chats.unmount(); + + // The Gallery lists the video that stayed and the other chat's video, not the one removed. + const { GalleryScreen } = require('../../../src/screens/GalleryScreen'); + const gallery = rtl.render(React.createElement(GalleryScreen)); + expect(await gallery.findByText('Videos (2)')).toBeTruthy(); + gallery.unmount(); +}); diff --git a/__tests__/integration/chats/emptyChatWhereItRuns.rendered.test.tsx b/__tests__/integration/chats/emptyChatWhereItRuns.rendered.test.tsx new file mode 100644 index 000000000..651cda012 --- /dev/null +++ b/__tests__/integration/chats/emptyChatWhereItRuns.rendered.test.tsx @@ -0,0 +1,46 @@ +/** + * A new chat says where it runs. + * + * SPEC (OGAM user's view): with a model on this phone, the empty chat says everything is processed + * on the device. With a model on a paired computer it never says that: it says the chat runs on a + * remote server. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk and the RAM sensor. + */ +import { launchApp, openNewChat, passOnboarding, restoreBuildFlavor, type AnalyticsApp } from '../../harness/analyticsApp'; + +let app: AnalyticsApp | undefined; + +afterEach(() => { + app?.view.unmount(); + app?.endpoint.restore(); + require('../../../src/stores/remoteServerStore').useRemoteServerStore.getState().clearAllServers(); + app = undefined; + restoreBuildFlavor(); +}); + +it('a model on this phone: the empty chat says it is all processed on the device', async () => { + app = await launchApp({ release: false }); + await passOnboarding(app, false); + await openNewChat(app); + await app.view.findByText(/All processing happens on your device/); +}, 60_000); + +it('a model on a paired computer: the empty chat says it runs on a remote server, never "on your device"', async () => { + app = await launchApp({ release: false }); + const { view, rtl } = app; + await passOnboarding(app, false); + const { remoteServerManager } = require('../../../src/services/remoteServerManager'); + const server = await remoteServerManager.addServer({ + name: 'Studio Mac', + endpoint: 'http://192.168.1.50:7878', + providerType: 'openai-compatible', + modelCatalog: { text: [{ id: 'qwen', name: 'Qwen on the Mac' }] }, + }); + rtl.fireEvent.press(await view.findByTestId('browse-models-button')); + rtl.fireEvent.press(await view.findByTestId(`remote-text-model-${server.id}-qwen`)); + rtl.fireEvent.press(await view.findByTestId('new-chat-button', {}, { timeout: 5000 })); + await view.findByTestId('chat-input'); + await view.findByText(/runs on a remote server/); + expect(view.queryByText(/All processing happens on your device/)).toBeNull(); +}, 60_000); diff --git a/__tests__/integration/chats/searchPairedComputer.rendered.test.tsx b/__tests__/integration/chats/searchPairedComputer.rendered.test.tsx new file mode 100644 index 000000000..4a03d44cb --- /dev/null +++ b/__tests__/integration/chats/searchPairedComputer.rendered.test.tsx @@ -0,0 +1,192 @@ +/** + * Searching chats also searches the paired computer. + * + * SPEC (OGAM user's view): with Off Grid AI Desktop paired, what I type in the Chats search also + * searches my computer. My phone's chats stay first; the computer's results sit under "On ", each with its snippet and where it came from. A chat that is on my phone opens here; a + * chat my phone already lists is not listed twice. If the computer can't be reached, or refuses + * my phone's key, my phone's results still show and the section says why in one line with the step + * that fixes it (search again, or enter the key) — and after that step, the results come. + * + * The real App on the real navigator. Device boundaries only: the llama.rn engine, the disk, the + * RAM sensor, the PostHog endpoint and Desktop's gateway at `fetch`. + */ +jest.unmock('@react-navigation/native'); +// The device Keychain, where pairing keeps the computer's key. +jest.mock('react-native-keychain', () => require('../../harness/keyedRemoteServer').keychainModule()); + +import { + launchApp, + openNewChat, + passOnboarding, + restoreBuildFlavor, + sendMessage, + type AnalyticsApp, +} from '../../harness/analyticsApp'; +import { + DESKTOP_SEARCH_ENDPOINT, + serveDesktopSearch, + type DesktopSearch, +} from '../../harness/desktopSearchServer'; + +const KEY = 'ogk-test-phone-key'; +const COMPUTER = 'Studio Mac'; +const MEETING = { + id: 'meet:7', + title: 'Budget review', + snippet: 'We cut the budget for travel', + kind: 'meeting', + source: 'Meeting', + timestamp: Date.parse('2026-10-20T09:00:00Z'), +}; +const COMPUTER_ONLY_CHAT = { + id: 'chat:desk-only', + title: 'Quarter budget', + snippet: 'Draft the quarter budget', + kind: 'chat', + source: 'Chat', + timestamp: Date.parse('2026-10-21T09:00:00Z'), + deepLinkId: 'desk-only', +}; + +let app: AnalyticsApp | undefined; +let desktop: DesktopSearch | undefined; + +afterEach(async () => { + app?.view.unmount(); + desktop?.restore(); + app?.endpoint.restore(); + await require('../../../src/services/remoteServerManager').remoteServerManager.clearAllServers(); + app = undefined; + desktop = undefined; + restoreBuildFlavor(); +}); + +/** A chat made on the phone, the computer paired with `key`, then the Chats tab. */ +async function chatsWithComputer(key: string): Promise<{ app: AnalyticsApp; phoneChatId: string }> { + app = await launchApp({ release: false }); + await passOnboarding(app, false); + await openNewChat(app); + await sendMessage(app, 'Plan the budget offsite', 'Here is a plan for the offsite.'); + desktop = serveDesktopSearch({ apiKey: KEY, records: [MEETING, COMPUTER_ONLY_CHAT] }); + // What pairing leaves behind: the computer's record and its key in the Keychain. + await require('../../../src/services/remoteServerManager').remoteServerManager.addServer({ + name: COMPUTER, + endpoint: DESKTOP_SEARCH_ENDPOINT, + providerType: 'openai-compatible', + modelManagement: 'offgrid-desktop-v1', + apiKey: key, + }); + const { view, rtl } = app; + rtl.fireEvent.press(view.getAllByLabelText('Back')[0]); + rtl.fireEvent.press(await view.findByTestId('chats-tab')); + await view.findByTestId('chat-search-input'); + const phoneChatId = require('../../../src/stores/chatStore').useChatStore.getState().conversations[0].id as string; + return { app, phoneChatId }; +} + +it('shows the computer’s results under its name, with the phone’s chats kept and none listed twice', async () => { + const { app: { view, rtl }, phoneChatId } = await chatsWithComputer(KEY); + // The computer has the phone's chat too (synced). + desktop!.remember({ id: `chat:${phoneChatId}`, title: 'Plan the budget offsite', snippet: 'Plan the budget offsite', kind: 'chat', source: 'Chat', timestamp: Date.now(), deepLinkId: phoneChatId }); + expect(view.queryByText(`On ${COMPUTER}`)).toBeNull(); + + rtl.fireEvent.changeText(view.getByTestId('chat-search-input'), 'budget'); + + await view.findByText(`On ${COMPUTER}`); + await view.findByText('Budget review', {}, { timeout: 5000 }); + expect(view.getByText('We cut the budget for travel')).toBeTruthy(); + expect(view.getByText(/^Meeting · /)).toBeTruthy(); + expect(view.getByText('Quarter budget')).toBeTruthy(); + // The phone's own chat stays, and the computer's copy of it is not listed again. + expect(view.getByTestId('conversation-item-0')).toBeTruthy(); + expect(view.queryByTestId(`computer-result-chat:${phoneChatId}`)).toBeNull(); + // A chat only on the computer has nothing to open on the phone. + expect(view.getByTestId('computer-result-chat:desk-only').props.accessibilityRole).toBeUndefined(); +}, 60_000); + +it('opens a computer result for a chat that is on this phone', async () => { + const { app: { view, rtl }, phoneChatId } = await chatsWithComputer(KEY); + desktop!.remember({ id: `chat:${phoneChatId}`, title: 'Offsite', snippet: 'Catering for the offsite', kind: 'chat', source: 'Chat', timestamp: Date.now(), deepLinkId: phoneChatId }); + + rtl.fireEvent.changeText(view.getByTestId('chat-search-input'), 'catering'); + // Only the computer matched it: the phone lists nothing for this word. + await view.findByText('Catering for the offsite', {}, { timeout: 5000 }); + expect(view.getByText('No chats match your search.')).toBeTruthy(); + expect(view.queryByTestId('chat-input')).toBeNull(); + + rtl.fireEvent.press(view.getByTestId(`computer-result-chat:${phoneChatId}`)); + await view.findByTestId('chat-input'); + expect(view.getByText('Here is a plan for the offsite.')).toBeTruthy(); +}, 60_000); + +it('keeps the phone’s results when the computer is off, says why, and searches again once it is back', async () => { + const { app: { view, rtl } } = await chatsWithComputer(KEY); + desktop!.setOffline(true); + + rtl.fireEvent.changeText(view.getByTestId('chat-search-input'), 'budget'); + + await view.findByText(/Can't reach Studio Mac/, {}, { timeout: 5000 }); + expect(view.getByTestId('conversation-item-0')).toBeTruthy(); + expect(view.queryByText('Budget review')).toBeNull(); + + desktop!.setOffline(false); + rtl.fireEvent.press(view.getByLabelText(`Search ${COMPUTER} again`)); + await view.findByText('Budget review', {}, { timeout: 5000 }); + expect(view.queryByText(/Can't reach Studio Mac/)).toBeNull(); +}, 60_000); + +it('says the computer refused the phone’s key, and Enter key brings the results', async () => { + const { app: { view, rtl } } = await chatsWithComputer('ogk-revoked-key'); + + rtl.fireEvent.changeText(view.getByTestId('chat-search-input'), 'budget'); + + await view.findByText(/didn't accept this phone's key/, {}, { timeout: 5000 }); + expect(view.getByTestId('conversation-item-0')).toBeTruthy(); + expect(view.queryByText('Budget review')).toBeNull(); + + rtl.fireEvent.press(view.getByText('Enter key')); + // The editor arrives showing the key the computer refused; the person replaces it. + const keyField = await view.findByTestId('server-api-key'); + await rtl.waitFor(() => expect(view.getByTestId('server-api-key').props.value).toBe('ogk-revoked-key')); + rtl.fireEvent.changeText(keyField, KEY); + // Save opens once the key passes Test connection. + rtl.fireEvent.press(view.getByTestId('test-connection')); + await rtl.waitFor( + () => expect(view.getByTestId('save-server').props.accessibilityState).toEqual({ disabled: false }), + { timeout: 20_000 }, + ); + rtl.fireEvent.press(view.getByTestId('save-server')); + // Saving returns to Chats. + await rtl.waitFor(() => expect(view.queryByTestId('save-server')).toBeNull(), { timeout: 20_000 }); + + await view.findByText('Budget review', {}, { timeout: 10_000 }); + expect(view.queryByText(/didn't accept this phone's key/)).toBeNull(); +}, 60_000); + +it('lets a phone with no chats yet search its paired computer', async () => { + app = await launchApp({ release: false }); + await passOnboarding(app, false); + const { view, rtl } = app; + rtl.fireEvent.press(await view.findByTestId('chats-tab')); + // BEFORE: nothing paired, no chats: the empty state and no search. + await view.findByText('No Chats Yet'); + expect(view.queryByTestId('chat-search-input')).toBeNull(); + + desktop = serveDesktopSearch({ apiKey: KEY, records: [MEETING] }); + await rtl.act(async () => { + await require('../../../src/services/remoteServerManager').remoteServerManager.addServer({ + name: COMPUTER, + endpoint: DESKTOP_SEARCH_ENDPOINT, + providerType: 'openai-compatible', + modelManagement: 'offgrid-desktop-v1', + apiKey: KEY, + }); + }); + + // Paired: the search is there even with no chats, and it finds what is on the computer. + rtl.fireEvent.changeText(await view.findByTestId('chat-search-input'), 'budget'); + await view.findByText(`On ${COMPUTER}`); + await view.findByText('Budget review', {}, { timeout: 5000 }); + expect(view.queryByText('No Chats Yet')).toBeNull(); +}, 60_000); diff --git a/__tests__/integration/documents/searchModelSwitch.integration.test.ts b/__tests__/integration/documents/searchModelSwitch.integration.test.ts new file mode 100644 index 000000000..d4f350496 --- /dev/null +++ b/__tests__/integration/documents/searchModelSwitch.integration.test.ts @@ -0,0 +1,57 @@ +/** + * Changing the search (embedding) model on the phone's document store. + * + * SPEC (OGAM user's view, PR #739): picking another search model takes effect at once and keeps every + * document. The old model's vectors are dropped in the same transaction as the new choice, so search + * never ranks vectors of two models together; keyword search answers until each document is re-embedded + * the next time it is used. + * + * Everything we own runs (documentService, the shared ingest, prepareTurnDocuments, the documents.db + * store). Device leaves only (harness/documentEngine): SQLite is a real in-memory engine, the + * filesystem is memfs, the PDF reader reads the file at its path, MiniLM answers through llama.rn. + */ +import { installNativeBoundary, GB } from '../../harness/nativeBoundary'; +import { installDocumentEngine } from '../../harness/documentEngine'; + +function setup() { + const boundary = installNativeBoundary({ fs: true, ram: { platform: 'ios', totalBytes: 8 * GB, availBytes: 6 * GB } }); + const engine = installDocumentEngine(boundary.fs!); + const { documentService } = require('../../../src/services/documentService'); + const store = require('../../../src/services/documents/store'); + const { prepareTurnDocuments } = require('../../../src/services/documents/chatTurn'); + return { engine, documentService, store, prepareTurnDocuments }; +} + +const L12 = { id: 'minilm-l12', name: 'MiniLM L12', filePath: '/models/minilm-l12.gguf', size: 35 * 1024 * 1024 }; + +it('a new search model is kept at once, its predecessor\'s vectors go, and the document still answers', async () => { + const h = setup(); + h.engine.seedPdf('/docs/Downloads/zenland.pdf', [ + 'Zenland: an introduction. Geography and climate.', + 'The capital of Zenland is Quixotic City.', + ]); + const attachment = await h.documentService.attachDocument('/docs/Downloads/zenland.pdf', 'zenland.pdf'); + await h.engine.indexed(attachment.documentId); + expect(h.store.activeEmbeddingModel()).toBeNull(); + expect((await h.store.documentStore.getDocument(attachment.documentId))?.embeddedCount).toBeGreaterThan(0); + + h.store.commitEmbeddingModel(L12); + + expect(h.store.activeEmbeddingModel()).toEqual(L12); + // No vector of the old model survives the switch. + expect((await h.store.documentStore.getDocument(attachment.documentId))?.embeddedCount).toBe(0); + // The document is kept, and a turn still finds its answer (keyword search, then re-embedding as used). + const turn = await h.prepareTurnDocuments({ + conversationId: 'chat-1', + question: 'What is the capital of Zenland?', + systemPrompt: 'You are a helpful assistant.', + contextLength: 4096, + isRemote: false, + attachments: [attachment], + }); + expect(`${turn?.stable}\n${turn?.turn}`).toContain('The capital of Zenland is Quixotic City.'); + + // Back to the built-in model: the choice is cleared the same way. + h.store.commitEmbeddingModel(null); + expect(h.store.activeEmbeddingModel()).toBeNull(); +}); diff --git a/__tests__/integration/downloads/downloadedCountBadge.rendered.happy.test.tsx b/__tests__/integration/downloads/downloadedCountBadge.rendered.happy.test.tsx index ac678c2e7..215ed053e 100644 --- a/__tests__/integration/downloads/downloadedCountBadge.rendered.happy.test.tsx +++ b/__tests__/integration/downloads/downloadedCountBadge.rendered.happy.test.tsx @@ -27,12 +27,13 @@ import { createDownloadedModel } from '../../utils/factories'; // Three recommended-model ids that render in the "More models" list at this test's RAM profile. A // downloaded row whose id STARTS WITH a recommended id makes that recommended card render its downloaded // mark (the real match rule: downloadedModels.some(m => m.id.startsWith(item.id))). On this 12 GB phone -// the Qwen 3.5 models are the device picks (quality-ranked): a pick is shown once, in the "For this -// phone" row, and left out of the list these cards belong to — so none of them is one of the three. +// the picks rank by quality alone (afdb05f): LFM2.5 8B A1B and MiniCPM5 2B are picks now. A pick is +// shown once, in the "For this phone" row, and left out of the list these cards belong to — so none +// of them is one of the three. const RECOMMENDED_IDS = [ 'unsloth/gemma-4-E2B-it-GGUF', - 'LiquidAI/LFM2.5-8B-A1B-GGUF', - 'openbmb/MiniCPM5-2B-GGUF', + 'unsloth/Qwen3.5-0.8B-GGUF', + 'unsloth/Qwen3.5-2B-GGUF', ]; async function mountWithNDownloaded(n: number) { diff --git a/__tests__/integration/downloads/videoPackFileRowsStayInternal.integration.test.ts b/__tests__/integration/downloads/videoPackFileRowsStayInternal.integration.test.ts new file mode 100644 index 000000000..9b88cfa26 --- /dev/null +++ b/__tests__/integration/downloads/videoPackFileRowsStayInternal.integration.test.ts @@ -0,0 +1,37 @@ +/** + * A video pack is one download to the person, however many files it has. + * + * SPEC (OGAM user's view): the Download Manager shows a Wan pack as one row with the pack's size and + * progress. Each file of the pack is its own native download (key `video-file:/`, model id + * `video:`), and after a relaunch those native file rows never become rows of their own: a file's + * row once replaced the pack's row and showed the file's size (100% before the pack was done). + * + * The real hydrateDownloadStore and download store. Device boundary only: the native download module + * (installNativeBoundary({ download: true })), seeded with what is in flight at relaunch. + */ +import { installNativeBoundary, GB } from '../../harness/nativeBoundary'; + +it('a relaunch hydrates other downloads but never a video pack\'s file as its own row', async () => { + const boundary = installNativeBoundary({ fs: true, download: true, ram: { platform: 'android', totalBytes: 12 * GB, availBytes: 8 * GB } }); + boundary.download!.seedActive({ + downloadId: 'dl-encoder', modelId: 'video:off-grid-ai/Wan2.2-TI2V-5B-GGUF-Q4', + fileName: 'off-grid-ai%2FWan2.2-TI2V-5B-GGUF-Q4_umt5-xxl-encoder-Q4_K_M.gguf', modelType: 'video', + bytesDownloaded: 3.3 * GB, totalBytes: 3.4 * GB, + modelKey: 'video-file:off-grid-ai/Wan2.2-TI2V-5B-GGUF-Q4/umt5-xxl-encoder-Q4_K_M.gguf', createdAt: 2, + }); + boundary.download!.seedActive({ + downloadId: 'dl-text', modelId: 'author/chat-model', fileName: 'chat-Q4_K_M.gguf', modelType: 'text', + bytesDownloaded: GB, totalBytes: 2 * GB, + modelKey: 'author/chat-model/chat-Q4_K_M.gguf', combinedTotalBytes: 2 * GB, createdAt: 1, + }); + const { hydrateDownloadStore } = require('../../../src/services/downloadHydration'); + const { useDownloadStore } = require('../../../src/stores/downloadStore'); + + await hydrateDownloadStore(); + + const keys = Object.keys(useDownloadStore.getState().downloads); + expect(keys).toContain('author/chat-model/chat-Q4_K_M.gguf'); + expect(keys.filter(key => key.startsWith('video-file:'))).toEqual([]); + // The file's native row is still there, for the video provider to reconcile by its pack id. + expect(boundary.download!.active().map(row => row.downloadId)).toContain('dl-encoder'); +}); diff --git a/__tests__/integration/happy/imageBackends.happy.test.tsx b/__tests__/integration/happy/imageBackends.happy.test.tsx index d7bec468b..e7f66ca9f 100644 --- a/__tests__/integration/happy/imageBackends.happy.test.tsx +++ b/__tests__/integration/happy/imageBackends.happy.test.tsx @@ -1,7 +1,7 @@ /** * HAPPY-PATH (UI, BEHAVIORAL) — image generation across compute backends via the REAL ChatScreen: the user * turns on the image-mode toggle (ON/force) and sends; the generated image's details show the correct - * backend label: NPU→"QNN (NPU)" · MNN(GPU)→"MNN (GPU)" · Metal(iOS)→"Core ML (ANE)". + * backend label: NPU→"QNN (NPU)" · MNN(GPU)→"MNN (GPU)" · Metal(iOS)→"Core ML". * * Only the native diffusion + LiteRT leaves are faked. The image model is DOWNLOADED (boundary) and ACTIVATED * by the real toggle gesture (not setState). Generation details are turned on via the real toggle. @@ -21,7 +21,7 @@ jest.mock('@react-navigation/native', () => ({ type Cfg = { label: string; backend: 'mnn' | 'qnn'; platform: 'ios' | 'android'; expected: string }; const CONFIGS: Cfg[] = [ { label: 'MNN GPU (Android)', backend: 'mnn', platform: 'android', expected: 'MNN (GPU)' }, - { label: 'Metal (Core ML, iOS)', backend: 'mnn', platform: 'ios', expected: 'Core ML (ANE)' }, + { label: 'Metal (Core ML, iOS)', backend: 'mnn', platform: 'ios', expected: 'Core ML' }, ]; describe('happy — image generation shows the correct backend label (heavy entry point)', () => { diff --git a/__tests__/integration/happy/imageIntentRouting.happy.test.tsx b/__tests__/integration/happy/imageIntentRouting.happy.test.tsx index 5579dfbc8..074fb1909 100644 --- a/__tests__/integration/happy/imageIntentRouting.happy.test.tsx +++ b/__tests__/integration/happy/imageIntentRouting.happy.test.tsx @@ -39,6 +39,17 @@ describe('happy — prompt routing picks the right model (heavy entry point)', ( await h.rtl.waitFor(() => { expect(h.boundary.diffusion.calls.generateImage.length).toBe(1); }); }); + it('a prompt-style description ("a red fox in fresh snow at dawn, photo") routes to the image model, not a text answer', async () => { + const h = await setupChatScreen({ engine: 'litert', platform: 'ios' }); + await withImageModel(h); + h.render(); + + await h.send('a red fox in fresh snow at dawn, photo', { content: 'I do not have image tools.' }); + + await h.rtl.waitFor(() => { expect(h.boundary.diffusion.calls.generateImage.length).toBe(1); }); + expect(h.view!.queryByText(/I do not have image tools\./)).toBeNull(); + }); + it('control: a normal prompt routes to the text model (no image generated)', async () => { const h = await setupChatScreen({ engine: 'litert', platform: 'ios' }); await withImageModel(h); diff --git a/__tests__/integration/image/gallerySelectModeImagesOnly.rendered.test.tsx b/__tests__/integration/image/gallerySelectModeImagesOnly.rendered.test.tsx new file mode 100644 index 000000000..7604373cf --- /dev/null +++ b/__tests__/integration/image/gallerySelectModeImagesOnly.rendered.test.tsx @@ -0,0 +1,51 @@ +/** + * Selecting images is an Images-tab action. + * + * SPEC (OGAM user's view): with images in the Gallery, the Images tab offers "Select images". The + * Videos tab does not, because a selection of images cannot be seen or acted on there. Moving to + * the Videos tab while selecting ends the selection. + * + * The real Gallery screen over the real store, rehydrated from device storage. Device boundaries + * only: the file system and the RAM sensor. No Pro, so the Videos tab stays on this screen. + */ +import { GB, installNativeBoundary, requireRTL } from '../../harness/nativeBoundary'; + +const image = (id: string) => ({ + id, + prompt: `A lighthouse ${id}`, + imagePath: `/data/user/0/ai.offgridmobile/files/generated_images/${id}.png`, + width: 512, + height: 512, + steps: 20, + seed: 1, + modelId: 'image-model', + createdAt: '2026-10-27T00:00:00.000Z', +}); + +it('offers Select images on the Images tab only, and leaving it ends the selection', async () => { + installNativeBoundary({ fs: true, ram: { platform: 'android', totalBytes: 8 * GB, availBytes: 5 * GB } }); + const AsyncStorage = require('@react-native-async-storage/async-storage'); + await AsyncStorage.clear(); + await AsyncStorage.setItem( + 'local-llm-app-storage', + JSON.stringify({ state: { generatedImages: [image('one'), image('two')] }, version: 0 }), + ); + const React = require('react'); + const rtl = requireRTL(); + const { useAppStore } = require('../../../src/stores/appStore'); + await useAppStore.persist.rehydrate(); + const { GalleryScreen } = require('../../../src/screens/GalleryScreen'); + const gallery = rtl.render(React.createElement(GalleryScreen)); + + await gallery.findByTestId('gallery-image-one'); + rtl.fireEvent.press(gallery.getByLabelText('Select images')); + expect(gallery.getByText('0 selected')).toBeTruthy(); + + rtl.fireEvent.press(gallery.getByText('Videos (0)')); + await rtl.waitFor(() => expect(gallery.queryByText('0 selected')).toBeNull()); + expect(gallery.queryByLabelText('Select images')).toBeNull(); + + rtl.fireEvent.press(gallery.getByText('Images (2)')); + expect(await gallery.findByLabelText('Select images')).toBeTruthy(); + expect(gallery.queryByText('0 selected')).toBeNull(); +}); diff --git a/__tests__/integration/image/imageDrawFailureTitle.rendered.test.tsx b/__tests__/integration/image/imageDrawFailureTitle.rendered.test.tsx new file mode 100644 index 000000000..54d2808de --- /dev/null +++ b/__tests__/integration/image/imageDrawFailureTitle.rendered.test.tsx @@ -0,0 +1,51 @@ +/** + * The failure card says where an image failed. + * + * SPEC (OGAM user's view): a model that loaded and then could not draw the image did not "fail to load". + * Its card reads "Image model: Couldn't make the image", gives the engine's one-line reason, and offers + * Retry. A model that fails to load keeps "Failed to Load Model". + * + * The REAL ChatScreen, imageGenerationService and ModelFailureCard. Only the native diffusion module is + * faked, rejecting as React Native delivers a native reject (the sd.cpp engine's IMAGE_FAILED). + */ +import { setupChatScreen } from '../../harness/chatHarness'; + +jest.mock('@react-navigation/native', () => ({ + useNavigation: () => ({ navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {} }), + useRoute: () => require('../../harness/chatHarness').routeHolder, + useFocusEffect: () => {}, + useIsFocused: () => true, +})); + +const nativeReject = (code: string, message: string) => Object.assign(new Error(message), { code }); + +async function openChatWithImageModeOn() { + const h = await setupChatScreen({ engine: 'litert', platform: 'android' }); + h.render(); + await h.placeImageModel({ backend: 'mnn' }); + await h.cycleImageMode(); + await h.rtl.waitFor(() => { expect(h.view!.queryByTestId('image-mode-force-badge')).not.toBeNull(); }); + return h; +} + +it('a model that loaded but could not draw says it could not make the image, with the reason and Retry', async () => { + const h = await openChatWithImageModeOn(); + h.boundary.diffusion.module.generateImage.mockRejectedValueOnce(nativeReject('IMAGE_FAILED', 'The image engine produced no image.')); + + await h.tapSend('a fox in the snow'); + + expect(await h.view!.findByText("Image model: Couldn't make the image")).toBeTruthy(); + expect(h.view!.getByText('The image engine produced no image.')).toBeTruthy(); + expect(h.view!.queryByText(/Failed to Load Model/)).toBeNull(); + expect(h.view!.getByTestId('model-failure-retry-image')).toBeTruthy(); +}); + +it('a model that cannot load keeps "Failed to Load Model"', async () => { + const h = await openChatWithImageModeOn(); + h.boundary.diffusion.module.loadModel.mockRejectedValueOnce(nativeReject('LOAD_FAILED', 'Could not load the image model pack.')); + + await h.tapSend('a fox in the snow'); + + expect(await h.view!.findByText(/Failed to Load Model/)).toBeTruthy(); + expect(h.view!.queryByText("Image model: Couldn't make the image")).toBeNull(); +}); diff --git a/__tests__/integration/image/imageParameterPolicy.test.ts b/__tests__/integration/image/imageParameterPolicy.test.ts index bff031452..daa55bf13 100644 --- a/__tests__/integration/image/imageParameterPolicy.test.ts +++ b/__tests__/integration/image/imageParameterPolicy.test.ts @@ -1,3 +1,4 @@ +import { standardImageModelDefaults } from '@offgrid/models'; import { resolveMobileImageParameters } from '../../../src/services/imageParameterPolicy'; describe('shared image parameter policy at the Mobile generation boundary', () => { @@ -6,14 +7,35 @@ describe('shared image parameter policy at the Mobile generation boundary', () = name: 'DreamShaper XL v2 Turbo', }; - it('uses current settings for both local and remote callers', () => { + it('an sd.cpp model uses its own values, not the native engines\' global ones', () => { + const lightning = { id: 'realvisxl-lightning', name: 'RealVisXL v5.0 Lightning (Light)', backend: 'sd' }; + const global = { imageSteps: 50, imageGuidanceScale: 7.5, imageWidth: 512 }; + // Its shared defaults (a few-step Lightning model, as on Desktop), however the global steps are set. + const own = standardImageModelDefaults(lightning.name); + expect(own.fewStep).toBe(true); + expect(resolveMobileImageParameters(lightning, global)).toEqual({ + steps: own.defaultSteps, + guidanceScale: own.defaultCfg, + size: own.defaultSize, + }); + expect(own.defaultSteps).toBeLessThan(global.imageSteps); + // Its own override, when the person set one for it. expect( - resolveMobileImageParameters(model, { + resolveMobileImageParameters(lightning, { + ...global, + imageParameterOverrides: { [lightning.id]: { steps: 6, cfgScale: 3, size: 512 } }, + }), + ).toEqual({ steps: 6, guidanceScale: 3, size: 512 }); + }); + + it('keeps a native image pack within its fixed 512 size', () => { + expect( + resolveMobileImageParameters({ ...model, backend: 'mnn' }, { imageSteps: 42, imageGuidanceScale: 5.5, imageWidth: 768, }), - ).toEqual({ steps: 42, guidanceScale: 5.5, size: 768 }); + ).toEqual({ steps: 42, guidanceScale: 5.5, size: 512 }); }); it('lets an explicit request win and falls back from damaged persisted values', () => { diff --git a/__tests__/integration/image/sdModelKeepsOwnSteps.rendered.test.tsx b/__tests__/integration/image/sdModelKeepsOwnSteps.rendered.test.tsx new file mode 100644 index 000000000..b7119f7ec --- /dev/null +++ b/__tests__/integration/image/sdModelKeepsOwnSteps.rendered.test.tsx @@ -0,0 +1,63 @@ +/** + * An sd.cpp model keeps its own image settings. + * + * SPEC (OGAM user's view): the steps the phone uses for an image depend on the model. A few-step + * sd.cpp model (RealVisXL Lightning) starts at its own steps from the shared policy (10, as on Off Grid + * Desktop), not the 50 the native image engines use; a change made while it is active is kept for that + * model alone, and the native engine's model still shows its own value. + * + * The real ImageQualityBasicSliders over the real app store, rehydrated from device storage the way a + * launch does. Device boundary only: storage (AsyncStorage) and the RAM sensor. + */ +import { GB, installNativeBoundary, requireRTL } from '../../harness/nativeBoundary'; + +const LIGHTNING = { + id: 'sd-offgrid-ai--realvisxl-v5.0-lightning-GGUF-Q4', name: 'RealVisXL v5.0 Lightning (Light)', + description: '', modelPath: '/models/realvisxl', downloadedAt: '2026-10-27T00:00:00.000Z', size: 2.6 * GB, backend: 'sd', +}; +const NATIVE = { + id: 'naianimev2_npu_min', name: 'Nai Anime V2 (NPU non-flagship)', + description: '', modelPath: '/models/nai', downloadedAt: '2026-10-27T00:00:00.000Z', size: 0.9 * GB, backend: 'qnn', +}; + +async function launchSliders(activeImageModelId: string, overrides?: Record) { + installNativeBoundary({ fs: true, ram: { platform: 'android', totalBytes: 12 * GB, availBytes: 6 * GB } }); + const AsyncStorage = require('@react-native-async-storage/async-storage'); + await AsyncStorage.clear(); + await AsyncStorage.setItem('local-llm-app-storage', JSON.stringify({ + state: { + downloadedImageModels: [LIGHTNING, NATIVE], activeImageModelId, + settings: { imageSteps: 50, imageGuidanceScale: 7.5, imageWidth: 512, imageHeight: 512, ...(overrides ? { imageParameterOverrides: overrides } : {}) }, + }, + version: 0, + })); + const React = require('react'); + const rtl = requireRTL(); + const { useAppStore } = require('../../../src/stores/appStore'); + await useAppStore.persist.rehydrate(); + const { ImageQualityBasicSliders } = require('../../../src/components/GenerationSettingsModal/ImageQualitySliders'); + return { rtl, view: rtl.render(React.createElement(ImageQualityBasicSliders)), useAppStore }; +} + +const stepsShown = (view: any) => Number(view.getByTestId('image-steps-value').props.children); + +it('a Lightning model starts at its own steps, and a change stays with it', async () => { + const first = await launchSliders(LIGHTNING.id); + expect(stepsShown(first.view)).toBe(10); + first.rtl.fireEvent.press(first.view.getByTestId('image-steps-value-button')); + first.rtl.fireEvent.changeText(first.view.getByTestId('image-steps-input'), '6'); + first.rtl.fireEvent(first.view.getByTestId('image-steps-input'), 'submitEditing'); + await first.rtl.waitFor(() => expect(stepsShown(first.view)).toBe(6)); + const saved = first.useAppStore.getState().settings.imageParameterOverrides; + first.view.unmount(); + + // The native engine's model, after a relaunch with what was saved: still the global 50. + const native = await launchSliders(NATIVE.id, saved); + expect(stepsShown(native.view)).toBe(50); + native.view.unmount(); + + // Back on the Lightning model: its own 6. + const again = await launchSliders(LIGHTNING.id, saved); + expect(stepsShown(again.view)).toBe(6); + again.view.unmount(); +}); diff --git a/__tests__/integration/memory/imageLowSystemMemory.happy.test.tsx b/__tests__/integration/memory/imageLowSystemMemory.happy.test.tsx new file mode 100644 index 000000000..2c60eebfb --- /dev/null +++ b/__tests__/integration/memory/imageLowSystemMemory.happy.test.tsx @@ -0,0 +1,48 @@ +/** + * HAPPY-PATH (UI, BEHAVIORAL — iPhone short of pages) — the phone can be short of memory while this + * app's own headroom still looks large. The OS then kills the app in the middle of the load. The + * memory guard must go by the smaller of the two, so the person sees the "Not Enough Memory" card + * and no crash. + * + * Device evidence: jetsam reports from the iPhone 17 Pro Max (docs/pr-screenshots/release-analytics-video/ + * acceptance/ios): the app was killed at ~3.3 GB with `vm-pageshortage` while os_proc_available_memory said + * several GB were free; the same phone generated normally when it had room. + * + * Real ChatScreen, real residency gate and image service; only the native leaves and the RAM sensor are faked. + */ +import { setupChatScreen } from '../../harness/chatHarness'; +import { GB } from '../../harness/nativeBoundary'; + +jest.mock('@react-navigation/native', () => ({ + useNavigation: () => ({ navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {} }), + useRoute: () => require('../../harness/chatHarness').routeHolder, + useFocusEffect: () => {}, + useIsFocused: () => true, +})); + +async function sendImagePromptOnPhoneWith(systemAvailBytes: number) { + const h = await setupChatScreen({ engine: 'litert', platform: 'ios' }); + h.render(); + await h.placeImageModel({ backend: 'coreml' }); + await h.cycleImageMode(); + await h.rtl.waitFor(() => { expect(h.view!.queryByTestId('image-mode-force-badge')).not.toBeNull(); }); + // A 12 GB iPhone: this app may still allocate 9 GB, but the whole phone has only `systemAvailBytes` to give. + h.boundary.setRam({ platform: 'ios', totalBytes: 12 * GB, availBytes: 9 * GB, systemAvailBytes }); + await require('../../../src/services/hardware').hardwareService.refreshMemoryInfo(); + await h.tapSend('a fox in the snow'); + return h; +} + +describe('happy — an iPhone short of pages is refused with the card, not killed mid-load', () => { + it('the app has headroom but the phone has 100 MB free: the card shows and no image is made', async () => { + const h = await sendImagePromptOnPhoneWith(100 * 1024 * 1024); + await h.rtl.waitFor(() => { expect(h.view!.queryByText(/Not Enough Memory/)).not.toBeNull(); }); + expect(h.boundary.diffusion.calls.generateImage.length).toBe(0); + }); + + it('control: the same phone with 8 GB free generates the image', async () => { + const h = await sendImagePromptOnPhoneWith(8 * GB); + await h.rtl.waitFor(() => { expect(h.boundary.diffusion.calls.generateImage.length).toBe(1); }); + expect(h.view!.queryByText(/Not Enough Memory/)).toBeNull(); + }); +}); diff --git a/__tests__/integration/memory/sdRecommendedIsAdmitted.test.ts b/__tests__/integration/memory/sdRecommendedIsAdmitted.test.ts new file mode 100644 index 000000000..6d672cc0f --- /dev/null +++ b/__tests__/integration/memory/sdRecommendedIsAdmitted.test.ts @@ -0,0 +1,44 @@ +/** + * A recommended sd.cpp model is not then refused by the memory guard. + * + * SPEC (OGAM user's view): the Image tab recommends only sd.cpp models this phone can load, and loading + * one at the phone's usual free memory works without "Run anyway". One estimate decides both: the + * recommendation and the guard read hardwareService.estimateImageModelRam, which for sd.cpp follows the + * device measurement (RealVisXL Lightning Q4, 2.67 GB file, peaked at 3.33 GB on a OnePlus SM8635). + * + * The real recommendation (huggingFaceModelBrowser) and residency manager over the RAM sensor boundary, + * set to that OnePlus: 11 GB, with the 4.5 GB free it usually had. + */ +import { modelResidencyManager } from '../../../src/services/modelResidency'; +import { setDeviceMemory, resetDeviceMemory } from '../../harness/deviceMemory'; + +afterEach(() => resetDeviceMemory()); + +const LIGHT = 'RealVisXL v5.0 Lightning (Light)'; + +it('the recommended RealVisXL Light loads at the OnePlus\'s usual free memory, without Run anyway', async () => { + setDeviceMemory({ platform: 'android', totalGB: 11, availGB: 4.5 }); + const { getRecommendedSDImageModels } = require('../../../src/services/huggingFaceModelBrowser'); + const { hardwareService } = require('../../../src/services/hardware'); + const light = getRecommendedSDImageModels(11).find((model: { name: string }) => model.name === LIGHT); + expect(light).toBeTruthy(); + + const sizeMB = Math.round(hardwareService.estimateImageModelRam({ size: light.size, backend: 'sd' }) / (1024 * 1024)); + // Close to the measured 3.33 GB peak, not the old 2.5x (6.7 GB). + expect(sizeMB).toBeGreaterThan(3330); + expect(sizeMB).toBeLessThan(3700); + const { fits } = await modelResidencyManager.makeRoomFor({ + key: 'image', type: 'image', modelId: light.id, sizeMB, dirtyMemory: true, + }); + expect(fits).toBe(true); +}); + +it('a model too large for the budget is not recommended at all', () => { + setDeviceMemory({ platform: 'android', totalGB: 8, availGB: 4 }); + const { getRecommendedSDImageModels } = require('../../../src/services/huggingFaceModelBrowser'); + const { hardwareService } = require('../../../src/services/hardware'); + const { computeBudgetMB } = require('../../../src/services/modelResidency/budget'); + const budget = computeBudgetMB(8 * 1024) * 1024 * 1024; + for (const model of getRecommendedSDImageModels(8)) + expect(hardwareService.estimateImageModelRam({ size: model.size, backend: 'sd' })).toBeLessThanOrEqual(budget); +}); diff --git a/__tests__/integration/memory/textLoadLowSystemMemory.rendered.test.tsx b/__tests__/integration/memory/textLoadLowSystemMemory.rendered.test.tsx new file mode 100644 index 000000000..008760311 --- /dev/null +++ b/__tests__/integration/memory/textLoadLowSystemMemory.rendered.test.tsx @@ -0,0 +1,49 @@ +/** + * An iPhone short of free pages is refused a text-model load with the memory message, not killed. + * + * DEVICE GROUND TRUTH (iPhone 17 Pro Max, 2026-10-27, jetsam + [WIRE-RAM] log): a 5.3 GB Qwen3.5-9B Q4 was + * loaded with its weights mapped while the phone had 1.0 GB of free pages (systemAvailableBytes 1033961472) + * and the app's own headroom read 4.5 GB. The gate said safe (mapped weights are not charged to the app's + * limit); the GPU wired them, the OS ran out of pages and killed the app within seconds. + * + * The gate now also needs the phone's free pages to cover mapped weights + KV cache + headroom. + * + * Real ChatScreen and llm service; only the native engine, the disk and the RAM sensor are faked. + */ +import { setupChatScreen } from '../../harness/chatHarness'; +import { GB } from '../../harness/nativeBoundary'; + +jest.mock('@react-navigation/native', () => ({ + useNavigation: () => ({ navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {} }), + useRoute: () => require('../../harness/chatHarness').routeHolder, + useFocusEffect: () => {}, + useIsFocused: () => true, +})); + +async function sendWithSystemFree(systemAvailBytes: number) { + const h = await setupChatScreen({ + engine: 'llama', + platform: 'ios', + modelFileSizeBytes: 3.5 * GB, + ram: { platform: 'ios', totalBytes: 12 * GB, availBytes: 8 * GB, systemAvailBytes }, + deferInitialLoad: true, + }); + h.boundary.fs!.seedFile('/docs/models/ggml-small.gguf', Math.round(5 * GB)); + h.render(); + await h.send('what is the capital of France', { text: 'Paris.' }); + return h; +} + +describe('iPhone text load goes by the phone\'s free pages too', () => { + it('1 GB of free pages: the memory refusal shows and no reply is made', async () => { + const h = await sendWithSystemFree(1 * GB); + await h.rtl.waitFor(() => { expect(h.view!.queryByText(/it needs ~/)).not.toBeNull(); }, { timeout: 8000 }); + expect(h.view!.queryByText(/Paris\./)).toBeNull(); + }, 30000); + + it('control: 9 GB of free pages loads the model and the reply renders', async () => { + const h = await sendWithSystemFree(9 * GB); + await h.rtl.waitFor(() => { expect(h.view!.queryByText(/Paris\./)).not.toBeNull(); }, { timeout: 8000 }); + expect(h.view!.queryByText(/it needs ~/)).toBeNull(); + }, 30000); +}); diff --git a/__tests__/integration/models/catalogChoice.rendered.test.tsx b/__tests__/integration/models/catalogChoice.rendered.test.tsx index 24a0599d9..2507f2d9c 100644 --- a/__tests__/integration/models/catalogChoice.rendered.test.tsx +++ b/__tests__/integration/models/catalogChoice.rendered.test.tsx @@ -28,10 +28,11 @@ describe('Model choice on the phone', () => { const screen = render(React.createElement(ModelsScreen)); await waitFor(() => { - const minicpm = within(screen.getByLabelText('MiniCPM5 2B')); - expect(minicpm.getByText('MiniCPM5 2B')).toBeTruthy(); - expect(minicpm.getByLabelText('Thinking likely')).toBeTruthy(); - expect(minicpm.getByLabelText('Tool calling likely')).toBeTruthy(); + // MiniCPM5 2B is now a pick on this phone (afdb05f, ddaa6c1), shown above the cards. + const qwen = within(screen.getByLabelText('Qwen 3.5 2B')); + expect(qwen.getByText('Qwen 3.5 2B')).toBeTruthy(); + expect(qwen.getByLabelText('Thinking likely')).toBeTruthy(); + expect(qwen.getByLabelText('Tool calling likely')).toBeTruthy(); const gemma = within(screen.getByLabelText('Gemma 4 E2B')); expect(gemma.getByText('Gemma 4 E2B')).toBeTruthy(); diff --git a/__tests__/integration/models/catalogGgufDisplayName.rendered.test.tsx b/__tests__/integration/models/catalogGgufDisplayName.rendered.test.tsx index 2fc770a1e..c12b0949f 100644 --- a/__tests__/integration/models/catalogGgufDisplayName.rendered.test.tsx +++ b/__tests__/integration/models/catalogGgufDisplayName.rendered.test.tsx @@ -105,7 +105,8 @@ describe('catalog GGUF model names', () => { download: true, fs: true, llama: true, - ram: { platform: 'android', totalBytes: 12 * GB, availBytes: 8 * GB }, + // 16 GB: LFM2.5 8B A1B is a list card here. On 12 GB it is now a Fast pick in the picks row (ddaa6c1). + ram: { platform: 'android', totalBytes: 16 * GB, availBytes: 10 * GB }, }); const restoreFetch = installHuggingFace(); try { diff --git a/__tests__/integration/models/chatHomeSelectorParity.test.ts b/__tests__/integration/models/chatHomeSelectorParity.test.ts index f5afc33bd..59d9b27ce 100644 --- a/__tests__/integration/models/chatHomeSelectorParity.test.ts +++ b/__tests__/integration/models/chatHomeSelectorParity.test.ts @@ -36,7 +36,7 @@ import { handleModelSelectFn } from '../../../src/screens/ChatScreen/useChatMode jest.mock('../../../src/services/llm'); jest.mock('../../../src/services/localDreamGenerator'); jest.mock('../../../src/services/hardware'); -jest.mock('../../../src/utils/imageModelIntegrity', () => ({ +jest.mock('../../../src/services/imageModelIntegrity', () => ({ validateImageModelDir: jest.fn(async () => ({ complete: true, missing: [] })), ensureImageExtractionComplete: jest.fn(async () => {}), })); diff --git a/__tests__/integration/models/checkForNewModels.rendered.test.tsx b/__tests__/integration/models/checkForNewModels.rendered.test.tsx index 8c35962eb..b7629b33b 100644 --- a/__tests__/integration/models/checkForNewModels.rendered.test.tsx +++ b/__tests__/integration/models/checkForNewModels.rendered.test.tsx @@ -14,14 +14,29 @@ import { installNativeBoundary, requireRTL, GB } from '../../harness/nativeBound const CATALOG_URL = 'https://raw.githubusercontent.com/off-grid-ai/OGAM/main/catalog/models.json'; /** The catalog file as it is published: the envelope with the sections the app reads. */ -function catalogFile(text: unknown[]): string { +function catalogFile(text: unknown[], models: unknown[] = []): string { return JSON.stringify({ schemaVersion: 1, updatedAt: '2026-10-22', - sections: { models: [], text, transcription: [], coreml: [], litert: [] }, + sections: { models, text, transcription: [], coreml: [], litert: [] }, }); } +/** An sd.cpp image model added to the catalog after this build shipped (the full `models` shape). */ +const NEW_IMAGE_MODEL = { + id: 'someone/Brandnew-Image-GGUF', + name: 'Brandnew Image', + kind: 'image', + org: 'Someone', + minRamGb: 8, + files: [{ + name: 'brandnew-image-q4_0.gguf', + url: 'https://huggingface.co/someone/Brandnew-Image-GGUF/resolve/main/brandnew-image-q4_0.gguf', + sizeBytes: 2_000_000_000, + role: 'primary', + }], +}; + /** A chat model added to the catalog after this build shipped (the README's `text` shape). */ const NEW_MODEL = { id: 'someone/Brandnew-3B-GGUF', @@ -149,4 +164,29 @@ describe('Check for new models on the Models screen', () => { const listAfter = within(models.getByTestId('models-list')).getAllByText(/./).map(n => n.props.children); expect(listAfter).toEqual(listBefore); }); + + it('lists an sd.cpp image model the catalog adds on the Image tab, without an app update', async () => { + installNativeBoundary({ fs: true, ram: { platform: 'android', totalBytes: 12 * GB, availBytes: 8 * GB } }); + network = installNetwork(catalogFile([], [NEW_IMAGE_MODEL])); + const React = require('react'); + const { render, fireEvent, waitFor, within } = requireRTL(); + await freshLaunch(); + const { ModelsScreen } = require('../../../src/screens/ModelsScreen'); + + const models = render(React.createElement(ModelsScreen)); + await waitFor(() => expect(models.getByTestId('device-picks')).toBeTruthy()); + fireEvent.press(models.getByText('Image')); + // BEFORE: a bundled sd.cpp model is listed; the catalog's new one is not. + await waitFor(() => expect(models.getAllByText(/^FLUX\.2 klein 4B/).length).toBeGreaterThan(0), { timeout: 4000 }); + expect(models.queryByText('Brandnew Image')).toBeNull(); + + fireEvent.press(models.getByTestId('check-new-models-button')); + + // AFTER: the sheet names it, and the Image tab lists it now (a second copy, outside the sheet), + // beside the bundled ones. + const sheet = await waitFor(() => models.getByTestId('catalog-check-sheet')); + expect(within(sheet).getByText('Brandnew Image')).toBeTruthy(); + await waitFor(() => expect(models.getAllByText('Brandnew Image')).toHaveLength(2), { timeout: 4000 }); + expect(models.getAllByText(/^FLUX\.2 klein 4B/).length).toBeGreaterThan(0); + }); }); diff --git a/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx b/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx index a4022ef72..7a01ce8b2 100644 --- a/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx +++ b/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx @@ -96,6 +96,7 @@ jest.mock('../../../src/services/huggingFaceModelBrowser', () => ({ fetchAvailableModels: jest.fn(() => Promise.resolve([])), getVariantLabel: jest.fn(() => 'Standard'), guessStyle: jest.fn(() => 'creative'), + getRecommendedSDImageModels: jest.fn(() => []), })); jest.mock('../../../src/services/coreMLModelBrowser', () => ({ diff --git a/__tests__/integration/models/devicePicks.rendered.test.tsx b/__tests__/integration/models/devicePicks.rendered.test.tsx index 97b17ea2d..7f604b03d 100644 --- a/__tests__/integration/models/devicePicks.rendered.test.tsx +++ b/__tests__/integration/models/devicePicks.rendered.test.tsx @@ -73,29 +73,30 @@ describe('Fast, Balanced and Best on the Models screen', () => { const { render, fireEvent, waitFor, within, act } = requireRTL(); await clearStorage(); const { CATALOG } = require('@offgrid/models'); - const fastEntry = CATALOG.find((e: { name: string }) => e.name === 'Qwen 3.5 0.8B'); + const fastEntry = CATALOG.find((e: { name: string }) => e.name === 'LFM2.5 8B A1B'); const primary = fastEntry.files.find((f: { role?: string }) => f.role === 'primary'); const { ModelsScreen } = require('../../../src/screens/ModelsScreen'); const models = render(React.createElement(ModelsScreen)); await waitFor(() => expect(models.getByTestId('device-picks')).toBeTruthy()); expect(within(models.getByTestId('device-picks')).getByText('For this phone')).toBeTruthy(); - // Ranked by the catalog's quality score among models that run well here, from makers people - // know; replaced models (Qwen3-VL, the Qwen 3.8 distills) are never picked. - expect(within(models.getByTestId('pick-fast')).getByText('Qwen 3.5 0.8B')).toBeTruthy(); - expect(within(models.getByTestId('pick-balanced')).getByText('Qwen 3.5 2B')).toBeTruthy(); - expect(within(models.getByTestId('pick-best')).getByText('Qwen 3.5 4B')).toBeTruthy(); + // Ranked by the catalog's quality score alone among models that run well here, whoever made + // them (afdb05f), on Android's budget (ddaa6c1); replaced models (Qwen3-VL, the Qwen 3.8 + // distills) are never picked. + expect(within(models.getByTestId('pick-fast')).getByText('LFM2.5 8B A1B')).toBeTruthy(); + expect(within(models.getByTestId('pick-balanced')).getByText('Qwen 3.5 4B')).toBeTruthy(); + expect(within(models.getByTestId('pick-best')).getByText('MiniCPM5 2B')).toBeTruthy(); expect(models.queryByText('Qwen3-VL 2B')).toBeNull(); - // A lesser-known lab's model stays in the list below, not in the picks. - expect(within(models.getByTestId('device-picks')).queryByText('MiniCPM5 2B')).toBeNull(); - expect(models.getByText('MiniCPM5 2B')).toBeTruthy(); + // A model that is not picked stays in the list below, not in the picks. + expect(within(models.getByTestId('device-picks')).queryByText('Qwen 3.5 0.8B')).toBeNull(); + expect(models.getByText('Qwen 3.5 0.8B')).toBeTruthy(); // The old RAM banner is gone. expect(models.queryByText(/RAM — models up to/)).toBeNull(); // BEFORE: nothing downloading, the Fast pick offers a download and no "Chat now". expect(boundary.download!.active()).toHaveLength(0); expect(models.queryByTestId('pick-fast-next')).toBeNull(); - expect(models.getByTestId('pick-fast-download').props.accessibilityLabel).toBe('Download Qwen 3.5 0.8B'); + expect(models.getByTestId('pick-fast-download').props.accessibilityLabel).toBe('Download LFM2.5 8B A1B'); fireEvent.press(models.getByTestId('pick-fast-download')); @@ -160,11 +161,12 @@ describe('Fast, Balanced and Best on the Models screen', () => { const models = render(React.createElement(ModelsScreen)); await waitFor(() => expect(models.getByTestId('device-picks')).toBeTruthy()); - expect(within(models.getByTestId('pick-best')).getByText('Qwen 3.5 0.8B')).toBeTruthy(); - expect(models.queryByTestId('pick-fast')).toBeNull(); - expect(models.queryByTestId('pick-balanced')).toBeNull(); - // None of the 12 GB picks is offered to this phone. - expect(within(models.getByTestId('device-picks')).queryByText('MiniCPM5 2B')).toBeNull(); + // Android runs well up to 52% of RAM (ddaa6c1): 2.1 GB here, so three small models qualify. + expect(within(models.getByTestId('pick-fast')).getByText('Qwen 3.5 0.8B')).toBeTruthy(); + expect(within(models.getByTestId('pick-balanced')).getByText('Qwen 3.5 2B')).toBeTruthy(); + expect(within(models.getByTestId('pick-best')).getByText('MiniCPM5 2B')).toBeTruthy(); + // None of the 12 GB phone's larger picks is offered to this phone. + expect(within(models.getByTestId('device-picks')).queryByText('LFM2.5 8B A1B')).toBeNull(); expect(within(models.getByTestId('device-picks')).queryByText('Qwen 3.5 4B')).toBeNull(); }); @@ -306,8 +308,9 @@ describe('Text search answers from the catalog first', () => { expect(within(gemma).getByLabelText('MTP: replies faster')).toBeTruthy(); // A screen reader focuses the whole card, so the card itself says it. expect(gemma.props.accessibilityHint).toContain('MTP: replies faster'); - const minicpm = models.getByLabelText('MiniCPM5 2B'); - expect(within(minicpm).queryByLabelText('MTP: replies faster')).toBeNull(); + // MiniCPM5 2B is now one of this phone's picks (afdb05f), so the card without an MTP head is Qwen 3.5 4B's. + const qwen = models.getByLabelText('Qwen 3.5 4B'); + expect(within(qwen).queryByLabelText('MTP: replies faster')).toBeNull(); }); }); diff --git a/__tests__/integration/models/downloadButtonsNameTheirModel.rendered.test.tsx b/__tests__/integration/models/downloadButtonsNameTheirModel.rendered.test.tsx index a385b027b..96629ea21 100644 --- a/__tests__/integration/models/downloadButtonsNameTheirModel.rendered.test.tsx +++ b/__tests__/integration/models/downloadButtonsNameTheirModel.rendered.test.tsx @@ -68,9 +68,10 @@ function downloadCards(tree: unknown, cardId: RegExp): { card: string; shows: st return out; } -const TEXT_REPO = 'openbmb/MiniCPM5-2B-GGUF'; -const TEXT_NAME = 'MiniCPM5 2B'; -const TEXT_FILES = [['MiniCPM5-2B-Q4_K_M.gguf', 1500 * MB], ['MiniCPM5-2B-Q8_0.gguf', 2600 * MB]] as const; +// A list model, not one of this phone's picks (afdb05f made MiniCPM5 2B a pick on 12 GB). +const TEXT_REPO = 'unsloth/Qwen3.5-2B-GGUF'; +const TEXT_NAME = 'Qwen 3.5 2B'; +const TEXT_FILES = [['Qwen3.5-2B-Q4_K_M.gguf', 1300 * MB], ['Qwen3.5-2B-Q8_0.gguf', 2200 * MB]] as const; const ZIP_NAME = 'AnythingV5.zip'; const IMAGE_NAME = 'Anything V5 (GPU)'; @@ -91,7 +92,7 @@ function installNetwork(): () => void { const parsed = new URL(url); if (parsed.pathname === `/api/models/${TEXT_REPO}` && parsed.searchParams.get('blobs') === 'true') { return new Response(JSON.stringify({ - id: TEXT_REPO, author: 'openbmb', downloads: 1, likes: 0, tags: ['gguf'], + id: TEXT_REPO, author: 'unsloth', downloads: 1, likes: 0, tags: ['gguf'], siblings: TEXT_FILES.map(([rfilename, size]) => ({ rfilename, size, lfs: { size } })), }), { status: 200 }); } @@ -136,10 +137,11 @@ describe('Download buttons say which model they download', () => { const { waitFor, fireEvent } = rtl; await waitFor(() => expect(models.getByTestId('device-picks')).toBeTruthy()); - // The picks row: each pick's download button says the pick's name. - for (const slot of ['fast', 'balanced', 'best']) { - const pick = models.getByTestId(`pick-${slot}`); - const name = texts(rtl.within(pick).getByText(/^Qwen /).props.children).join(''); + // The picks row: each pick's download button says the pick's name (this 12 GB phone's + // quality-ranked picks, afdb05f and ddaa6c1). + const picks = { fast: 'LFM2.5 8B A1B', balanced: 'Qwen 3.5 4B', best: 'MiniCPM5 2B' }; + for (const [slot, name] of Object.entries(picks)) { + expect(rtl.within(models.getByTestId(`pick-${slot}`)).getByText(name)).toBeTruthy(); expect(models.getByTestId(`pick-${slot}-download`).props.accessibilityLabel).toBe(`Download ${name}`); } @@ -149,16 +151,16 @@ describe('Download buttons say which model they download', () => { await waitFor(() => expect(models.getByTestId('file-card-1')).toBeTruthy()); const cards = downloadCards(models.toJSON(), /^file-card-\d+$/); expect(cards).toHaveLength(2); - expect(cards.map(c => c.shows).sort()).toEqual(['MiniCPM5-2B-Q4_K_M', 'MiniCPM5-2B-Q8_0']); + expect(cards.map(c => c.shows).sort()).toEqual(['Qwen3.5-2B-Q4_K_M', 'Qwen3.5-2B-Q8_0']); for (const { shows, says } of cards) expect(says).toBe(`Download ${shows}`); expect(models.queryAllByRole('button', { name: 'Download model' })).toHaveLength(0); // BEFORE: nothing downloading. expect(boundary.download!.active()).toHaveLength(0); - fireEvent.press(models.getByLabelText('Download MiniCPM5-2B-Q8_0')); + fireEvent.press(models.getByLabelText('Download Qwen3.5-2B-Q8_0')); // The device is asked for exactly that file. - await waitFor(() => expect(boundary.download!.active().map(row => row.fileName)).toEqual(['MiniCPM5-2B-Q8_0.gguf'])); + await waitFor(() => expect(boundary.download!.active().map(row => row.fileName)).toEqual(['Qwen3.5-2B-Q8_0.gguf'])); }); it('Image: each card names its own model, and that button downloads that model', async () => { diff --git a/__tests__/integration/models/failedRemoteRefreshClearsSelection.rendered.test.tsx b/__tests__/integration/models/failedRemoteRefreshClearsSelection.rendered.test.tsx index 103c01c11..e70c5b5e4 100644 --- a/__tests__/integration/models/failedRemoteRefreshClearsSelection.rendered.test.tsx +++ b/__tests__/integration/models/failedRemoteRefreshClearsSelection.rendered.test.tsx @@ -17,12 +17,14 @@ function ModelsSheet() { labels={{ text: textModel ?? 'Local text', image: imageModel ?? 'Local image', + video: mediaServers.video ? 'Cloud video' : 'Local video', voice: mediaServers.voice ? 'Cloud voice' : 'Local voice', speech: mediaServers.transcription ? 'Cloud speech' : 'Local speech', }} remote={{ text: !!textModel, image: !!mediaServers.image, + video: !!mediaServers.video, voice: !!mediaServers.voice, speech: !!mediaServers.transcription, }} @@ -46,6 +48,7 @@ describe('remote model refresh failure', () => { providerType: 'openai-compatible', mediaModels: { image: 'cloud-image', + video: 'cloud-video', transcription: 'cloud-speech', voice: 'cloud-voice', }, @@ -56,6 +59,7 @@ describe('remote model refresh failure', () => { store.setActiveRemoteTextModelId('cloud-text'); store.setActiveRemoteImageModelId('cloud-image'); store.setActiveRemoteMediaServerId('image', 'unavailable-gateway'); + store.setActiveRemoteMediaServerId('video', 'unavailable-gateway'); store.setActiveRemoteMediaServerId('transcription', 'unavailable-gateway'); store.setActiveRemoteMediaServerId('voice', 'unavailable-gateway'); global.fetch = async () => @@ -64,7 +68,7 @@ describe('remote model refresh failure', () => { it('unsets only the unreachable server models after Refresh Models', async () => { const view = render(); - expect(view.getAllByTestId(/models-row-.*-remote/)).toHaveLength(4); + expect(view.getAllByTestId(/models-row-.*-remote/)).toHaveLength(5); fireEvent.press(view.getByTestId('models-refresh-remote')); @@ -74,6 +78,7 @@ describe('remote model refresh failure', () => { }); expect(view.getByText('Local text')).toBeTruthy(); expect(view.getByText('Local image')).toBeTruthy(); + expect(view.getByText('Local video')).toBeTruthy(); expect(view.getByText('Local voice')).toBeTruthy(); expect(view.getByText('Local speech')).toBeTruthy(); }); diff --git a/__tests__/integration/models/imageDownloadRecovery.test.ts b/__tests__/integration/models/imageDownloadRecovery.test.ts index 4cc0f7fcf..198055cbb 100644 --- a/__tests__/integration/models/imageDownloadRecovery.test.ts +++ b/__tests__/integration/models/imageDownloadRecovery.test.ts @@ -14,7 +14,7 @@ const mockModelManager = { // Integrity is a boundary for this recovery test (it exercises the resume/move // orchestration, not the completeness rule — that has its own dedicated tests). -jest.mock('../../../src/utils/imageModelIntegrity', () => ({ +jest.mock('../../../src/services/imageModelIntegrity', () => ({ validateImageModelDir: jest.fn(async () => ({ complete: true, missing: [] })), ensureImageExtractionComplete: jest.fn(async () => {}), })); diff --git a/__tests__/integration/models/imageModels2026Picks.rendered.test.tsx b/__tests__/integration/models/imageModels2026Picks.rendered.test.tsx new file mode 100644 index 000000000..13b68865a --- /dev/null +++ b/__tests__/integration/models/imageModels2026Picks.rendered.test.tsx @@ -0,0 +1,66 @@ +/** + * The Image tab offers this phone the 2026 image models it can run. + * + * A model is offered only where the memory guard will load it (934ffd377: one estimate decides both). + * An 8 GB Snapdragon phone (it reports about 7.4 GB) has about 4.5 GB for a model, and FLUX.2 klein 4B + * (Light) needs about 6.4 GB, so it is not offered there. A 12 GB Snapdragon phone (about 11.2 GB) lists + * it, and Z-Image Turbo (Light), with how long an image takes; they are not "For this phone" picks, + * because sd.cpp runs only on the CPU on Snapdragon (fb5ab710f) and Local Dream is the pick there. + * SDXL Lightning and SDXL Turbo, which FLUX.2 klein replaces, are no longer offered. + * + * Real ModelsScreen, Image tab, the shared catalog and pack rules, the sd.cpp model list and the + * memory recommendation. Fakes only at the device boundary: the RAM sensor, the file system and the + * network (Hugging Face through fetch, answering with no extra models). + */ +import { installNativeBoundary, requireRTL, GB } from '../../harness/nativeBoundary'; + +describe('2026 image models on the Image tab', () => { + let originalFetch: typeof fetch; + beforeEach(() => { + originalFetch = globalThis.fetch; + globalThis.fetch = (async () => new Response('[]', { status: 200 })) as typeof fetch; + }); + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it.each([ + { phone: '8 GB', reportedBytes: 7.4 * GB, flux: false, zImageLight: false }, + { phone: '12 GB', reportedBytes: 11.2 * GB, flux: true, zImageLight: true }, + ])('a $phone Snapdragon phone is offered only what it can load, never SDXL Lightning or Turbo', async ({ reportedBytes, flux, zImageLight }) => { + installNativeBoundary({ fs: true, deviceHardware: 'qcom', socModel: 'SM8635', ram: { platform: 'android', totalBytes: reportedBytes, availBytes: reportedBytes / 2 } }); + const React = require('react'); + const { render, fireEvent, waitFor } = requireRTL(); + const AsyncStorage = + require('@react-native-async-storage/async-storage').default ?? + require('@react-native-async-storage/async-storage'); + await AsyncStorage.clear(); + // What App.tsx does at launch: read the device first. + await require('../../../src/services/hardware').hardwareService.getDeviceInfo(); + const { ModelsScreen } = require('../../../src/screens/ModelsScreen'); + + const models = render(React.createElement(ModelsScreen)); + // Before: the Text tab shows, with no image model on screen. + expect(models.queryAllByText(/FLUX\.2 klein 4B/)).toHaveLength(0); + + fireEvent.press(models.getByText('Image')); + if (flux) { + // Listed with its honest time on this phone, and not a pick. + await waitFor(() => expect(models.getByText('FLUX.2 klein 4B (Light)')).toBeTruthy(), { timeout: 4000 }); + expect(models.getAllByText(/About \d+ min per image on this phone/).length).toBeGreaterThan(0); + expect(models.queryByTestId('image-picks')).toBeNull(); + } else { + // The tab has finished loading and rendered its (empty) list before absence is checked: the empty + // state shows only once loading is over. + await waitFor(() => expect(models.getByText(/^No models (available|match your filters)$/)).toBeTruthy(), { timeout: 4000 }); + expect(models.queryAllByText(/FLUX\.2 klein/)).toHaveLength(0); + } + + // The SDXL finetunes stay as style options; the replaced SDXL distills are gone. + expect(models.queryAllByText(/SDXL Lightning/)).toHaveLength(0); + expect(models.queryAllByText(/SDXL Turbo/)).toHaveLength(0); + expect(models.queryAllByText(/Z-Image Turbo \(Light\)/).length > 0).toBe(zImageLight); + // FLUX 3 Image Klein needs a newer engine than phones ship: never offered here. + expect(models.queryAllByText(/FLUX 3 Image Klein/)).toHaveLength(0); + }); +}); diff --git a/__tests__/integration/models/imageModelsOffGridMark.rendered.test.tsx b/__tests__/integration/models/imageModelsOffGridMark.rendered.test.tsx new file mode 100644 index 000000000..c29e25638 --- /dev/null +++ b/__tests__/integration/models/imageModelsOffGridMark.rendered.test.tsx @@ -0,0 +1,51 @@ +/** + * Off Grid AI's own image builds carry its mark on the Image tab, in the picks and in the list, + * as text and video models do. Another source's model does not. + * + * Real ModelsScreen, Image tab, picks row, cards and the shared credibility rule. Fakes only at the + * device boundary: the RAM sensor, the file system and the network (Hugging Face through fetch). + */ +import { installNativeBoundary, requireRTL, GB } from '../../harness/nativeBoundary'; + +describe('Off Grid AI mark on image models', () => { + it('marks exactly Off Grid AI builds, in the list and the picks', async () => { + installNativeBoundary({ fs: true, ram: { platform: 'android', totalBytes: 12 * GB, availBytes: 8 * GB } }); + const originalFetch = globalThis.fetch; + globalThis.fetch = (async () => new Response('[]', { status: 200 })) as typeof fetch; + try { + const React = require('react'); + const { render, fireEvent, waitFor, within } = requireRTL(); + const AsyncStorage = + require('@react-native-async-storage/async-storage').default ?? + require('@react-native-async-storage/async-storage'); + await AsyncStorage.clear(); + // What App.tsx does at launch: read the device first. + await require('../../../src/services/hardware').hardwareService.getDeviceInfo(); + const { ModelsScreen } = require('../../../src/screens/ModelsScreen'); + + const models = render(React.createElement(ModelsScreen)); + fireEvent.press(models.getByText('Image')); + // Off Grid AI's own image builds (huggingface.co/offgrid-ai): the SDXL style family. + const OURS = /\b(RealVisXL|Juggernaut|DreamShaper|Animagine|Illustrious)\b/; + await waitFor(() => expect(models.queryAllByTestId(/^image-model-card-\d+$/).length).toBeGreaterThan(0), { timeout: 4000 }); + + // Every card and pick shown: the mark appears exactly on Off Grid AI's builds, whatever this + // phone is offered (the offer follows the phone's memory). + const named = (node: any) => within(node).queryAllByText(/./).map((t: any) => String(t.props.children)).join(' '); + const shown = [ + ...models.queryAllByTestId(/^image-model-card-\d+$/), + ...['fast', 'balanced', 'best'].map(slot => models.queryByTestId(`pick-${slot}`)) + .filter((node): node is NonNullable => node !== null), + ]; + let ourCount = 0; + for (const node of shown) { + const ours = OURS.test(named(node).trim()); + if (ours) ourCount++; + expect([named(node), within(node).queryAllByTestId('made-by-off-grid').length > 0]).toEqual([named(node), ours]); + } + expect(ourCount).toBeGreaterThan(0); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); diff --git a/__tests__/integration/models/makerFilters.rendered.test.tsx b/__tests__/integration/models/makerFilters.rendered.test.tsx index 5c0787c3b..74afb144a 100644 --- a/__tests__/integration/models/makerFilters.rendered.test.tsx +++ b/__tests__/integration/models/makerFilters.rendered.test.tsx @@ -61,9 +61,17 @@ describe('Maker filters on the Models screen', () => { const models = render(React.createElement(ModelsScreen)); await waitFor(() => expect(models.getByTestId('device-picks')).toBeTruthy()); - // BEFORE: no maker chosen, both models are offered (a device pick sits above the list). + // BEFORE: no maker chosen, both models are offered (a device pick sits above the list). The + // list draws a window of cards; with today's larger phone picks (ddaa6c1, afdb05f) the largest + // models sit past it, so the person scrolls to the end. const list = () => within(models.getByTestId('models-list')); - expect(models.getAllByText(model).length).toBeGreaterThan(0); + const scroller = models.getByTestId('models-list'); + fireEvent(scroller, 'layout', { nativeEvent: { layout: { x: 0, y: 0, width: 400, height: 800 } } }); + fireEvent(scroller, 'contentSizeChange', 400, 4000); + fireEvent.scroll(scroller, { + nativeEvent: { contentOffset: { x: 0, y: 3200 }, contentSize: { width: 400, height: 4000 }, layoutMeasurement: { width: 400, height: 800 } }, + }); + await waitFor(() => expect(models.getAllByText(model).length).toBeGreaterThan(0)); expect(models.getAllByText(otherMaker).length).toBeGreaterThan(0); expect(models.queryByText(chip)).toBeNull(); diff --git a/__tests__/integration/models/modelCardReplacedAndResearchLicence.rendered.test.tsx b/__tests__/integration/models/modelCardReplacedAndResearchLicence.rendered.test.tsx index a4664319c..2203c8303 100644 --- a/__tests__/integration/models/modelCardReplacedAndResearchLicence.rendered.test.tsx +++ b/__tests__/integration/models/modelCardReplacedAndResearchLicence.rendered.test.tsx @@ -135,7 +135,8 @@ describe('Model cards say when a newer model replaces them and when the licence // BEFORE: the bundled list is there, the two catalog models are not, and no card says // "Research licence" (no bundled chat model has a research licence). const list = () => within(models.getByTestId('models-list')); - expect(list().getByText('MiniCPM5 2B')).toBeTruthy(); + // A bundled model shown as a card (MiniCPM5 2B is now one of this phone's picks, afdb05f). + expect(list().getByText('Qwen 3.5 2B')).toBeTruthy(); expect(models.queryByText('Labonly 3B')).toBeNull(); expect(models.queryByText('Openuse 3B')).toBeNull(); expect(screenText(models.toJSON())).not.toMatch(/Research licence/); @@ -146,7 +147,7 @@ describe('Model cards say when a newer model replaces them and when the licence // The research model says so on its card; the normal one and a bundled one do not. expect(cardText(models, 'Labonly 3B')).toMatch(/Research licence/); - for (const name of ['Openuse 3B', 'MiniCPM5 2B']) { + for (const name of ['Openuse 3B', 'Qwen 3.5 2B']) { const text = cardText(models, name); expect(text).not.toMatch(/Research licence/); expect(text).not.toMatch(/Newer:/); diff --git a/__tests__/integration/models/preferredQuantOrder.rendered.test.tsx b/__tests__/integration/models/preferredQuantOrder.rendered.test.tsx index 47f5a132e..ae5d213f3 100644 --- a/__tests__/integration/models/preferredQuantOrder.rendered.test.tsx +++ b/__tests__/integration/models/preferredQuantOrder.rendered.test.tsx @@ -60,7 +60,7 @@ async function openDetailFileOrder(opts: { platform: 'android' | 'ios'; files?: const { BackendSelector } = require('../../../src/components/settings/textGenAdvancedSections'); const settings = render(React.createElement(BackendSelector, {})); fireEvent.press(settings.getByTestId('backend-opencl-button')); - await waitFor(() => expect(settings.queryByText('GPU Layers (OpenCL)')).not.toBeNull()); + await waitFor(() => expect(settings.queryByText('Text GPU Layers (OpenCL)')).not.toBeNull()); settings.unmount(); } diff --git a/__tests__/integration/models/recommendedDetailsFailKeepCatalogFacts.rendered.test.tsx b/__tests__/integration/models/recommendedDetailsFailKeepCatalogFacts.rendered.test.tsx index cb7567f55..de8c2c42e 100644 --- a/__tests__/integration/models/recommendedDetailsFailKeepCatalogFacts.rendered.test.tsx +++ b/__tests__/integration/models/recommendedDetailsFailKeepCatalogFacts.rendered.test.tsx @@ -46,24 +46,25 @@ describe('recommended model details fail to load', () => { const { render, waitFor, within, act, fireEvent } = requireRTL(); await launchReadsDevice(); const { CATALOG } = require('@offgrid/models'); - const minicpm: CatalogEntry = CATALOG.find((e: CatalogEntry) => e.name === 'MiniCPM5 2B'); + // A model outside this phone's picks (MiniCPM5 2B is a pick since afdb05f). + const listed: CatalogEntry = CATALOG.find((e: CatalogEntry) => e.name === 'Qwen 3.5 2B'); const { ModelsScreen } = require('../../../src/screens/ModelsScreen'); const models = render(React.createElement(ModelsScreen)); // The app asked Hugging Face for the recommended models' details, and every request failed. - await waitFor(() => expect(network.detailRequests).toContain(minicpm.id)); + await waitFor(() => expect(network.detailRequests).toContain(listed.id)); await act(async () => { await new Promise(resolve => setTimeout(resolve, 50)); }); // The card for a model outside the picks shows the catalog's name and description. const card = await waitFor(() => { const cards = models.queryAllByTestId(/^model-card-\d+$/); - const found = cards.find(c => within(c).queryByText(minicpm.name)); + const found = cards.find(c => within(c).queryByText(listed.name)); expect(found).toBeTruthy(); return found!; }); - expect(within(card).getByText(new RegExp(minicpm.description.slice(0, 20).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')))).toBeTruthy(); - expect(within(card).getByText(new RegExp(`${String(minicpm.params).replace('.', '\\.')}B params`))).toBeTruthy(); + expect(within(card).getByText(new RegExp(listed.description.slice(0, 20).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')))).toBeTruthy(); + expect(within(card).getByText(new RegExp(`${String(listed.params).replace('.', '\\.')}B params`))).toBeTruthy(); // The picks above the list are there too. expect(models.getByTestId('device-picks')).toBeTruthy(); // Nothing on screen says the details failed. @@ -71,7 +72,7 @@ describe('recommended model details fail to load', () => { // The card still opens the model's detail. expect(models.queryByText('Sizes')).toBeNull(); - fireEvent.press(within(card).getByText(minicpm.name)); + fireEvent.press(within(card).getByText(listed.name)); await waitFor(() => expect(models.getByText('Sizes')).toBeTruthy()); }); }); diff --git a/__tests__/integration/models/scanFindsLockedGatewayOnLan.rendered.happy.test.tsx b/__tests__/integration/models/scanFindsLockedGatewayOnLan.rendered.happy.test.tsx new file mode 100644 index 000000000..83e1281b3 --- /dev/null +++ b/__tests__/integration/models/scanFindsLockedGatewayOnLan.rendered.happy.test.tsx @@ -0,0 +1,54 @@ +/** + * Scanning a network whose Off Grid AI Desktop asks for an API key. + * + * SPEC (OGAM user's view): the computer is on my Wi-Fi but its gateway needs a key. The scan must not + * report "No new servers"; it adds the computer to my list and takes me to where I enter its key. + * + * Device ground truth (2026-10-27): the Mac answered 401 to the phone; the saved key was rejected. + * + * Fakes sit at the device boundary only: this phone's address and `fetch` (the network). + */ +import React from 'react'; +import { render, fireEvent, waitFor } from '@testing-library/react-native'; + +const mockNavigate = jest.fn(); +jest.mock('@react-navigation/native', () => ({ + useNavigation: () => ({ navigate: mockNavigate, goBack: () => {}, setOptions: () => {}, addListener: () => () => {} }), + useIsFocused: () => true, + useFocusEffect: () => {}, +})); + +import { RemoteServersScreen } from '../../../src/screens/RemoteServersScreen'; +import { useRemoteServerStore } from '../../../src/stores'; +import { installLanProbe, gatewayModelList, type LanProbeHandle } from '../../harness/lanProbe'; + +const PHONE_IP = '192.168.1.10'; +const MAC_IP = '192.168.1.30'; + +describe('scanning a network whose Desktop needs an API key', () => { + let lan: LanProbeHandle; + + beforeEach(() => { + mockNavigate.mockClear(); + useRemoteServerStore.setState({ servers: [], serverHealth: {}, discoveredModels: {} }); + const DeviceInfo = require('react-native-device-info'); + DeviceInfo.isEmulator = jest.fn(async () => false); + DeviceInfo.getIpAddress = jest.fn(async () => PHONE_IP); + lan = installLanProbe({ [`${MAC_IP}:7878`]: { paths: ['/v1/'], body: gatewayModelList, locked: true } }); + }); + + afterEach(() => lan.uninstall()); + + it('adds the computer and opens the key entry for it, instead of reporting an empty network', async () => { + const ui = render(); + expect(ui.queryByText('No servers yet')).not.toBeNull(); + + fireEvent.press(ui.getByText('Scan network')); + + await waitFor(() => { expect(ui.queryByText(`http://${MAC_IP}:7878`)).not.toBeNull(); }, { timeout: 8000 }); + await waitFor(() => { + expect(mockNavigate).toHaveBeenCalledWith('RemoteServerEditor', expect.objectContaining({ needsKey: true })); + }); + expect(ui.queryByText(/No new servers answered/)).toBeNull(); + }, 20000); +}); diff --git a/__tests__/integration/models/sdCppCpuOnlyNotPicked.rendered.test.tsx b/__tests__/integration/models/sdCppCpuOnlyNotPicked.rendered.test.tsx new file mode 100644 index 000000000..1940898dd --- /dev/null +++ b/__tests__/integration/models/sdCppCpuOnlyNotPicked.rendered.test.tsx @@ -0,0 +1,53 @@ +/** + * Where sd.cpp can only run on the CPU, its models are not picks, and their cards say how long an image takes. + * + * SPEC (OGAM user's view): on an Android phone with a Snapdragon (Adreno) GPU, sd.cpp images run on the + * CPU (native/video/common/VideoRuntime.hpp), about 26 minutes for an SDXL image on a OnePlus SM8635. The + * Image tab never makes such a model a "For this phone" pick, and each sd.cpp card says about how many + * minutes one image takes on this phone, so nobody starts a long render by surprise. A phone whose GPU + * runs sd.cpp keeps it among its picks. + * + * Real ModelsScreen, Image tab, shared catalog and memory rule. Fakes only at the device boundary: the + * RAM sensor and SoC (Snapdragon reports its hardware as qcom), the file system, the network. + */ +import { installNativeBoundary, requireRTL, GB } from '../../harness/nativeBoundary'; + +let originalFetch: typeof fetch; +beforeEach(() => { + originalFetch = globalThis.fetch; + globalThis.fetch = (async () => new Response('[]', { status: 200 })) as typeof fetch; +}); +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +async function openImageTab(deviceHardware: string) { + installNativeBoundary({ fs: true, deviceHardware, socModel: 'SM8635', ram: { platform: 'android', totalBytes: 11.2 * GB, availBytes: 5 * GB } }); + const React = require('react'); + const { render, fireEvent } = requireRTL(); + const AsyncStorage = require('@react-native-async-storage/async-storage').default ?? require('@react-native-async-storage/async-storage'); + await AsyncStorage.clear(); + await require('../../../src/services/hardware').hardwareService.getDeviceInfo(); + const { ModelsScreen } = require('../../../src/screens/ModelsScreen'); + const models = render(React.createElement(ModelsScreen)); + fireEvent.press(models.getByText('Image')); + return models; +} + +it('a Snapdragon phone lists sd.cpp models with their time per image, and never picks them', async () => { + const models = await openImageTab('qcom'); + const { waitFor } = requireRTL(); + const card = await waitFor(() => models.getByText(/^RealVisXL v5\.0 Lightning \(Light\)$/), { timeout: 5000 }); + expect(card).toBeTruthy(); + expect(models.getAllByText(/About \d+ min per image on this phone/).length).toBeGreaterThan(0); + // No sd.cpp family is a pick here. + expect(models.queryByTestId('image-picks')).toBeNull(); + expect(models.queryAllByText(/^FLUX\.2 klein 4B$/)).toHaveLength(0); +}); + +it('a phone whose GPU runs sd.cpp keeps sd.cpp picks and shows no CPU time', async () => { + const models = await openImageTab('mt6989'); + const { waitFor } = requireRTL(); + await waitFor(() => expect(models.getAllByText(/^FLUX\.2 klein 4B/).length).toBeGreaterThan(0), { timeout: 5000 }); + expect(models.queryAllByText(/min per image on this phone/)).toHaveLength(0); +}); diff --git a/__tests__/integration/models/sdSearchRootWeightsOnly.integration.test.ts b/__tests__/integration/models/sdSearchRootWeightsOnly.integration.test.ts new file mode 100644 index 000000000..f931ea810 --- /dev/null +++ b/__tests__/integration/models/sdSearchRootWeightsOnly.integration.test.ts @@ -0,0 +1,70 @@ +/** + * Searching Hugging Face for an sd.cpp checkpoint offers only weights the phone can load. + * + * SPEC (OGAM user's view): a result can be downloaded and then used. A pack is read from the top of + * its folder, so a checkpoint inside a sub-folder of a repository is not offered (it would download + * in full and then not be found). A root-level checkpoint is offered with its plain file URL. + * + * The real search (huggingFaceModelBrowser.searchSDImageModels) runs; Hugging Face is faked at + * `fetch` with its own API shapes, and each weight answers the 0-N byte Range probe with a real + * safetensors header that names the SD model, VAE and text-encoder tensors. + */ +import { installNativeBoundary, GB } from '../../harness/nativeBoundary'; + +const REPO = 'acme/photo-xl'; +const SHA = 'a'.repeat(40); + +/** A safetensors header: 8-byte little-endian length, then JSON naming the tensors. */ +function checkpointHeader(): ArrayBuffer { + const json = JSON.stringify({ + 'model.diffusion_model.input_blocks.0.0.weight': { dtype: 'F16', shape: [1], data_offsets: [0, 2] }, + 'first_stage_model.decoder.conv_in.weight': { dtype: 'F16', shape: [1], data_offsets: [2, 4] }, + 'conditioner.embedders.0.transformer.text_model.embeddings.position_ids': { dtype: 'I64', shape: [1], data_offsets: [4, 12] }, + }); + const body = new TextEncoder().encode(json); + const bytes = new Uint8Array(8 + body.length); + new DataView(bytes.buffer).setUint32(0, body.length, true); + bytes.set(body, 8); + return bytes.buffer; +} + +it('offers a root-level checkpoint with its plain URL, and not one in a sub-folder', async () => { + installNativeBoundary({ fs: true, ram: { platform: 'android', totalBytes: 12 * GB, availBytes: 8 * GB } }); + const probed: string[] = []; + const original = globalThis.fetch; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); + const json = (body: unknown) => new Response(JSON.stringify(body), { status: 200, headers: { 'Content-Type': 'application/json' } }); + if (url.pathname === '/api/models') return json([{ id: REPO }]); + if (url.pathname === `/api/models/${REPO}`) { + return json({ + sha: SHA, + siblings: [ + { rfilename: 'photo-xl.safetensors', size: 6.9 * GB }, + { rfilename: 'unet/photo-xl.safetensors', size: 6.9 * GB }, + ], + }); + } + if (url.pathname.startsWith(`/${REPO}/resolve/${SHA}/`)) { + probed.push(url.pathname); + const header = checkpointHeader(); + const range = (init?.headers as Record | undefined)?.Range; + if (!range) return new Response(header, { status: 200 }); + return new Response(header, { + status: 206, + headers: { 'content-range': `bytes 0-${header.byteLength - 1}/${6.9 * GB}`, 'content-length': String(header.byteLength) }, + }); + } + return new Response('not found', { status: 404 }); + }) as typeof fetch; + try { + const { searchSDImageModels } = require('../../../src/services/huggingFaceModelBrowser'); + const results = await searchSDImageModels('photo xl'); + expect(results.map((model: { fileName: string }) => model.fileName)).toEqual(['photo-xl.safetensors']); + expect(results[0].downloadUrl).toBe(`https://huggingface.co/${REPO}/resolve/${SHA}/photo-xl.safetensors`); + // The sub-folder weight is not even probed. + expect(probed).toEqual([`/${REPO}/resolve/${SHA}/photo-xl.safetensors`]); + } finally { + globalThis.fetch = original; + } +}); diff --git a/__tests__/integration/models/searchHidesUnrunnableModels.rendered.test.tsx b/__tests__/integration/models/searchHidesUnrunnableModels.rendered.test.tsx index da60f2c53..e8889dbfa 100644 --- a/__tests__/integration/models/searchHidesUnrunnableModels.rendered.test.tsx +++ b/__tests__/integration/models/searchHidesUnrunnableModels.rendered.test.tsx @@ -86,7 +86,8 @@ describe('Models search leaves out what this phone cannot run', () => { const SMALL = 'someone/Demo-Chat-1B-GGUF'; it.each([ - { phone: '8 GB', totalBytes: 8 * GB, bigShown: false }, + // Android loads up to 62% of RAM (ddaa6c1): an 8B's estimate fits 8 GB now, not 6 GB. + { phone: '6 GB', totalBytes: 6 * GB, bigShown: false }, { phone: '16 GB', totalBytes: 16 * GB, bigShown: true }, ])('a $phone phone: an 8B search result with no file sizes is offered only when it fits', async ({ totalBytes, bigShown }) => { installNativeBoundary({ fs: true, ram: { platform: 'android', totalBytes, availBytes: totalBytes / 2 } }); diff --git a/__tests__/integration/onboarding/autoSetupImageChoice.rendered.test.tsx b/__tests__/integration/onboarding/autoSetupImageChoice.rendered.test.tsx index f50f13961..969d46ebe 100644 --- a/__tests__/integration/onboarding/autoSetupImageChoice.rendered.test.tsx +++ b/__tests__/integration/onboarding/autoSetupImageChoice.rendered.test.tsx @@ -61,6 +61,12 @@ const catalog: AutoSetupCatalogBoundaries = { ], }; +/** Open a plan's details; the device's own plan starts open, and its heading toggles. */ +function openPlan(ui: ReturnType, tier: string): void { + const heading = ui.getByTestId(`auto-setup-plan-heading-${tier}`); + if (!heading.props.accessibilityState?.expanded) fireEvent.press(heading); +} + test('Auto Setup shows Absolute Reality when the device catalog would choose NAI Anime', async () => { const navigation = { push() {}, replace() {} } as never; const ui = render( @@ -74,7 +80,7 @@ test('Auto Setup shows Absolute Reality when the device catalog would choose NAI expect(ui.getByTestId('auto-setup-plan-lean')).toBeTruthy(), ); for (const tier of ['lean', 'balanced', 'extreme']) { - fireEvent.press(ui.getByTestId(`auto-setup-plan-${tier}`)); + openPlan(ui, tier); expect(ui.getByText('Absolute Reality (NPU 8gen2)')).toBeTruthy(); expect(ui.queryByText('Absolute Reality (GPU)')).toBeNull(); expect(ui.queryByText('NAI Anime (NPU 8gen2)')).toBeNull(); @@ -104,7 +110,7 @@ test('Auto Setup offers the GPU model when the device recommends GPU', async () await waitFor(() => expect(ui.getByTestId('auto-setup-plan-balanced')).toBeTruthy(), ); - fireEvent.press(ui.getByTestId('auto-setup-plan-balanced')); + openPlan(ui, 'balanced'); expect(ui.getByText('Absolute Reality (GPU)')).toBeTruthy(); expect(ui.queryByText('Absolute Reality (NPU 8gen2)')).toBeNull(); }); @@ -143,7 +149,7 @@ test('Android Auto Setup presents complementary image models across its three pl ['balanced', 'Absolute Reality (NPU 8gen2)'], ['extreme', 'Dream Shaper V8 (NPU 8gen2)'], ]) { - fireEvent.press(ui.getByTestId(`auto-setup-plan-${tier}`)); + openPlan(ui, tier); expect(ui.getByText(name)).toBeTruthy(); } }); diff --git a/__tests__/integration/onboarding/autoSetupJourney.test.tsx b/__tests__/integration/onboarding/autoSetupJourney.test.tsx index 574c84772..82caa6526 100644 --- a/__tests__/integration/onboarding/autoSetupJourney.test.tsx +++ b/__tests__/integration/onboarding/autoSetupJourney.test.tsx @@ -66,6 +66,10 @@ class NativeDownloadBoundary implements DownloadProvider { if (request.modelType === 'image') { return this.begin(request.model.id, request.model.name, request.model.size); } + if (request.modelType === 'video') { + const size = request.model.files.reduce((sum, file) => sum + (file.sizeBytes ?? 0), 0); + return this.begin(request.model.id, request.model.name, size); + } return this.begin(request.modelId, request.modelId, 1 * MB); } @@ -216,17 +220,23 @@ describe('Auto Setup release journey', () => { expect(ui.getByTestId('auto-setup-plan-balanced')).toBeTruthy(), ); - expect(ui.getAllByText('INCLUDES')).toHaveLength(1); - // Balanced aims at 4B: the catalog now offers Qwen 3.5 4B, which leads Gemma 4 E4B. - expect(ui.getByText('Qwen 3.5 4B')).toBeTruthy(); - fireEvent.press(ui.getByTestId('auto-setup-plan-lean')); - expect(ui.getByText('Qwen 3.5 2B')).toBeTruthy(); + // Only the chosen plan is expanded. + expect(ui.getAllByTestId('auto-setup-selected-plan')).toHaveLength(1); + // Lean, Balanced and Extreme are this 12 GB phone's Fast, Balanced and Best picks (37759c684), + // ranked by quality alone (shared afdb05f). + expect(ui.getByText('MiniCPM5 2B')).toBeTruthy(); + fireEvent.press(ui.getByTestId('auto-setup-plan-heading-lean')); + expect(ui.getByText('LFM2.5 8B A1B')).toBeTruthy(); expect(ui.queryByText('SmolVLM 2B')).toBeNull(); - fireEvent.press(ui.getByTestId('auto-setup-plan-extreme')); + fireEvent.press(ui.getByTestId('auto-setup-plan-heading-extreme')); expect(ui.getByText('Qwen 3.5 9B')).toBeTruthy(); - expect(ui.queryByText('Qwen 3.5 4B')).toBeNull(); - expect(useAppStore.getState().settings.modelLoadingMode).toBe('aggressive'); + expect(ui.queryByText('MiniCPM5 2B')).toBeNull(); + // A tier only previews downloads; it never changes the live loading policy. + expect(useAppStore.getState().settings.modelLoadingMode).toBe('balanced'); + // Text is preselected; image and speech-to-text are opt-in. + fireEvent.press(ui.getByTestId('auto-setup-choice-image')); + fireEvent.press(ui.getByTestId('auto-setup-choice-stt')); fireEvent.press(ui.getByTestId('auto-setup-download')); await waitFor(() => expect(ui.getByTestId('auto-setup-continue')).toBeTruthy(), @@ -284,6 +294,8 @@ describe('Auto Setup release journey', () => { expect(ui.getByTestId('auto-setup-download')).toBeTruthy(), ); + fireEvent.press(ui.getByTestId('auto-setup-choice-image')); + fireEvent.press(ui.getByTestId('auto-setup-choice-stt')); fireEvent.press(ui.getByTestId('auto-setup-download')); await waitFor(() => @@ -313,6 +325,8 @@ describe('Auto Setup release journey', () => { await waitFor(() => expect(ui.getByTestId('auto-setup-download')).toBeTruthy(), ); + fireEvent.press(ui.getByTestId('auto-setup-choice-image')); + fireEvent.press(ui.getByTestId('auto-setup-choice-stt')); fireEvent.press(ui.getByTestId('auto-setup-download')); await waitFor(() => expect(ui.getAllByText(/STARTING|0%/).length).toBeGreaterThan(0), @@ -363,9 +377,9 @@ describe('Auto Setup release journey', () => { await waitFor(() => expect(ui.getAllByText(/STARTING|0%/).length).toBeGreaterThan(0), ); - fireEvent.press(ui.getByTestId('auto-setup-plan-extreme')); + fireEvent.press(ui.getByTestId('auto-setup-plan-heading-extreme')); - expect(ui.getByText('Qwen 3.5 4B')).toBeTruthy(); + expect(ui.getByText('MiniCPM5 2B')).toBeTruthy(); expect(ui.queryByText('Qwen 3.5 9B')).toBeNull(); expect(useAppStore.getState().settings.modelLoadingMode).toBe('balanced'); ui.unmount(); @@ -374,7 +388,8 @@ describe('Auto Setup release journey', () => { it('ends a stalled catalog request at its deadline', async () => { const stalledCatalog: AutoSetupCatalogBoundaries = { ...catalogBoundaries, - fetchTextFiles: () => new Promise(() => undefined), + // Chat plans come from the bundled picks (37759c684); the image catalog is still a live load. + imageModels: () => new Promise(() => undefined), }; const ui = render( ({ hide: jest.fn(async () => {}) }), { virtual: true }); + +const RENDER_WAIT = { timeout: 8000 }; +const PLAN_TO_PICK = { lean: 'fast', balanced: 'balanced', extreme: 'best' } as const; + +/** Hugging Face as the phone sees it: every Core ML image repo has one compiled zip; anything else + * is empty. */ +function installHuggingFace(): () => void { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = new URL(String(input)); + const json = (body: unknown, status = 200) => + ({ ok: status < 400, status, json: async () => body }) as Response; + if (url.hostname !== 'huggingface.co') return json({}, 404); + if (/^\/api\/models\/.+\/tree\/main$/.test(url.pathname)) { + const size = 1.2 * GB; + return json([{ type: 'file', path: 'coreml-split_einsum-compiled.zip', size, lfs: { oid: 'x', size, pointerSize: 1 } }]); + } + return json([]); + }) as typeof fetch; + return () => { + globalThis.fetch = originalFetch; + }; +} + +/** The strings a rendered subtree shows, in order. */ +function texts(scope: { getAllByText: (m: RegExp) => Array<{ props: { children?: unknown } }> }): string[] { + return scope + .getAllByText(/.+/) + .map(text => text.props.children) + .filter((child): child is string => typeof child === 'string'); +} + +describe('Auto Setup plans are the phone picks', () => { + it.each([ + ['android', 12], + ['ios', 8], + ] as const)('on a %s phone with %i GB, Lean, Balanced and Extreme name Fast, Balanced and Best', async (platform, totalGB) => { + installNativeBoundary({ fs: true, ram: { platform, totalBytes: totalGB * GB, availBytes: totalGB * 0.5 * GB } }); + const restoreFetch = installHuggingFace(); + const React = require('react'); + const { render, fireEvent, waitFor, within } = requireRTL(); + const AsyncStorage = + require('@react-native-async-storage/async-storage').default ?? + require('@react-native-async-storage/async-storage'); + try { + await AsyncStorage.clear(); + await AsyncStorage.setItem( + 'local-llm-app-storage', + JSON.stringify({ state: { hasCompletedOnboarding: true }, version: 0 }), + ); + + // Auto Setup, as the person meets it at launch. + const App = require('../../../App').default; + const app = render(React.createElement(App)); + expect(app.queryByTestId('auto-setup-plan-lean')).toBeNull(); + await waitFor(() => expect(app.getByTestId('auto-setup-plan-lean')).toBeTruthy(), RENDER_WAIT); + const planModels: Partial> = {}; + for (const tier of Object.keys(PLAN_TO_PICK) as Array) { + const card = app.queryByTestId(`auto-setup-plan-${tier}`); + if (!card) continue; + if (!within(card).queryByTestId('auto-setup-selected-plan')) { + fireEvent.press(within(card).getByTestId(`auto-setup-plan-heading-${tier}`)); + } + const expanded = within(app.getByTestId(`auto-setup-plan-${tier}`)).getByTestId('auto-setup-selected-plan'); + const labels = texts(within(expanded)); + const at = labels.indexOf('TEXT'); + expect(at).toBeGreaterThan(0); + planModels[tier] = labels[at - 1]; + } + app.unmount(); + + // The Models screen's picks for the same phone. + const { ModelsScreen } = require('../../../src/screens/ModelsScreen'); + const { NavigationContainer } = require('@react-navigation/native'); + const { createNativeStackNavigator } = require('@react-navigation/native-stack'); + const Stack = createNativeStackNavigator(); + const models = render( + React.createElement( + NavigationContainer, + null, + React.createElement( + Stack.Navigator, + { screenOptions: { headerShown: false, animation: 'none' } }, + React.createElement(Stack.Screen, { name: 'Models', component: ModelsScreen }), + ), + ), + ); + await waitFor(() => expect(models.getByTestId('device-picks')).toBeTruthy(), RENDER_WAIT); + const pickModels: Record = {}; + for (const slot of Object.values(PLAN_TO_PICK)) { + const card = models.queryByTestId(`pick-${slot}`); + // A pick card shows its slot label, then the model name. + pickModels[slot] = card ? texts(within(card))[1] : undefined; + } + models.unmount(); + + // Every plan shown names its pick, and every pick has its plan. + const shownPlans = Object.keys(planModels) as Array; + expect(shownPlans.length).toBeGreaterThan(0); + for (const tier of shownPlans) expect(planModels[tier]).toBe(pickModels[PLAN_TO_PICK[tier]]); + for (const [tier, slot] of Object.entries(PLAN_TO_PICK)) { + if (pickModels[slot]) expect(planModels[tier as keyof typeof PLAN_TO_PICK]).toBe(pickModels[slot]); + } + expect(new Set(Object.values(planModels)).size).toBe(shownPlans.length); + } finally { + restoreFetch(); + } + }, 30000); +}); diff --git a/__tests__/integration/onboarding/autoSetupPlansDistinctChatModels.rendered.test.tsx b/__tests__/integration/onboarding/autoSetupPlansDistinctChatModels.rendered.test.tsx index fa933770a..7fe1650ce 100644 --- a/__tests__/integration/onboarding/autoSetupPlansDistinctChatModels.rendered.test.tsx +++ b/__tests__/integration/onboarding/autoSetupPlansDistinctChatModels.rendered.test.tsx @@ -106,18 +106,21 @@ async function plansSeenOnPhone(totalGB: number) { for (const tier of TIERS) { const card = ui.queryByTestId(`auto-setup-plan-${tier}`); if (!card) continue; - fireEvent.press(card); + // One plan is open at a time; its heading opens it. + if (!within(card).queryByTestId('auto-setup-selected-plan')) { + fireEvent.press(within(card).getByTestId(`auto-setup-plan-heading-${tier}`)); + } const expanded = within(ui.getByTestId(`auto-setup-plan-${tier}`)).getByTestId( 'auto-setup-selected-plan', ); - // The chat model is the name printed under the "TEXT + VISION" label. + // The chat model is the name printed before its "TEXT" label. const labels = within(expanded).getAllByText(/.+/).map(node => { const children = (node.props as { children?: unknown }).children; return typeof children === 'string' ? children : null; }); - const at = labels.indexOf('TEXT + VISION'); - expect(at).toBeGreaterThanOrEqual(0); - seen.push({ tier, chatModel: labels[at + 1] as string }); + const at = labels.indexOf('TEXT'); + expect(at).toBeGreaterThan(0); + seen.push({ tier, chatModel: labels[at - 1] as string }); } return seen; } finally { @@ -140,16 +143,16 @@ describe('Auto Setup plans by phone memory', () => { it('a 12 GB iPhone sees three plans, Extreme with the 9B, never the same chat model twice', async () => { const plans = await plansSeenOnPhone(12); expect(plans.map(plan => plan.tier)).toEqual(['lean', 'balanced', 'extreme']); - expect(chatModels(plans)).toEqual(['Qwen 3.5 2B', 'Qwen 3.5 4B', 'Qwen 3.5 9B']); + // The plans are the phone's Fast, Balanced and Best picks (37759c684), by quality alone (afdb05f). + expect(chatModels(plans)).toEqual(['LFM2.5 8B A1B', 'MiniCPM5 2B', 'Qwen 3.5 9B']); }); - it('a 6 GB phone sees fewer plans than a 16 GB phone, and no chat model twice', async () => { - const big = await plansSeenOnPhone(16); + // Setup's plans are the phone's picks (37759c684): a 6 GB phone has three distinct models that run + // well, so it sees all three plans, each with its own chat model. + it('a 6 GB phone sees its own three picks, and no chat model twice', async () => { const small = await plansSeenOnPhone(6); - expect(small.length).toBeGreaterThanOrEqual(1); - expect(small.length).toBeLessThan(big.length); + expect(small.map(plan => plan.tier)).toEqual(['lean', 'balanced', 'extreme']); + expect(chatModels(small)).toEqual(['Qwen 3.5 2B', 'Qwen 3.5 4B', 'MiniCPM5 2B']); expect(new Set(chatModels(small)).size).toBe(small.length); - // The chat models the small phone sees are the smaller ones the big phone also offers. - expect(chatModels(small).every(name => chatModels(big).includes(name))).toBe(true); }); }); diff --git a/__tests__/integration/onboarding/autoSetupVideoFindComputer.rendered.test.tsx b/__tests__/integration/onboarding/autoSetupVideoFindComputer.rendered.test.tsx new file mode 100644 index 000000000..15e52d7d7 --- /dev/null +++ b/__tests__/integration/onboarding/autoSetupVideoFindComputer.rendered.test.tsx @@ -0,0 +1,119 @@ +/** + * Auto Setup on a phone too small for any video model offers "Find my computer"; once an Off Grid AI + * Desktop is paired, it names that computer as where videos are made (commit 035c87386). + * + * Product-correct outcome (the person's view): with Pro, on an 8 GB phone no video pack fits, so + * the plan's VIDEO row says videos are made on a computer and offers Find my computer. Tapping it + * opens Remote servers. The person adds their Desktop there and comes back: the row now reads + * "Videos are made on Studio Mac." and no longer offers Find my computer. + * + * Entry point: the REAL App with Pro installed the way the bootstrap does it (harness/proHarness), + * onboarding done and no model on the phone, so Auto Setup opens first. Every step after launch is + * a gesture: Find my computer, Add server, type name and address, Test connection, Save, Back. + * + * Fakes (device boundary only): the RAM sensor, Hugging Face (fetch, for the image candidates) and + * the Desktop's gateway over fetch, in Desktop's own wire shapes (desktop/src/main/model-server.ts: + * /v1/models/catalog, /installed, /active). Everything we own runs for real. + */ +import { installNativeBoundary, requireRTL, GB } from '../../harness/nativeBoundary'; + +jest.unmock('@react-navigation/native'); +jest.mock('react-native-bootsplash', () => ({ hide: jest.fn(async () => {}) }), { virtual: true }); + +const RENDER_WAIT = { timeout: 8000 }; +const DESKTOP_NAME = 'Studio Mac'; +const DESKTOP_ENDPOINT = 'http://192.168.7.21:7878'; // NOSONAR - private LAN test fixture + +/** The network: Hugging Face's image repos, and the person's Desktop with one video model. */ +function installNetwork(): () => void { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(String(input)); + const json = (body: unknown, status = 200) => + ({ ok: status < 400, status, json: async () => body }) as Response; + if (url.hostname === 'huggingface.co') { + if (/^\/api\/models\/.+\/tree\/main$/.test(url.pathname)) { + const size = 1.2 * GB; + return json([{ type: 'file', path: 'coreml-split_einsum-compiled.zip', size, lfs: { oid: 'x', size, pointerSize: 1 } }]); + } + return json([]); + } + if (url.origin !== DESKTOP_ENDPOINT) throw new TypeError('Network request failed'); + const method = (init?.method ?? 'GET').toUpperCase(); + switch (`${method} ${url.pathname}`) { + case 'GET /v1/models': + return json({ object: 'list', data: [] }); + case 'GET /v1/models/catalog': + return json({ + kinds: ['text', 'vision', 'image', 'video', 'speech', 'transcription'], + memoryGb: 64, + models: [{ id: 'off-grid-ai/Wan2.1-T2V-1.3B-GGUF', name: 'Wan 2.1 (1.3B)', kind: 'video', org: 'Off Grid AI', tags: [], files: [] }], + }); + case 'GET /v1/models/installed': + return json({ installed: ['off-grid-ai/Wan2.1-T2V-1.3B-GGUF'] }); + case 'GET /v1/models/active': + return json({ text: null, image: null, video: 'off-grid-ai/Wan2.1-T2V-1.3B-GGUF', speech: null, transcription: null }); + default: + return json({ error: 'not found' }, 404); + } + }) as typeof fetch; + return () => { + globalThis.fetch = originalFetch; + }; +} + +describe('Auto Setup video on a small phone', () => { + it('offers Find my computer, and names the Desktop once it is added', async () => { + installNativeBoundary({ fs: true, ram: { platform: 'android', totalBytes: 8 * GB, availBytes: 4 * GB } }); + const restoreFetch = installNetwork(); + const React = require('react'); + const { render, fireEvent, waitFor, within } = requireRTL(); + const AsyncStorage = + require('@react-native-async-storage/async-storage').default ?? + require('@react-native-async-storage/async-storage'); + const ui = { current: null as null | ReturnType }; + try { + await AsyncStorage.clear(); + await AsyncStorage.setItem( + 'local-llm-app-storage', + JSON.stringify({ state: { hasCompletedOnboarding: true }, version: 0 }), + ); + await require('../../harness/proHarness').installPro(); + const App = require('../../../App').default; + const app = render(React.createElement(App)); + ui.current = app; + + // BEFORE: the open plan's VIDEO row says videos are made on a computer, and offers to find it. + const videoRow = () => within(app.getByTestId('auto-setup-selected-plan')).getByTestId('auto-setup-video-row'); + await waitFor(() => expect(videoRow()).toBeTruthy(), RENDER_WAIT); + expect(within(videoRow()).getByText('Videos are made on your computer with Off Grid AI Desktop.')).toBeTruthy(); + expect(within(videoRow()).queryByText(`Videos are made on ${DESKTOP_NAME}.`)).toBeNull(); + + // Find my computer opens Remote servers. + fireEvent.press(within(videoRow()).getByTestId('auto-setup-find-computer')); + fireEvent.press(await waitFor(() => app.getByTestId('add-server'), RENDER_WAIT)); + + // The person adds their Desktop. + fireEvent.changeText(await waitFor(() => app.getByTestId('server-name'), RENDER_WAIT), DESKTOP_NAME); + fireEvent.changeText(app.getByTestId('server-endpoint'), DESKTOP_ENDPOINT); + fireEvent.press(app.getByTestId('test-connection')); + await waitFor(() => expect(app.getByText(/Connected \(/)).toBeTruthy(), RENDER_WAIT); + fireEvent.press(app.getByTestId('save-server')); + await waitFor(() => expect(app.queryByTestId('server-name')).toBeNull(), RENDER_WAIT); + await waitFor(() => expect(app.getByText(DESKTOP_NAME)).toBeTruthy(), RENDER_WAIT); + + // Back on Auto Setup: the row names the Desktop and no longer offers to find a computer. + fireEvent.press(app.getByLabelText('Back')); + await waitFor( + () => expect(within(videoRow()).getByText(`Videos are made on ${DESKTOP_NAME}.`)).toBeTruthy(), + RENDER_WAIT, + ); + expect(within(videoRow()).queryByTestId('auto-setup-find-computer')).toBeNull(); + } finally { + ui.current?.unmount(); + restoreFetch(); + const { remoteServerManager } = require('../../../src/services/remoteServerManager'); + await remoteServerManager.clearAllServers(); + } + }, 45000); +}); diff --git a/__tests__/integration/pro/proExpiryRedirect.integration.test.tsx b/__tests__/integration/pro/proExpiryRedirect.integration.test.tsx index 342f26922..97a6f3d54 100644 --- a/__tests__/integration/pro/proExpiryRedirect.integration.test.tsx +++ b/__tests__/integration/pro/proExpiryRedirect.integration.test.tsx @@ -60,7 +60,7 @@ describe('the live Pro expiry redirect', () => { expect(resetRoot).toHaveBeenCalledTimes(1); expect(resetRoot).toHaveBeenCalledWith({ index: 1, - routes: [{ name: 'Main' }, { name: 'ProDetail' }], + routes: [{ name: 'Main', params: { screen: 'HomeTab' } }, { name: 'ProDetail' }], }); }); }); diff --git a/__tests__/integration/pro/proPricingSettingsEntry.rendered.test.tsx b/__tests__/integration/pro/proPricingSettingsEntry.rendered.test.tsx index f5a3c7874..b194d0f51 100644 --- a/__tests__/integration/pro/proPricingSettingsEntry.rendered.test.tsx +++ b/__tests__/integration/pro/proPricingSettingsEntry.rendered.test.tsx @@ -90,9 +90,9 @@ describe('Pro pricing reached from Settings (rendered, real App)', () => { expect(view.queryByText(/\$119/)).toBeNull(); expect(view.queryByText('Intelligence, democratized.')).toBeNull(); - // The Settings upsell banner (its LIVE SYNC feature grid) carries its own "Get Pro" button. + // The Settings upsell banner carries its own "Get Pro" button. // Home stays mounted behind the tabs with a "Get Pro" of its own, so scope to the banner. - let banner = view.getByText('LIVE SYNC').parent; + let banner = view.getByText('Keep your work in context with memory, Sync, and actions you approve.').parent; while (banner && rtl.within(banner).queryAllByText('Get Pro').length === 0) { banner = banner.parent; } diff --git a/__tests__/integration/remote/editedServerRechecksOnSave.rendered.test.tsx b/__tests__/integration/remote/editedServerRechecksOnSave.rendered.test.tsx new file mode 100644 index 000000000..bed2884b8 --- /dev/null +++ b/__tests__/integration/remote/editedServerRechecksOnSave.rendered.test.tsx @@ -0,0 +1,129 @@ +/** + * Saving an edited remote server checks it again, so the Remote Servers list shows its new status + * (commit b47b2253c). + * + * Product-correct outcome (the person's view): a server's key changed on the server, so its row on + * Remote Servers reads "Not answering". The person opens Edit, types the new key, checks it and saves. Back on + * the list, without tapping Test, the row reads "Connected". The list screen stays mounted under the + * editor, so only the save itself can bring the new status. + * + * Real RemoteServersScreen + RemoteServerEditorScreen on a real native stack; real form, manager, + * stores and health check. Fakes only at the device boundary: the network (harness/keyedRemoteServer, + * a server that refuses requests without its current key) and the Keychain (a stateful store). + */ +import React from 'react'; +import { Text, TouchableOpacity, View } from 'react-native'; +import { fireEvent, render, waitFor, within } from '@testing-library/react-native'; +import { NavigationContainer, useNavigation } from '@react-navigation/native'; +import { createNativeStackNavigator } from '@react-navigation/native-stack'; +import { RemoteServersScreen } from '../../../src/screens/RemoteServersScreen'; +import { RemoteServerEditorScreen } from '../../../src/screens/RemoteServerEditorScreen'; +import { remoteServerManager } from '../../../src/services/remoteServerManager'; +import { + installKeyedServer, + resetKeychain, + type KeyedServerHandle, +} from '../../harness/keyedRemoteServer'; + +jest.unmock('@react-navigation/native'); +jest.mock('react-native-keychain', () => + require('../../harness/keyedRemoteServer').keychainModule(), +); + +const ORIGIN = 'https://studio.example.test'; +const OLD_KEY = 'sk-lm-studio-before'; +const NEW_KEY = 'sk-lm-studio-rotated'; +const MODEL = 'qwen3-8b'; +const SERVER_NAME = 'Studio LM Studio'; + +const Stack = createNativeStackNavigator(); + +const Launcher: React.FC = () => { + const navigation = useNavigation(); + return ( + + navigation.navigate('RemoteServers')}> + Remote Servers + + + ); +}; + +function mountApp() { + return render( + + + + + + + , + ); +} + +type Ui = ReturnType; + +function serverRow(ui: Ui) { + const { useRemoteServerStore } = require('../../../src/stores'); + const id = useRemoteServerStore.getState().servers[0].id; + return within(ui.getByTestId(`server-${id}`)); +} + +describe('saving an edited remote server', () => { + let server: KeyedServerHandle; + + beforeEach(async () => { + resetKeychain(); + await remoteServerManager.clearAllServers(); + server = installKeyedServer({ origin: ORIGIN, flavour: 'lmstudio', apiKey: OLD_KEY, modelId: MODEL }); + }); + + afterEach(async () => { + server.restore(); + await remoteServerManager.clearAllServers(); + }); + + it('a new key saved in the editor shows the server Connected on the list, without tapping Test', async () => { + const ui = mountApp(); + + // Add the server with its key through the real add-server screen. + fireEvent.press(ui.getByTestId('open-remote-servers')); + await waitFor(() => expect(ui.getByTestId('add-server')).toBeTruthy()); + fireEvent.press(ui.getByTestId('add-server')); + await waitFor(() => expect(ui.getByText('Add a server')).toBeTruthy()); + fireEvent.changeText(ui.getByTestId('server-name'), SERVER_NAME); + fireEvent.changeText(ui.getByTestId('server-endpoint'), ORIGIN); + fireEvent.changeText(ui.getByTestId('server-api-key'), OLD_KEY); + fireEvent.press(ui.getByTestId('test-connection')); + await waitFor(() => expect(ui.getByText(/Connected \(\d+ms\)/)).toBeTruthy()); + fireEvent.press(ui.getByTestId('save-server')); + fireEvent.press(await waitFor(() => ui.getByText('Continue'))); + await waitFor(() => expect(serverRow(ui).getByText('Connected')).toBeTruthy()); + + // The key is rotated on the server: the old one is refused, and Test says so on the row. + server.restore(); + server = installKeyedServer({ origin: ORIGIN, flavour: 'lmstudio', apiKey: NEW_KEY, modelId: MODEL }); + fireEvent.press(serverRow(ui).getByText('Test')); + await waitFor(() => expect(ui.getByText('Could not connect')).toBeTruthy()); + fireEvent.press(ui.getByText('OK')); + // BEFORE: the row reads Not answering. + await waitFor(() => expect(serverRow(ui).getByText('Not answering')).toBeTruthy()); + expect(serverRow(ui).queryByText('Connected')).toBeNull(); + + // The person types the new key in the editor and saves. + fireEvent.press(serverRow(ui).getByText('Edit')); + await waitFor(() => expect(ui.getByText('Edit server')).toBeTruthy()); + fireEvent.changeText(ui.getByTestId('server-api-key'), NEW_KEY); + // The editor saves only after its own check of the values typed in it. + fireEvent.press(ui.getByTestId('test-connection')); + await waitFor(() => expect(ui.getByText(/Connected \(\d+ms\)/)).toBeTruthy()); + fireEvent.press(ui.getByTestId('save-server')); + fireEvent.press(await waitFor(() => ui.getByText('Continue'))); + await waitFor(() => expect(ui.queryByText('Edit server')).toBeNull()); + + // AFTER: back on the list, the row shows the status of the server as it is now. + await waitFor(() => expect(serverRow(ui).getByText('Connected')).toBeTruthy()); + expect(serverRow(ui).queryByText('Not answering')).toBeNull(); + expect(ui.queryByText('Could not connect')).toBeNull(); + }); +}); diff --git a/__tests__/integration/remote/ollamaKeyedDiscovery.rendered.test.tsx b/__tests__/integration/remote/ollamaKeyedDiscovery.rendered.test.tsx index b7f926f0d..d6623897b 100644 --- a/__tests__/integration/remote/ollamaKeyedDiscovery.rendered.test.tsx +++ b/__tests__/integration/remote/ollamaKeyedDiscovery.rendered.test.tsx @@ -79,7 +79,7 @@ describe('Ollama over HTTPS with a saved key', () => { // Back on Home, the picker lists the Ollama's model. fireEvent.press(ui.getByLabelText('Back')); - await waitFor(() => expect(ui.getByText('Select a text or image model to start')).toBeTruthy()); + await waitFor(() => expect(ui.getByText('Select a text, image, or video model to start')).toBeTruthy()); fireEvent.press(ui.getByTestId('browse-models-button')); await waitFor(() => expect(ui.getByText(MODEL)).toBeTruthy()); }); diff --git a/__tests__/integration/remote/slowDesktopNoDeadline.rendered.test.tsx b/__tests__/integration/remote/slowDesktopNoDeadline.rendered.test.tsx new file mode 100644 index 000000000..ac2b5779b --- /dev/null +++ b/__tests__/integration/remote/slowDesktopNoDeadline.rendered.test.tsx @@ -0,0 +1,144 @@ +/** + * Calls to Off Grid AI Desktop have no deadline (commit ae9c0c950): a Desktop that takes 6 seconds + * to load a model or to accept an image still finishes the job. + * + * Product-correct outcome (the person's view): the person picks Desktop's image model in the remote + * model picker. Desktop takes 6 seconds to load it (longer than the 5 seconds the phone used to + * wait), then confirms. The pick shows as selected. They then ask the chat for an image; Desktop + * again takes 6 seconds to accept the job, runs it, and the image appears in the chat. No step is + * cut midway and no failure card shows. + * + * Real ChatScreen, image-mode toggle, RemoteModelOptionsSection picker, remoteServerManager, + * offGridDesktopModels (gatewayFetch), imageGenerationService and remoteMediaRuntime. Fakes only at + * the device boundary: Desktop's gateway over fetch (Desktop's own wire shapes, see + * harness/remoteImageServer), answering two routes late, and the native file system. + */ +import React from 'react'; +import { setupChatScreen } from '../../harness/chatHarness'; + +jest.mock('@react-navigation/native', () => ({ + useNavigation: () => ({ navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {} }), + useRoute: () => require('../../harness/chatHarness').routeHolder, + useFocusEffect: () => {}, + useIsFocused: () => true, +})); + +const ENDPOINT = 'http://192.168.1.63:7878'; // NOSONAR - private LAN test fixture +const MODEL = { id: 'flux-schnell', name: 'Flux Schnell' }; +const SLOW_MS = 6_000; +/** A real 1x1 PNG, as Desktop returns it in a completed job's `result.data[0].b64_json`. */ +const PNG_1X1 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + +/** Desktop's gateway, answering model activation and image submission after SLOW_MS. A request the + * phone aborts rejects, as a real fetch does. */ +function serveSlowDesktop(): { aborted: string[]; restore: () => void } { + const original = globalThis.fetch; + const aborted: string[] = []; + let activeImage: string | null = null; + const polls = new Map(); + const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); + const late = (route: string, signal: AbortSignal | null | undefined, answer: () => Response) => + new Promise((resolve, reject) => { + const timer = setTimeout(() => resolve(answer()), SLOW_MS); + signal?.addEventListener('abort', () => { + clearTimeout(timer); + aborted.push(route); + reject(new Error('Aborted')); + }, { once: true }); + }); + + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const href = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + const url = new URL(href); + if (url.origin !== ENDPOINT) return original(input, init); + if (init?.signal?.aborted) throw new Error('Aborted'); + const method = (init?.method ?? 'GET').toUpperCase(); + const route = `${method} ${url.pathname}`; + if (route === 'GET /v1/models') return json({ object: 'list', data: [] }); + if (route === 'GET /v1/models/catalog') { + return json({ + kinds: ['text', 'vision', 'image', 'speech', 'transcription'], + models: [{ ...MODEL, kind: 'image', org: 'Black Forest Labs', tags: [], files: [] }], + }); + } + if (route === 'GET /v1/models/installed') return json({ installed: [MODEL.id] }); + if (route === 'GET /v1/models/active') { + return json({ text: null, image: activeImage, speech: null, transcription: null }); + } + if (route === 'POST /v1/models/activate') { + const body = JSON.parse(String(init?.body ?? '{}')) as { id?: string; kind?: string }; + return late(route, init?.signal, () => { + const known = body.kind === 'image' && body.id === MODEL.id; + if (known) activeImage = MODEL.id; + return json({ success: known }); + }); + } + if (route === 'POST /v1/images/generations') { + return late(route, init?.signal, () => { + polls.set('img-1', 0); + return json({ request_id: 'img-1', object: 'request', kind: 'image', status: 'queued', poll_url: '/v1/images/img-1' }, 202); + }); + } + const poll = url.pathname.match(/^\/v1\/requests\/([^/]+)$/); + if (poll && method === 'GET' && polls.has(poll[1])) { + const count = polls.get(poll[1])! + 1; + polls.set(poll[1], count); + const base = { request_id: poll[1], object: 'request', kind: 'image', poll_url: `/v1/images/${poll[1]}` }; + return count === 1 + ? json({ ...base, status: 'running', progress: { step: 1, total: 4 } }) + : json({ ...base, status: 'completed', result: { data: [{ b64_json: PNG_1X1 }] } }); + } + return json({ error: { message: 'Unknown endpoint', type: 'not_found' } }, 404); + }) as typeof globalThis.fetch; + return { aborted, restore: () => { globalThis.fetch = original; } }; +} + +describe('a slow Off Grid AI Desktop', () => { + let desktop: ReturnType | undefined; + + afterEach(async () => { + desktop?.restore(); + desktop = undefined; + const { remoteServerManager } = require('../../../src/services/remoteServerManager'); + await remoteServerManager.clearAllServers(); + }); + + it('activates the picked model and draws the image when each answer takes 6 seconds', async () => { + const h = await setupChatScreen({ engine: 'llama' }); + desktop = serveSlowDesktop(); + const { remoteServerManager } = require('../../../src/services/remoteServerManager'); + const { RemoteModelOptionsSection } = require('../../../src/components/models/RemoteModelOptionsSection'); + // The paired Desktop's saved record, as pairing leaves it. + const server = await remoteServerManager.addServer({ + name: 'Studio Mac', + endpoint: ENDPOINT, + providerType: 'openai-compatible', + modelManagement: 'offgrid-desktop-v1', + modelCatalog: { image: [MODEL] }, + }); + const rowId = `remote-image-model-${server.id}:${MODEL.id}`; + + const picker = h.rtl.render(React.createElement(RemoteModelOptionsSection, { category: 'image' })); + // BEFORE: Desktop's image model is listed and not selected. + expect(picker.getByTestId(rowId).props.accessibilityState?.selected).not.toBe(true); + h.rtl.fireEvent.press(picker.getByTestId(rowId)); + await h.rtl.waitFor(() => { + expect(picker.getByTestId(rowId).props.accessibilityState?.selected).toBe(true); + }, { timeout: SLOW_MS + 4_000 }); + picker.unmount(); + + h.render(); + await h.cycleImageMode(); + await h.rtl.waitFor(() => { expect(h.view!.queryByTestId('image-mode-force-badge')).not.toBeNull(); }); + expect(h.view!.queryByTestId('generated-image')).toBeNull(); + + await h.tapSend('a lighthouse at dusk'); + await h.rtl.waitFor(() => { + expect(h.view!.queryByTestId('generated-image')).not.toBeNull(); + }, { timeout: SLOW_MS + 10_000 }); + expect(h.view!.queryByTestId('model-failure-image')).toBeNull(); + expect(desktop.aborted).toEqual([]); + }, 45_000); +}); diff --git a/__tests__/integration/settings/devToolingReleaseGate.rendered.test.tsx b/__tests__/integration/settings/devToolingReleaseGate.rendered.test.tsx index 4dca9411e..1450b3d0d 100644 --- a/__tests__/integration/settings/devToolingReleaseGate.rendered.test.tsx +++ b/__tests__/integration/settings/devToolingReleaseGate.rendered.test.tsx @@ -63,11 +63,11 @@ describe('developer tooling release gate (rendered, real App)', () => { const { view, rtl } = await bootReturningUser(false); // BEFORE the Settings tap: the user is on Home, the Settings screen is not yet shown. - expect(view.queryByText('Privacy First')).toBeNull(); + expect(view.queryByText('Report a problem')).toBeNull(); expect(view.queryByTestId('dev-sync-strip')).toBeNull(); rtl.fireEvent.press(view.getByTestId('settings-tab')); - await rtl.waitFor(() => expect(view.getByText('Privacy First')).toBeTruthy()); + await rtl.waitFor(() => expect(view.getByText('Report a problem')).toBeTruthy()); expect(view.queryByTestId('seed-demo-data')).toBeNull(); expect(view.queryByText('Load demo data')).toBeNull(); @@ -82,11 +82,11 @@ describe('developer tooling release gate (rendered, real App)', () => { it('a developer build shows the sync strip and both developer controls in Settings', async () => { const { view, rtl } = await bootReturningUser(true); - expect(view.queryByText('Privacy First')).toBeNull(); + expect(view.queryByText('Report a problem')).toBeNull(); expect(view.getByTestId('dev-sync-strip')).toBeTruthy(); rtl.fireEvent.press(view.getByTestId('settings-tab')); - await rtl.waitFor(() => expect(view.getByText('Privacy First')).toBeTruthy()); + await rtl.waitFor(() => expect(view.getByText('Report a problem')).toBeTruthy()); expect(view.getByTestId('seed-demo-data')).toBeTruthy(); expect(view.getByText('Load demo data')).toBeTruthy(); diff --git a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx index 84a19af5b..4d489de3c 100644 --- a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx +++ b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx @@ -17,21 +17,20 @@ it('offers only CPU and GPU in Chat and Model Settings even with a saved NPU pre const { useAppStore } = require('../../../src/stores'); await useAppStore.persist.rehydrate(); const { GenerationSettingsModal } = require('../../../src/components/GenerationSettingsModal'); - const { TextGenerationSection } = require('../../../src/screens/ModelSettingsScreen/TextGenerationSection'); + const { GeneralGenerationControls } = require('../../../src/components/settings/GeneralGenerationControls'); const model = createDownloadedModel({ id: 'gemma-litert', engine: 'litert', filePath: '/models/gemma.litertlm' }); useAppStore.getState().setDownloadedModels([model]); useAppStore.getState().setActiveModelId(model.id); useAppStore.getState().updateSettings({ liteRTBackend: 'npu' }); const chat = render(React.createElement(GenerationSettingsModal, { visible: true, onClose: () => {} })); - fireEvent.press(chat.getByText('TEXT GENERATION')); - fireEvent.press(chat.getByTestId('modal-text-advanced-toggle')); + // Acceleration is a General control on both surfaces. + fireEvent.press(chat.getByTestId('modal-general-accordion')); expect(chat.queryByText('NPU (Beta)')).toBeNull(); expect(chat.getByText(/Run on GPU via OpenCL/)).toBeTruthy(); fireEvent.press(chat.getByText('CPU')); expect(useAppStore.getState().settings.liteRTBackend).toBe('cpu'); chat.unmount(); - const settings = render(React.createElement(TextGenerationSection)); - fireEvent.press(settings.getByTestId('text-advanced-toggle')); + const settings = render(React.createElement(GeneralGenerationControls)); expect(settings.queryByText('NPU (Beta)')).toBeNull(); fireEvent.press(settings.getByText('GPU')); expect(useAppStore.getState().settings.liteRTBackend).toBe('gpu'); diff --git a/__tests__/integration/settings/loadParamsPerDevice.test.tsx b/__tests__/integration/settings/loadParamsPerDevice.test.tsx index f5c43166d..2724477a0 100644 --- a/__tests__/integration/settings/loadParamsPerDevice.test.tsx +++ b/__tests__/integration/settings/loadParamsPerDevice.test.tsx @@ -54,7 +54,9 @@ async function firstSendLoads(h: Harness): Promise> { /** Open the chat's settings sheet, expand Advanced, and return the mounted section. */ function openAdvanced(h: Harness) { const { TextGenerationSection } = require('../../../src/components/GenerationSettingsModal/TextGenerationSection'); - const section = h.rtl.render(h.React.createElement(TextGenerationSection, {})); + // The sheet shows the shared General controls (backend, loading mode) above the Text section. + const { GeneralGenerationControls } = require('../../../src/components/settings/GeneralGenerationControls'); + const section = h.rtl.render(h.React.createElement(h.React.Fragment, null, h.React.createElement(GeneralGenerationControls), h.React.createElement(TextGenerationSection, {}))); h.rtl.fireEvent.press(section.getByTestId('modal-text-advanced-toggle')); return section; } diff --git a/__tests__/integration/settings/modelSettingsSurfaceParity.test.tsx b/__tests__/integration/settings/modelSettingsSurfaceParity.test.tsx index d8caed028..90b13ec87 100644 --- a/__tests__/integration/settings/modelSettingsSurfaceParity.test.tsx +++ b/__tests__/integration/settings/modelSettingsSurfaceParity.test.tsx @@ -40,6 +40,40 @@ describe('model settings surface parity', () => { _clearSlotsForTesting(); }); + it('keeps shared controls in General on both surfaces and preserves saved values', () => { + useAppStore.getState().updateSettings({ + modelLoadingMode: 'aggressive', + showGenerationDetails: true, + }); + const chatSettings = render( {}} />); + fireEvent.press(chatSettings.getByText('TEXT GENERATION')); + expect(chatSettings.queryByText('Model Loading')).toBeNull(); + expect(chatSettings.queryByText('Show Generation Details')).toBeNull(); + fireEvent.press(chatSettings.getByTestId('modal-general-accordion')); + expect(chatSettings.getByText('Text Model Backend')).toBeTruthy(); + expect(chatSettings.getByTestId('model-loading-mode-aggressive-button')).toBeTruthy(); + expect(chatSettings.getByTestId('show-gen-details-on-button')).toBeTruthy(); + chatSettings.unmount(); + + const modelSettings = renderModelSettings(); + fireEvent.press(modelSettings.getByTestId('text-generation-accordion')); + expect(modelSettings.queryByText('Model Loading')).toBeNull(); + fireEvent.press(modelSettings.getByTestId('general-generation-accordion')); + expect(modelSettings.getByText('Text Model Backend')).toBeTruthy(); + expect(modelSettings.getByTestId('model-loading-mode-aggressive-button')).toBeTruthy(); + expect(modelSettings.getByTestId('show-gen-details-on-button')).toBeTruthy(); + }); + + it('shows and edits the saved default system prompt in chat Text settings', () => { + useAppStore.getState().updateSettings({ systemPrompt: 'Saved instructions' }); + const chatSettings = render( {}} />); + fireEvent.press(chatSettings.getByText('TEXT GENERATION')); + const input = chatSettings.getByTestId('default-system-prompt-input'); + expect(input.props.value).toBe('Saved instructions'); + fireEvent.changeText(input, 'New instructions'); + expect(useAppStore.getState().settings.systemPrompt).toBe('New instructions'); + }); + it('caps output by context on both surfaces and writes one shared setting state', () => { useAppStore.getState().setModelMaxContext(262144); // A context wide enough for the output this test chooses. Max tokens is capped BY the context, diff --git a/__tests__/integration/settings/useFastestSettings.test.tsx b/__tests__/integration/settings/useFastestSettings.test.tsx index cc9c50ae2..f6f662e56 100644 --- a/__tests__/integration/settings/useFastestSettings.test.tsx +++ b/__tests__/integration/settings/useFastestSettings.test.tsx @@ -30,7 +30,8 @@ jest.mock('@react-native-community/slider', () => ({ }, })); -const CPU_DESC = 'Always available. Stable, predictable performance.'; +/** The CPU's description, inside the backend row's sentence. */ +const CPU_DESC = /Always available\. Stable, predictable performance\./; type Harness = Awaited>; @@ -49,6 +50,8 @@ function npuPhone() { function openModelSettingsAdvanced(h: Harness) { const { ModelSettingsScreen } = require('../../../src/screens/ModelSettingsScreen'); const screen = h.rtl.render(h.React.createElement(ModelSettingsScreen, {})); + // The backend is a shared General control; the fastest-settings run is in Text > Advanced. + h.rtl.fireEvent.press(screen.getByTestId('general-generation-accordion')); h.rtl.fireEvent.press(screen.getByTestId('text-generation-accordion')); h.rtl.fireEvent.press(screen.getByTestId('text-advanced-toggle')); return screen; @@ -63,14 +66,14 @@ describe('Use fastest settings', () => { // BEFORE: the CPU is selected (its description shows, no NPU layer control). await h.rtl.waitFor(() => expect(screen.getByTestId('backend-htp-button')).toBeTruthy()); expect(screen.queryByText(CPU_DESC)).not.toBeNull(); - expect(screen.queryByText('NPU Layers')).toBeNull(); + expect(screen.queryByText('Text NPU Layers')).toBeNull(); h.rtl.fireEvent.press(screen.getByTestId('fastest-settings-button')); await h.rtl.waitFor(() => expect(screen.queryByTestId('fastest-settings-cancel')).not.toBeNull()); await h.rtl.waitFor(() => expect(screen.queryByTestId('fastest-settings-button')).not.toBeNull(), { timeout: 8000 }); // AFTER: the NPU is selected and the batch follows it. - expect(screen.queryByText('NPU Layers')).not.toBeNull(); + expect(screen.queryByText('Text NPU Layers')).not.toBeNull(); expect(screen.queryByText(CPU_DESC)).toBeNull(); expect(String(screen.getByTestId('batch-size-stepper-value').props.children)).toBe('Auto (128)'); // Every backend this device class may run was really measured, each on its own load. @@ -87,7 +90,7 @@ describe('Use fastest settings', () => { // BEFORE: the user had picked the GPU. h.rtl.fireEvent.press(screen.getByTestId('backend-opencl-button')); - await h.rtl.waitFor(() => expect(screen.queryByText('GPU Layers (OpenCL)')).not.toBeNull()); + await h.rtl.waitFor(() => expect(screen.queryByText('Text GPU Layers (OpenCL)')).not.toBeNull()); expect(screen.queryByText(CPU_DESC)).toBeNull(); h.rtl.fireEvent.press(screen.getByTestId('fastest-settings-button')); @@ -97,8 +100,8 @@ describe('Use fastest settings', () => { // AFTER: the CPU, the only backend that really ran — all three were tried. expect(h.boundary.llama!.calls.bench.map(b => b.backend)).toEqual(['cpu', 'htp', 'opencl']); expect(screen.queryByText(CPU_DESC)).not.toBeNull(); - expect(screen.queryByText('GPU Layers (OpenCL)')).toBeNull(); - expect(screen.queryByText('NPU Layers')).toBeNull(); + expect(screen.queryByText('Text GPU Layers (OpenCL)')).toBeNull(); + expect(screen.queryByText('Text NPU Layers')).toBeNull(); }); it('a phone under 6 GB goes to the CPU without measuring anything', async () => { @@ -112,13 +115,13 @@ describe('Use fastest settings', () => { }); const screen = openModelSettingsAdvanced(h); h.rtl.fireEvent.press(screen.getByTestId('backend-opencl-button')); - await h.rtl.waitFor(() => expect(screen.queryByText('GPU Layers (OpenCL)')).not.toBeNull()); + await h.rtl.waitFor(() => expect(screen.queryByText('Text GPU Layers (OpenCL)')).not.toBeNull()); await h.rtl.act(async () => { h.rtl.fireEvent.press(screen.getByTestId('fastest-settings-button')); }); await h.rtl.waitFor(() => expect(screen.queryByText(CPU_DESC)).not.toBeNull()); - expect(screen.queryByText('GPU Layers (OpenCL)')).toBeNull(); + expect(screen.queryByText('Text GPU Layers (OpenCL)')).toBeNull(); expect(h.boundary.llama!.calls.bench).toHaveLength(0); expect(h.boundary.llama!.calls.initLlama).toHaveLength(0); }); @@ -130,7 +133,7 @@ describe('Use fastest settings', () => { const screen = openModelSettingsAdvanced(h); h.rtl.fireEvent.press(screen.getByTestId('backend-opencl-button')); - await h.rtl.waitFor(() => expect(screen.queryByText('GPU Layers (OpenCL)')).not.toBeNull()); + await h.rtl.waitFor(() => expect(screen.queryByText('Text GPU Layers (OpenCL)')).not.toBeNull()); h.rtl.fireEvent.press(screen.getByTestId('fastest-settings-button')); const cancel = await h.rtl.waitFor(() => screen.getByTestId('fastest-settings-cancel')); @@ -142,8 +145,8 @@ describe('Use fastest settings', () => { await h.rtl.waitFor(() => expect(screen.queryByTestId('fastest-settings-button')).not.toBeNull(), { timeout: 8000 }); // Nothing else was measured and nothing changed: still the GPU the user picked. expect(h.boundary.llama!.calls.bench).toHaveLength(1); - expect(screen.queryByText('GPU Layers (OpenCL)')).not.toBeNull(); - expect(screen.queryByText('NPU Layers')).toBeNull(); + expect(screen.queryByText('Text GPU Layers (OpenCL)')).not.toBeNull(); + expect(screen.queryByText('Text NPU Layers')).toBeNull(); expect(h.useAppStore.getState().speedBenchmarkBackend).toBeNull(); }); }); diff --git a/__tests__/pro/mcp/desktopToolsGrant.integration.test.tsx b/__tests__/pro/mcp/desktopToolsGrant.integration.test.tsx new file mode 100644 index 000000000..1ee57ac6e --- /dev/null +++ b/__tests__/pro/mcp/desktopToolsGrant.integration.test.tsx @@ -0,0 +1,183 @@ +/** + * Desktop tools: run a paired desktop's tools from this phone. + * + * SPEC (OGAM user's view): Pro tools lists each paired desktop — only desktops; a paired phone or + * tablet serves no tools and is not listed. Its switch is off until the desktop has granted its + * tools here. Turning it on asks that desktop; when the desktop grants, the switch stays on and its + * tools connect. Turning it off asks the desktop to take them back; when it revokes, the switch is off. + * + * The real app on the real navigation stack, the real sync service, MCP grant service and MCP + * client. Fakes only at the device boundaries: the TCP socket and discovery natives, the licence + * provider (licensedMesh), the desktop itself (a real sync engine on the other end of the socket, + * whose app messages are what it received) and the desktop's MCP server over HTTP (mcpHttpFake). + * Replaces the mockist rntl companionToolsSection test, which mocked our stores and grant service. + */ +import React from 'react'; +import AsyncStorage from '@react-native-async-storage/async-storage'; +import * as Keychain from 'react-native-keychain'; +import TcpSocket from 'react-native-tcp-socket'; +import { fireEvent, render, waitFor } from '@testing-library/react-native'; +import { NavigationContainer } from '@react-navigation/native'; +import type { DeviceInfo } from '@offgrid/sync'; +import type { RnTcpModule } from '@offgrid/sync/rn'; +import { buildSyncEngine } from '../../../src/services/sync/engine'; +import { AppNavigator } from '../../../src/navigation/AppNavigator'; +import { registerScreen, _clearScreensForTesting } from '../../../src/navigation/screenRegistry'; +import { _clearSectionsForTesting } from '../../../src/components/settings/sectionRegistry'; +import { useAppStore } from '../../../src/stores/appStore'; +import { syncService } from '../../../pro/sync/syncService'; +import { useSyncStore } from '../../../pro/sync/syncStore'; +import { SyncScreen } from '../../../pro/ui/SyncScreen'; +import { McpServersScreen } from '../../../pro/ui/McpServersScreen'; +import { McpToolsScreen } from '../../../pro/ui/McpToolsScreen'; +import { useMcpStore } from '../../../pro/mcp/mcpStore'; +import { disconnectServer } from '../../../pro/mcp/mcpService'; +import { initMcpToolGrants } from '../../../pro/mcp/mcpToolGrantService'; +import { + MCP_TOOL_GRANT_CHANNEL, + MCP_TOOL_REQUEST_CHANNEL, + MCP_TOOL_REVOKE_CHANNEL, +} from '../../../pro/mcp/toolGrant'; +import { getDiscoveryBoundaries, resetDiscoveryBoundaries } from '../../utils/nativeSyncBoundaries'; +import { pairingCodeOnScreen } from '../../utils/pairFromPeer'; +import { createDownloadedModel } from '../../utils/factories'; +import { createLicensedMesh, installLicensedPhone, registerThisPhone } from '../../harness/licensedMesh'; +import { installMcpHttpFake, type McpHttpFake } from '../../harness/mcpHttpFake'; + +jest.unmock('@react-navigation/native'); +jest.mock('react-native-tcp-socket', () => { + const { createNativeTcpBoundary } = require('../../utils/nativeSyncBoundaries'); + return { __esModule: true, default: createNativeTcpBoundary() }; +}); +jest.mock('react-native-zeroconf', () => { + const { createNativeDiscoveryBoundary } = require('../../utils/nativeSyncBoundaries'); + return { __esModule: true, default: createNativeDiscoveryBoundary() }; +}); + +const PHONE_FINGERPRINT = 'fp-this-phone'; +/** The desktop's companion MCP server, as its grant names it (Desktop's tools gateway). */ +const DESKTOP_MCP_URL = 'http://192.168.1.70:7878/mcp'; // NOSONAR - private LAN test fixture +const GRANT = { t: 'offgrid-mcp-pair', v: 1, url: DESKTOP_MCP_URL, token: 'desktop-tools-token-0123456789', name: 'Studio Mac' }; + +const mesh = createLicensedMesh(); + +describe('Desktop tools in Pro tools', () => { + let desktop: ReturnType | undefined; + let ui: ReturnType | undefined; + let mcp: McpHttpFake | undefined; + let stopGrants: (() => void) | undefined; + + beforeEach(async () => { + mesh.reset(); + await syncService.stop(); + await AsyncStorage.clear(); + resetDiscoveryBoundaries(); + _clearScreensForTesting(); + _clearSectionsForTesting(); + // Registered the way pro/index.ts registers them. + registerScreen({ name: 'Sync', component: SyncScreen }); + registerScreen({ name: 'McpServers', component: McpServersScreen }); + registerScreen({ name: 'McpTools', component: McpToolsScreen }); + useAppStore.getState().setOnboardingComplete(true); + // Pro is an entitlement the app is told about, so it is seeded like any other outside fact. + useAppStore.getState().setProActive(true); + useAppStore.getState().setDownloadedModels([createDownloadedModel({ engine: 'litert' })]); + useSyncStore.getState().reset(); + installLicensedPhone(mesh, { fingerprint: PHONE_FINGERPRINT }); + await registerThisPhone(mesh); + // A tablet already on the licence: a paired device that serves no tools. + mesh.register({ id: 'fp-kitchen-tablet', name: 'Kitchen Tablet', platform: 'android' }); + (Keychain.setGenericPassword as jest.Mock).mockResolvedValue(true); + mcp = installMcpHttpFake({ + [DESKTOP_MCP_URL]: { + label: 'Studio Mac', + tools: [ + { name: 'read_file', description: 'Read a file', inputSchema: { type: 'object', properties: {} } }, + { name: 'list_files', description: 'List files', inputSchema: { type: 'object', properties: {} } }, + ], + }, + }); + // What pro/index.ts starts on activation: the listener for grants and revokes. + stopGrants = initMcpToolGrants(); + }); + + afterEach(async () => { + ui?.unmount(); + ui = undefined; + stopGrants?.(); + for (const server of useMcpStore.getState().servers) { + disconnectServer(server.id); + useMcpStore.getState().removeServer(server.id); + } + mcp?.restore(); + mesh.restore(); + await desktop?.engine.stop(); + desktop = undefined; + await syncService.stop(); + _clearScreensForTesting(); + _clearSectionsForTesting(); + }); + + it('lists only desktops; the switch asks the desktop, and follows its grant and its revoke', async () => { + const received: Array<{ channel: string; data: unknown }> = []; + const desktopDevice: DeviceInfo = { + id: 'studio-mac', name: 'Studio Mac', platform: 'macos', version: '1', host: '127.0.0.1', port: 0, + }; + desktop = buildSyncEngine({ + pairingEntitlement: mesh.joiner({ name: desktopDevice.name, platform: desktopDevice.platform }), + localDevice: desktopDevice, + tcpModule: TcpSocket as unknown as RnTcpModule, + onAppMessage: (_deviceId, channel, data) => { + received.push({ channel, data }); + }, + }); + await desktop.engine.start(0); + desktopDevice.port = desktop.transport.boundPort ?? 0; + await syncService.start(); + + ui = render( + + + , + ); + // Settings › Sync, and the desktop pairs with the code on screen. + fireEvent.press(ui.getByTestId('settings-tab')); + await waitFor(() => expect(ui!.getByText('Model Settings')).toBeTruthy()); + fireEvent.press(ui.getByTestId('open-sync-settings')); + await waitFor(() => expect(ui!.getByTestId('sync-pairing-code-value')).toBeTruthy()); + const phone = useSyncStore.getState().thisDevice; + const discovery = getDiscoveryBoundaries().at(-1); + if (!phone || !discovery?.publishedPort) throw new Error('Sync did not publish this phone'); + await desktop.engine.pair({ ...phone, host: '127.0.0.1', port: discovery.publishedPort }, await pairingCodeOnScreen(ui)); + + // Back to Settings › Off Grid AI PRO › Your own tools, in your chats. + fireEvent.press(ui.getAllByLabelText('Back').pop()!); + fireEvent.press(await waitFor(() => ui!.getByText('Off Grid AI PRO'))); + fireEvent.press(await waitFor(() => ui!.getByText('Your own tools, in your chats'))); + const row = await waitFor(() => ui!.getByTestId(`companion-tools-${desktopDevice.id}`), { timeout: 10_000 }); + expect(row).toBeTruthy(); + expect(ui.queryByTestId('companion-tools-fp-kitchen-tablet')).toBeNull(); + expect(ui.queryByText('Kitchen Tablet')).toBeNull(); + const toggle = () => ui!.getByTestId(`companion-tools-switch-${desktopDevice.id}`); + expect(toggle().props.value).toBe(false); + + // On: the desktop is asked. + fireEvent(toggle(), 'valueChange', true); + await waitFor(() => + expect(received).toContainEqual({ channel: MCP_TOOL_REQUEST_CHANNEL, data: { allow: true } }), + ); + // The desktop grants: the switch reflects the grant, and its tools connect here. + desktop.engine.sendApp(phone.id, MCP_TOOL_GRANT_CHANNEL, GRANT); + await waitFor(() => expect(toggle().props.value).toBe(true), { timeout: 10_000 }); + await waitFor(() => expect(ui!.getByText(/^\d\/2 tools enabled$/)).toBeTruthy(), { timeout: 15_000 }); + + // Off: the desktop is asked to take them back; when it revokes, the switch is off. + fireEvent(toggle(), 'valueChange', false); + await waitFor(() => + expect(received).toContainEqual({ channel: MCP_TOOL_REQUEST_CHANNEL, data: { allow: false } }), + ); + desktop.engine.sendApp(phone.id, MCP_TOOL_REVOKE_CHANNEL, { url: DESKTOP_MCP_URL }); + await waitFor(() => expect(toggle().props.value).toBe(false), { timeout: 10_000 }); + expect(ui.queryByText(/^\d\/2 tools enabled$/)).toBeNull(); + }, 60_000); +}); diff --git a/__tests__/pro/sync/galleryRepointToSyncedCopy.integration.test.tsx b/__tests__/pro/sync/galleryRepointToSyncedCopy.integration.test.tsx index d4cc88039..3590023cf 100644 --- a/__tests__/pro/sync/galleryRepointToSyncedCopy.integration.test.tsx +++ b/__tests__/pro/sync/galleryRepointToSyncedCopy.integration.test.tsx @@ -276,7 +276,7 @@ describe('Gallery record repointed at its synced copy', () => { ui.getByLabelText(`Generated image unavailable: ${PROMPT}`), ).toBeTruthy(); expect(ui.queryByLabelText(`Generated image loaded: ${PROMPT}`)).toBeNull(); - fireEvent.press(ui.getByLabelText('Close gallery')); + fireEvent.press(ui.getByTestId('gallery-back')); // The user pairs the Desktop from Settings > Sync. fireEvent.press(await waitFor(() => ui!.getByTestId('settings-tab'))); diff --git a/__tests__/pro/sync/ghostDesktopRows.integration.test.tsx b/__tests__/pro/sync/ghostDesktopRows.integration.test.tsx new file mode 100644 index 000000000..4a4dd9963 --- /dev/null +++ b/__tests__/pro/sync/ghostDesktopRows.integration.test.tsx @@ -0,0 +1,143 @@ +import React from 'react'; +import AsyncStorage from '@react-native-async-storage/async-storage'; +import { NavigationContainer } from '@react-navigation/native'; +import { render, fireEvent, waitFor } from '@testing-library/react-native'; +import { PERSONAL_MESH_DEVICE_CAP } from '@offgrid/sync'; + +import { AppNavigator } from '../../../src/navigation/AppNavigator'; +import { + registerScreen, + _clearScreensForTesting, +} from '../../../src/navigation/screenRegistry'; +import { _clearSectionsForTesting } from '../../../src/components/settings/sectionRegistry'; +import { useAppStore } from '../../../src/stores/appStore'; +import { createDownloadedModel } from '../../utils/factories'; +import { SyncScreen } from '../../../pro/ui/SyncScreen'; +import { + selectSyncControlCenter, + useSyncStore, +} from '../../../pro/sync/syncStore'; +import { syncService } from '../../../pro/sync/syncService'; +import { pairingSecretStore } from '../../../pro/sync/pairingSecretStore'; +import { + createLicensedMesh, + installLicensedPhone, +} from '../../harness/licensedMesh'; + +/** + * One computer, one row. + * + * Pro tools listed the same Mac three times - two rows reading "Off Grid AI Desktop" and one + * "OGAD: Mac (Debug)" - because it built its own list from saved credentials, and a desktop keeps + * one credential per install. The shared projection is the roster authority and already drops a + * machine the licence no longer knows; the Sync screen was reading it and was right all along. + * + * So this proves one thing: both surfaces name the same computers. + */ + +jest.unmock('@react-navigation/native'); + +jest.mock('react-native-tcp-socket', () => { + const { + createNativeTcpBoundary, + } = require('../../utils/nativeSyncBoundaries'); + return { __esModule: true, default: createNativeTcpBoundary() }; +}); + +jest.mock('react-native-zeroconf', () => { + const { + createNativeDiscoveryBoundary, + } = require('../../utils/nativeSyncBoundaries'); + return { __esModule: true, default: createNativeDiscoveryBoundary() }; +}); + +const THIS_FINGERPRINT = 'fp-this-phone'; +/** The Mac the person is using: on the licence, and paired here. */ +const LIVE_MAC = 'fp-mac-live'; +/** The same Mac after a reinstall: paired here, gone from the licence. */ +const GHOST_MAC = 'fp-mac-ghost'; +const DESKTOP_NAME = 'Off Grid AI Desktop'; + +const mesh = createLicensedMesh(); +const storedSecrets = new Map(); + +/** Pair a desktop with this phone, the way the pairing flow leaves the store. */ +async function pairDesktop(id: string, port: number): Promise { + const device = { + id, + name: DESKTOP_NAME, + platform: 'macos' as const, + version: '1.0.0', + host: '192.168.1.34', + port, + sharedSecret: `secret-${id}`, + pairedAt: 1, + }; + await pairingSecretStore.beginPairing(device); + await pairingSecretStore.commitPairing(device); +} + +describe('Pro tools and Sync name the same computers', () => { + beforeEach(async () => { + mesh.reset(PERSONAL_MESH_DEVICE_CAP); + await syncService.stop(); + await AsyncStorage.clear(); + jest.clearAllMocks(); + _clearScreensForTesting(); + _clearSectionsForTesting(); + registerScreen({ name: 'Sync', component: SyncScreen }); + + const app = useAppStore.getState(); + app.setOnboardingComplete(true); + app.setDownloadedModels([createDownloadedModel()]); + app.setThemeMode('dark'); + app.setHasRegisteredPro(true); + app.setProActive(true); + useSyncStore.getState().reset(); + + storedSecrets.clear(); + installLicensedPhone(mesh, { + fingerprint: THIS_FINGERPRINT, + secrets: storedSecrets, + }); + + // The licence knows this phone and ONE Mac. The reinstalled identity is not on it. + mesh.register({ id: THIS_FINGERPRINT, name: 'My iPhone', platform: 'ios' }); + mesh.register({ id: LIVE_MAC, name: DESKTOP_NAME, platform: 'macos' }); + + // The phone, however, holds a credential for both. + await pairingSecretStore.load(); + await pairDesktop(LIVE_MAC, 37878); + await pairDesktop(GHOST_MAC, 37879); + }); + + afterEach(async () => { + mesh.restore(); + await syncService.stop(); + _clearScreensForTesting(); + _clearSectionsForTesting(); + }); + + it('shows the computer once, not once per identity it has ever had', async () => { + await syncService.start(); + const ui = render( + + + , + ); + fireEvent.press(ui.getByTestId('settings-tab')); + fireEvent.press(await waitFor(() => ui.getByTestId('open-sync-settings'))); + + // Both credentials are saved, so a list built from them would name this Mac twice. + await waitFor(() => + expect( + selectSyncControlCenter(useSyncStore.getState()) + .allDevices.map(device => device.id) + .filter(id => id === LIVE_MAC || id === GHOST_MAC), + ).toEqual([LIVE_MAC]), + ); + expect(ui.getAllByText(DESKTOP_NAME)).toHaveLength(1); + + ui.unmount(); + }); +}); diff --git a/__tests__/pro/sync/pairedGatewayKey.integration.test.tsx b/__tests__/pro/sync/pairedGatewayKey.integration.test.tsx new file mode 100644 index 000000000..a8869e0d4 --- /dev/null +++ b/__tests__/pro/sync/pairedGatewayKey.integration.test.tsx @@ -0,0 +1,197 @@ +/** + * A computer paired through Sync lends this phone its gateway with the phone's own key. + * + * SPEC (OGAM user's view): pair the phone with a desktop on the Sync screen. The desktop's gateway + * (image, video, search) then works from the phone, even when the owner requires API keys: the + * desktop hands its key over Sync, and nobody types it. The desktop stays one row when a scan finds + * it at another address. + * + * The real app on the real navigation stack, the real sync service, paired-gateway adoption, remote + * server manager, store and health check. Fakes only at device boundaries: the TCP socket and + * discovery natives, the licence provider and Keychain (licensedMesh), the + * LAN (lanProbe: the desktop's gateway, keyed, naming its Sync id on /health) and the desktop + * itself (a real sync engine on the other end of the socket). + */ +import React from 'react'; +import AsyncStorage from '@react-native-async-storage/async-storage'; +import TcpSocket from 'react-native-tcp-socket'; +import { fireEvent, render, waitFor } from '@testing-library/react-native'; +import { NavigationContainer } from '@react-navigation/native'; +import { GATEWAY_ACCESS_CHANNEL, type DeviceInfo } from '@offgrid/sync'; +import type { RnTcpModule } from '@offgrid/sync/rn'; +import { buildSyncEngine } from '../../../src/services/sync/engine'; +import { AppNavigator } from '../../../src/navigation/AppNavigator'; +import { registerScreen, _clearScreensForTesting } from '../../../src/navigation/screenRegistry'; +import { _clearSectionsForTesting } from '../../../src/components/settings/sectionRegistry'; +import { useAppStore } from '../../../src/stores/appStore'; +import { useRemoteServerStore } from '../../../src/stores/remoteServerStore'; +import { remoteServerManager } from '../../../src/services/remoteServerManager'; +import { syncService } from '../../../pro/sync/syncService'; +import { useSyncStore } from '../../../pro/sync/syncStore'; +import { startPairedGatewayAdoption } from '../../../pro/sync/pairedGatewayAdoption'; +import { SyncScreen } from '../../../pro/ui/SyncScreen'; +import { getDiscoveryBoundaries, resetDiscoveryBoundaries } from '../../utils/nativeSyncBoundaries'; +import { pairingCodeOnScreen } from '../../utils/pairFromPeer'; +import { createDownloadedModel } from '../../utils/factories'; +import { createLicensedMesh, installLicensedPhone, registerThisPhone } from '../../harness/licensedMesh'; +import { gatewayModelList, installLanProbe, type LanProbeHandle } from '../../harness/lanProbe'; + +jest.unmock('@react-navigation/native'); +jest.mock('react-native-tcp-socket', () => { + const { createNativeTcpBoundary } = require('../../utils/nativeSyncBoundaries'); + return { __esModule: true, default: createNativeTcpBoundary() }; +}); +jest.mock('react-native-zeroconf', () => { + const { createNativeDiscoveryBoundary } = require('../../utils/nativeSyncBoundaries'); + return { __esModule: true, default: createNativeDiscoveryBoundary() }; +}); + +const PHONE_FINGERPRINT = 'fp-this-phone'; +const DESKTOP_ID = 'studio-mac'; +const GATEWAY_KEY = `ogad_${'a1'.repeat(32)}`; +const GATEWAY = 'http://192.168.1.70:7878'; // NOSONAR - private LAN test fixture + +const mesh = createLicensedMesh(); + +/** The desktop's gateway on the LAN: keys required, its Sync id on /health. */ +const desktopGateway = (host: string) => ({ + [`${host}:7878`]: { + paths: ['/v1/models', '/health'], + body: gatewayModelList, + locked: true, + apiKey: GATEWAY_KEY, + computerId: DESKTOP_ID, + }, +}); + +describe('a paired computer lends this phone its gateway', () => { + let desktop: ReturnType | undefined; + let ui: ReturnType | undefined; + let lan: LanProbeHandle | undefined; + let stopAdoption: (() => void) | undefined; + + beforeEach(async () => { + mesh.reset(); + await syncService.stop(); + await AsyncStorage.clear(); + resetDiscoveryBoundaries(); + _clearScreensForTesting(); + _clearSectionsForTesting(); + registerScreen({ name: 'Sync', component: SyncScreen }); + useAppStore.getState().setOnboardingComplete(true); + useAppStore.getState().setProActive(true); + useAppStore.getState().setDownloadedModels([createDownloadedModel({ engine: 'litert' })]); + useSyncStore.getState().reset(); + await remoteServerManager.clearAllServers(); + // The phone's network, at the device boundary: a real handset on the desktop's Wi-Fi subnet. + const DeviceInfo = require('react-native-device-info'); + DeviceInfo.isEmulator = async () => false; + DeviceInfo.getIpAddress = async () => '192.168.1.42'; + installLicensedPhone(mesh, { fingerprint: PHONE_FINGERPRINT }); + await registerThisPhone(mesh); + // What pro/index.ts starts on activation. + stopAdoption = startPairedGatewayAdoption(); + }); + + afterEach(async () => { + ui?.unmount(); + ui = undefined; + stopAdoption?.(); + lan?.uninstall(); + lan = undefined; + mesh.restore(); + await desktop?.engine.stop(); + desktop = undefined; + await syncService.stop(); + _clearScreensForTesting(); + _clearSectionsForTesting(); + }); + + async function pairWithDesktop(): Promise<{ phoneId: string }> { + const desktopDevice: DeviceInfo = { + id: DESKTOP_ID, name: 'Studio Mac', platform: 'macos', version: '1', host: '127.0.0.1', port: 0, + }; + desktop = buildSyncEngine({ + pairingEntitlement: mesh.joiner({ name: desktopDevice.name, platform: desktopDevice.platform }), + localDevice: desktopDevice, + tcpModule: TcpSocket as unknown as RnTcpModule, + }); + await desktop.engine.start(0); + desktopDevice.port = desktop.transport.boundPort ?? 0; + await syncService.start(); + + ui = render( + + + , + ); + fireEvent.press(ui.getByTestId('settings-tab')); + await waitFor(() => expect(ui!.getByText('Model Settings')).toBeTruthy()); + fireEvent.press(ui.getByTestId('open-sync-settings')); + await waitFor(() => expect(ui!.getByTestId('sync-pairing-code-value')).toBeTruthy()); + const phone = useSyncStore.getState().thisDevice; + const discovery = getDiscoveryBoundaries().at(-1); + if (!phone || !discovery?.publishedPort) throw new Error('Sync did not publish this phone'); + await desktop.engine.pair({ ...phone, host: '127.0.0.1', port: discovery.publishedPort }, await pairingCodeOnScreen(ui)); + return { phoneId: phone.id }; + } + + const pairedServer = () => useRemoteServerStore.getState().getServerById(`paired:${DESKTOP_ID}`); + + /** The Remote Servers connection check on the desktop's row passes: the gateway took the key. */ + async function connectsWithItsKey(): Promise { + const deadline = Date.now() + 10_000; + for (;;) { + const result = pairedServer() + ? await remoteServerManager.testConnection(`paired:${DESKTOP_ID}`) + : { success: false }; + if (result.success) { + expect(result.models?.map(model => model.id)).toEqual(gatewayModelList.data.map(model => model.id)); + return; + } + if (Date.now() > deadline) throw new Error('the desktop gateway never accepted this phone'); + await new Promise(resolve => setTimeout(resolve, 50)); + } + } + + it('uses the key the desktop sends over Sync, and keeps one row when a scan finds it elsewhere', async () => { + const { phoneId } = await pairWithDesktop(); + // The LAN stands in for every request from here on, so it goes up once the licence has answered. + lan = installLanProbe(desktopGateway('192.168.1.70')); + + // Without a key the gateway refuses this phone, as it did before pairing gave one. + expect((await remoteServerManager.testConnectionByEndpoint(GATEWAY)).success).toBe(false); + + desktop!.engine.sendApp(phoneId, GATEWAY_ACCESS_CHANNEL, { + version: 1, computerId: DESKTOP_ID, name: 'Studio Mac', endpoint: GATEWAY, key: GATEWAY_KEY, + }); + + await connectsWithItsKey(); + expect(pairedServer()).toMatchObject({ endpoint: GATEWAY, computerId: DESKTOP_ID }); + + // The desktop's address changed: a scan finds the same computer, not a new server. + lan.uninstall(); + lan = installLanProbe(desktopGateway('192.168.1.71')); + const { found } = await remoteServerManager.scanAndReconcile({ kinds: ['gateway'] }); + expect(found).toEqual([]); + expect(remoteServerManager.getServers()).toHaveLength(1); + }, 60_000); + + it('ignores a key that names another computer than the one that sent it', async () => { + const { phoneId } = await pairWithDesktop(); + // The LAN stands in for every request from here on, so it goes up once the licence has answered. + lan = installLanProbe(desktopGateway('192.168.1.70')); + + desktop!.engine.sendApp(phoneId, GATEWAY_ACCESS_CHANNEL, { + version: 1, computerId: 'some-other-mac', name: 'Other Mac', endpoint: GATEWAY, key: GATEWAY_KEY, + }); + // The genuine grant after it is applied, so the wrong one had its chance to land first. + desktop!.engine.sendApp(phoneId, GATEWAY_ACCESS_CHANNEL, { + version: 1, computerId: DESKTOP_ID, name: 'Studio Mac', endpoint: GATEWAY, key: GATEWAY_KEY, + }); + + await connectsWithItsKey(); + expect(useRemoteServerStore.getState().getServerById('paired:some-other-mac')).toBeNull(); + expect(remoteServerManager.getServers()).toHaveLength(1); + }, 60_000); +}); diff --git a/__tests__/pro/sync/stateSync.integration.test.tsx b/__tests__/pro/sync/stateSync.integration.test.tsx index dbc5cf82f..81c301a2e 100644 --- a/__tests__/pro/sync/stateSync.integration.test.tsx +++ b/__tests__/pro/sync/stateSync.integration.test.tsx @@ -297,9 +297,12 @@ describe('Pro mobile state sync journey', () => { created_at: createdAt, }, ); + // Twenty real edits: a put that repeats the current value is not a new op, so each revision + // carries its own value and the Desktop's final 0.55 is genuinely later than the phone's + // own first-run settings. for (let revision = 0; revision < 20; revision += 1) { remoteLog.record(CORE_SYNC_ENTITIES.modelSetting, 'temperature', 'put', { - value_json: revision === 19 ? '0.55' : '0.5', + value_json: revision === 19 ? '0.55' : (0.3 + revision * 0.01).toFixed(2), }); } @@ -318,7 +321,7 @@ describe('Pro mobile state sync journey', () => { ); fireEvent.press(ui.getByTestId('settings-tab')); fireEvent.press(ui.getByText('Model Settings')); - fireEvent.press(ui.getByTestId('text-generation-accordion')); + fireEvent.press(ui.getByTestId('general-generation-accordion')); fireEvent.press(ui.getByTestId('show-gen-details-on-button')); fireEvent.press(ui.getByLabelText('Back')); fireEvent.press(await waitFor(() => ui!.getByTestId('open-sync-settings'))); diff --git a/__tests__/pro/sync/studioSync.integration.test.tsx b/__tests__/pro/sync/studioSync.integration.test.tsx new file mode 100644 index 000000000..3b295d940 --- /dev/null +++ b/__tests__/pro/sync/studioSync.integration.test.tsx @@ -0,0 +1,224 @@ +/** + * Films shaped on one device are on the other: Video Studio stories and characters over Sync. + * + * SPEC (OGAM user's view): after I pair my Mac, a film I storyboarded there — and the person kept in + * it — shows in Your videos on the phone, and opening it shows its shots and that person. A change I + * make to it on the phone reaches the Mac. A film I delete on the Mac leaves the phone too. + * + * The real app on the real navigation stack, the real sync service, state sync and its SQLite op log, + * and Pro's studio stores and sync wiring (registered the way pro/index.ts does). Fakes only at the + * device boundaries: the TCP socket and discovery natives, the licence provider (licensedMesh), and the + * Mac itself — a real sync engine with its own op log on the other end of the socket. + */ +import React from 'react'; +import AsyncStorage from '@react-native-async-storage/async-storage'; +import * as Keychain from 'react-native-keychain'; +import TcpSocket from 'react-native-tcp-socket'; +import { fireEvent, render, waitFor } from '@testing-library/react-native'; +import { NavigationContainer } from '@react-navigation/native'; +import { OpLog, StateSync, type DeviceInfo, type Materializer } from '@offgrid/sync'; +import type { RnTcpModule } from '@offgrid/sync/rn'; +import { buildSyncEngine } from '../../../src/services/sync/engine'; +import type { SyncMutation } from '../../../src/services/sync/mutation'; +import { AppNavigator } from '../../../src/navigation/AppNavigator'; +import { registerScreen, _clearScreensForTesting } from '../../../src/navigation/screenRegistry'; +import { _clearSectionsForTesting } from '../../../src/components/settings/sectionRegistry'; +import { HOOKS, _clearHooksForTesting, registerHook } from '../../../src/bootstrap/hookRegistry'; +import { useAppStore } from '../../../src/stores/appStore'; +import { syncService } from '../../../pro/sync/syncService'; +import { stateSyncService } from '../../../pro/sync/stateSyncService'; +import { useSyncStore } from '../../../pro/sync/syncStore'; +import { SyncScreen } from '../../../pro/ui/SyncScreen'; +import { ProRoot } from '../../../pro/ui/ProRoot'; +import { LibraryScreen } from '../../../pro/video/ui/studio/LibraryScreen'; +import { StoryScreen } from '../../../pro/video/ui/studio/StoryScreen'; +import { MakingScreen } from '../../../pro/video/ui/studio/MakingScreen'; +import { STUDIO_SCREENS } from '../../../pro/video/ui/studio/navigation'; +import { startStudioSync } from '../../../pro/video/studio/studioSync'; +import { useStudioStore } from '../../../pro/video/studio/studioStore'; +import { getDiscoveryBoundaries, resetDiscoveryBoundaries } from '../../utils/nativeSyncBoundaries'; +import { pairingCodeOnScreen } from '../../utils/pairFromPeer'; +import { createDownloadedModel } from '../../utils/factories'; +import { createLicensedMesh, installLicensedPhone } from '../../harness/licensedMesh'; + +const PHONE_FINGERPRINT = 'fp-this-phone'; + +jest.unmock('@react-navigation/native'); +// The sync op store keeps its history in SQLite: a real database for it, the inert mock elsewhere. +jest.mock('@op-engineering/op-sqlite', () => + require('../../harness/sqliteFake').realSqliteModule({ only: ['offgrid-sync.sqlite'] }), +); +jest.mock('react-native-tcp-socket', () => { + const { createNativeTcpBoundary } = require('../../utils/nativeSyncBoundaries'); + return { __esModule: true, default: createNativeTcpBoundary() }; +}); +jest.mock('react-native-zeroconf', () => { + const { createNativeDiscoveryBoundary } = require('../../utils/nativeSyncBoundaries'); + return { __esModule: true, default: createNativeDiscoveryBoundary() }; +}); + +/** What the Mac holds: every record its op log materialized, by entity:id. */ +class MacRecords implements Materializer { + readonly records = new Map>(); + put(entity: string, entityId: string, fields: Record): void { + this.records.set(`${entity}:${entityId}`, fields); + } + remove(entity: string, entityId: string): void { + this.records.delete(`${entity}:${entityId}`); + } +} + +const mesh = createLicensedMesh(); +const STORY = 'mac-story-fox'; +const CHARACTER = 'mac-character-mia'; + +describe('Video Studio over Sync', () => { + let mac: ReturnType | undefined; + let ui: ReturnType | undefined; + let stopStudioSync: (() => void) | undefined; + + beforeEach(async () => { + mesh.reset(); + _clearHooksForTesting(); + await stateSyncService.stop(); + await syncService.stop(); + await AsyncStorage.clear(); + resetDiscoveryBoundaries(); + _clearScreensForTesting(); + _clearSectionsForTesting(); + registerScreen({ name: 'Sync', component: SyncScreen }); + // Registered the way pro/video/index.ts registers them. + registerScreen({ name: STUDIO_SCREENS.library, component: LibraryScreen }); + registerScreen({ name: STUDIO_SCREENS.story, component: StoryScreen }); + registerScreen({ name: STUDIO_SCREENS.making, component: MakingScreen }); + useAppStore.getState().setOnboardingComplete(true); + // Pro is an entitlement the app is told about, so it is seeded like any other outside fact. + useAppStore.getState().setProActive(true); + useAppStore.getState().setDownloadedModels([createDownloadedModel({ engine: 'litert' })]); + useSyncStore.getState().reset(); + for (const id of Object.keys(useStudioStore.getState().stories)) useStudioStore.getState().removeStory(id); + for (const id of Object.keys(useStudioStore.getState().characters)) useStudioStore.getState().removeCharacter(id); + registerHook(HOOKS.syncRecordLocalMutation, (mutation: SyncMutation) => { + stateSyncService.recordMutation(mutation); + }); + // What pro/video/index.ts starts: studio changes become sync mutations. + stopStudioSync = startStudioSync(mutation => stateSyncService.recordMutation(mutation as unknown as SyncMutation)); + installLicensedPhone(mesh, { fingerprint: PHONE_FINGERPRINT }); + mesh.register({ id: PHONE_FINGERPRINT, name: 'This phone', platform: 'ios' }); + (Keychain.setGenericPassword as jest.Mock).mockResolvedValue(true); + }); + + afterEach(async () => { + ui?.unmount(); + ui = undefined; + stopStudioSync?.(); + mesh.restore(); + _clearHooksForTesting(); + await stateSyncService.stop(); + await mac?.engine.stop(); + mac = undefined; + await syncService.stop(); + _clearScreensForTesting(); + _clearSectionsForTesting(); + }); + + it('a film storyboarded on the Mac shows on the phone with its person; edits go back; a delete follows', async () => { + const macDevice: DeviceInfo = { + id: 'studio-mac', name: 'Studio Mac', platform: 'macos', version: '1', host: '127.0.0.1', port: 0, + }; + const macRecords = new MacRecords(); + let opIndex = 0; + const macLog = new OpLog({ + deviceId: macDevice.id, + materializer: macRecords, + uuid: () => `mac-op-${++opIndex}`, + now: () => Date.now(), + }); + let macState: StateSync; + mac = buildSyncEngine({ + pairingEntitlement: mesh.joiner({ name: macDevice.name, platform: macDevice.platform }), + localDevice: macDevice, + tcpModule: TcpSocket as unknown as RnTcpModule, + onPaired: device => macState.onConnect(device.id), + onAppMessage: (deviceId, channel, data) => { + if (channel === 'state') macState.onMessage(deviceId, data); + }, + }); + macState = new StateSync({ + oplog: macLog, + scheduleBackfill: send => send(), + send: (deviceId, message) => { + mac!.engine.sendApp(deviceId, 'state', message); + }, + }); + // What the Mac's studio wrote before pairing, in the shared wire shape (storyFields, characterFields). + const at = Date.parse('2026-10-20T09:00:00.000Z'); + macLog.record('video_character', CHARACTER, 'put', { name: 'Mia', consentAt: at, createdAt: at, updatedAt: at }); + macLog.record('video_story', STORY, 'put', { + title: 'Fox in the snow', + sentence: 'a fox trotting through fresh snow', + format: 'landscape', + characterIds: [CHARACTER], + shots: [ + { id: 'shot-a', index: 0, prompt: 'Wide shot of a snowy field at dawn', narration: 'It begins.', seconds: 5, start: 'cut', characterIds: [CHARACTER], status: 'pending' }, + { id: 'shot-b', index: 1, prompt: 'The fox leaps into the drift', narration: '', seconds: 7, start: 'cut', characterIds: [CHARACTER], status: 'pending' }, + ], + voice: 'default', + captions: true, + quality: 'final', + schedule: 'now', + status: 'storyboard', + createdAt: at, + updatedAt: at, + }); + + await mac.engine.start(0); + macDevice.port = mac.transport.boundPort ?? 0; + await stateSyncService.start(); + await syncService.start(); + + ui = render( + <> + + + + + , + ); + // Settings › Sync, and the Mac pairs with the code on screen. + fireEvent.press(ui.getByTestId('settings-tab')); + await waitFor(() => expect(ui!.getByText('Model Settings')).toBeTruthy()); + fireEvent.press(ui.getByTestId('open-sync-settings')); + await waitFor(() => expect(ui!.getByTestId('sync-pairing-code-value')).toBeTruthy()); + const phone = useSyncStore.getState().thisDevice; + const discovery = getDiscoveryBoundaries().at(-1); + if (!phone || !discovery?.publishedPort) throw new Error('Sync did not publish this phone'); + await mac.engine.pair({ ...phone, host: '127.0.0.1', port: discovery.publishedPort }, await pairingCodeOnScreen(ui)); + + // Home › Image Gallery › Videos: Your videos. + fireEvent.press(ui.getAllByLabelText('Back').pop()!); + fireEvent.press(await waitFor(() => ui!.getByTestId('home-tab'))); + fireEvent.press(await waitFor(() => ui!.getByText('Image Gallery'))); + fireEvent.press(await waitFor(() => ui!.getByText(/^Videos \(/))); + await waitFor(() => expect(ui!.getByTestId('library-grid')).toBeTruthy()); + const film = await waitFor(() => ui!.getByText('Fox in the snow'), { timeout: 15_000 }); + expect(film).toBeTruthy(); + + // Its storyboard: the Mac's shots, and Mia kept in every one. + fireEvent.press(ui.getByTestId(`library-item-${STORY}`)); + await waitFor(() => expect(ui!.getByLabelText('Shot 2, what happens').props.value).toBe('The fox leaps into the drift')); + expect(ui.getByLabelText('Shot 1, narration').props.value).toBe('It begins.'); + expect(ui.getByText('Mia')).toBeTruthy(); + expect(ui.getByLabelText('Captions on')).toBeTruthy(); + + // Captions off on the phone reaches the Mac. + fireEvent.press(ui.getByLabelText('Captions on')); + await waitFor(() => expect(ui!.getByLabelText('Captions off')).toBeTruthy()); + await waitFor(() => expect(macRecords.records.get(`video_story:${STORY}`)?.captions).toBe(false), { timeout: 10_000 }); + + // Deleted on the Mac, it leaves the phone. + macLog.record('video_story', STORY, 'delete'); + macState.sendRecord(phone.id, 'video_story', STORY); + await waitFor(() => expect(ui!.queryByLabelText('Shot 2, what happens')).toBeNull(), { timeout: 10_000 }); + }, 60_000); +}); diff --git a/__tests__/pro/video/animatePhoto.rendered.test.tsx b/__tests__/pro/video/animatePhoto.rendered.test.tsx new file mode 100644 index 000000000..a6ba60e69 --- /dev/null +++ b/__tests__/pro/video/animatePhoto.rendered.test.tsx @@ -0,0 +1,118 @@ +/** + * Animate a picture from a chat. + * + * SPEC (OGAM user's view): I open a picture the app made in a chat; its viewer offers Animate. + * Without a video model that can start from a picture, Animate is on screen but muted, and a tap + * opens one sheet: get the model on my computer, or make a new clip from the picture's description + * instead. Once the computer has the model, Animate opens the Animate screen: the picture, a motion + * (or my own words), where it runs, and Animate — which asks whether the picture is of me or of + * someone who agreed, then shows the clip being made. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the RAM sensor, the OS + * dialog, the image engine (diffusion), and Desktop's gateway at `fetch`. + */ +jest.unmock('@react-navigation/native'); + +import { closeStudioApp, launchStudioApp, type StudioApp } from '../../harness/videoStudioApp'; + +const LIGHT = 'off-grid-ai/Wan2.2-TI2V-5B-GGUF-Q4'; +const PICTURE = 'a lighthouse on a cliff at dusk'; + +afterEach(() => closeStudioApp(), 90_000); + +/** A picture made in a chat by the image model on the phone, opened in its viewer. */ +async function pictureInViewer(): Promise { + const app = await launchStudioApp(undefined, { imageModel: true }); + const { view, rtl } = app; + rtl.fireEvent.press(await view.findByTestId('new-chat-button', {}, { timeout: 5000 })); + await view.findByTestId('chat-input'); + rtl.fireEvent.press(view.getByTestId('quick-settings-button')); + rtl.fireEvent.press(await view.findByTestId('quick-image-mode')); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), PICTURE); + rtl.fireEvent.press(view.getByTestId('send-button')); + rtl.fireEvent.press(await view.findByTestId('generated-image-content', {}, { timeout: 15_000 })); + const animate = await view.findByTestId('animate-photo', {}, { timeout: 10_000 }); + expect(view.queryByTestId('start-picture-sheet')).toBeNull(); + expect(animate.props.accessibilityState?.disabled).toBe(true); + return app; +} + +it('muted Animate offers a new clip from the picture\'s description, made on the computer', async () => { + const app = await pictureInViewer(); + const { view, rtl } = app; + rtl.fireEvent.press(view.getByTestId('animate-photo')); + await view.findByText('Animate needs Wan 2.2 (5B), Light, a 7.9 GB download.', {}, { timeout: 10_000 }); + app.stories.hold(); + rtl.fireEvent.press(view.getAllByTestId('start-picture-describe')[0]); + await view.findByTestId('video-making', {}, { timeout: 10_000 }); + await rtl.waitFor(() => expect(app.stories.submissions).toHaveLength(1), { timeout: 10_000 }); + // Made from the description alone: no picture went to the computer. + const sent = app.stories.submissions[0] as { init_images: Record }; + expect(sent.init_images).toEqual({}); +}, 90_000); + +it('once the computer has the model, Animate opens; my own words and a yes start the clip', async () => { + const app = await pictureInViewer(); + const { view, rtl } = app; + app.models.install(LIGHT); + rtl.fireEvent.press(view.getByTestId('animate-photo')); + rtl.fireEvent.press(await view.findByTestId('start-picture-use', {}, { timeout: 10_000 })); + await view.findByTestId('video-animate', {}, { timeout: 15_000 }); + + expect(view.queryByTestId('motion-words')).toBeNull(); + rtl.fireEvent.press(view.getByTestId('motion-your-words')); + const words = await view.findByTestId('motion-words'); + // My own words, still empty: Animate waits for them. + expect(view.getByTestId('animate-go').props.accessibilityState?.disabled).toBe(true); + rtl.fireEvent.changeText(words, 'The beam sweeps across the sea'); + rtl.fireEvent.press(view.getByTestId('place-computer')); + await rtl.waitFor(() => expect(view.getByTestId('animate-go').props.accessibilityState?.disabled).toBe(false)); + + app.stories.hold(); + rtl.fireEvent.press(view.getByTestId('animate-go')); + await rtl.waitFor(() => expect(app.dialog()?.title).toBe('Is this you, or someone who agreed?'), { timeout: 10_000 }); + expect(app.stories.submissions).toHaveLength(0); + await app.answer('Yes'); + await view.findByTestId('video-making', {}, { timeout: 10_000 }); + await rtl.waitFor(() => expect(app.stories.submissions).toHaveLength(1), { timeout: 10_000 }); + const sent = app.stories.submissions[0] as { + consent: boolean; + init_images: Record; + story: { shots: Array<{ prompt: string }> }; + }; + expect(sent.consent).toBe(true); + expect(Object.values(sent.init_images)).toHaveLength(1); + expect(sent.story.shots[0].prompt).toBe('The beam sweeps across the sea'); +}, 90_000); + +it('a No to the photo question starts nothing', async () => { + const app = await pictureInViewer(); + const { view, rtl } = app; + app.models.install(LIGHT); + rtl.fireEvent.press(view.getByTestId('animate-photo')); + rtl.fireEvent.press(await view.findByTestId('start-picture-use', {}, { timeout: 10_000 })); + await view.findByTestId('video-animate', {}, { timeout: 15_000 }); + rtl.fireEvent.press(view.getByTestId('motion-camera-orbit')); + rtl.fireEvent.press(view.getByTestId('animate-go')); + await rtl.waitFor(() => expect(app.dialog()?.title).toBe('Is this you, or someone who agreed?'), { timeout: 10_000 }); + await app.answer('Cancel'); + await rtl.waitFor(() => expect(view.getByTestId('animate-go').props.accessibilityState?.busy).toBeFalsy()); + expect(view.queryByTestId('video-making')).toBeNull(); + expect(app.stories.submissions).toHaveLength(0); + rtl.fireEvent.press(view.getByTestId('animate-close')); + await rtl.waitFor(() => expect(view.queryByTestId('video-animate')).toBeNull()); +}, 90_000); + +it('the Gallery\'s full-screen picture offers the same muted Animate and sheet', async () => { + const app = await pictureInViewer(); + const { view, rtl } = app; + // Leave the chat's viewer and the chat, then open the picture from the Gallery. + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + rtl.fireEvent.press(await view.findByTestId('home-tab')); + rtl.fireEvent.press(await view.findByText('Image Gallery', {}, { timeout: 10_000 })); + rtl.fireEvent.press((await view.findAllByTestId(/^gallery-image-/, {}, { timeout: 10_000 }))[0]); + const animate = (await view.findAllByTestId('animate-photo', {}, { timeout: 10_000 })).pop()!; + expect(animate.props.accessibilityState?.disabled).toBe(true); + rtl.fireEvent.press(animate); + await view.findByText('Animate needs Wan 2.2 (5B), Light, a 7.9 GB download.', {}, { timeout: 10_000 }); +}, 90_000); diff --git a/__tests__/pro/video/characterInEveryShot.rendered.test.tsx b/__tests__/pro/video/characterInEveryShot.rendered.test.tsx new file mode 100644 index 000000000..a2ea58262 --- /dev/null +++ b/__tests__/pro/video/characterInEveryShot.rendered.test.tsx @@ -0,0 +1,105 @@ +/** + * Keep the same person in every shot of a film. + * + * SPEC (OGAM user's view): on a storyboard I tap "Keep this person in every shot?", name the person, + * add their photos from my library and confirm the photos are of me or of people who agreed. Keep + * is off until all three are done. Back on the storyboard the person is named; when the film is made + * on my computer it carries their photos. I can stop keeping them. On my next film the person I + * already made is offered with one tap. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the photo library + * (react-native-image-picker, faked in jest.setup), the RAM sensor and Desktop's gateway at `fetch`. + */ +jest.unmock('@react-navigation/native'); + +import { closeStudioApp, launchStudioApp, makeClipInChat, type StudioApp } from '../../harness/videoStudioApp'; + +afterEach(() => closeStudioApp(), 90_000); + +/** The photo the library hands back (jest.setup's picker answers file:///mock/image.jpg). */ +const LIBRARY_PHOTO = '/mock/image.jpg'; + +async function storyboard(app: StudioApp): Promise { + const { view, rtl } = app; + const chips = view.getAllByTestId('video-make-story'); + rtl.fireEvent.press(chips[chips.length - 1]); + await view.findByTestId('film-make-real', {}, { timeout: 15_000 }); +} + +async function makeCharacter(app: StudioApp, name: string): Promise { + const { view, rtl } = app; + rtl.fireEvent.press(view.getByTestId('add-character')); + const save = await view.findByTestId('character-save', {}, { timeout: 10_000 }); + expect(save.props.accessibilityState?.disabled).toBe(true); + rtl.fireEvent.changeText(view.getByTestId('character-name'), name); + rtl.fireEvent.press(view.getByTestId('character-add-photos')); + await view.findByLabelText('Remove photo'); + // A name and a photo are not enough: the consent switch is still off. + expect(view.getByTestId('character-save').props.accessibilityState?.disabled).toBe(true); + rtl.fireEvent(view.getByTestId('character-consent'), 'valueChange', true); + await rtl.waitFor(() => expect(view.getByTestId('character-save').props.accessibilityState?.disabled).toBe(false)); + rtl.fireEvent.press(view.getByTestId('character-save')); + await view.findByLabelText(`Stop keeping ${name}`, {}, { timeout: 10_000 }); +} + +it('keeps a named person with their photos in the film sent to the computer, and lets it go', async () => { + const app = await launchStudioApp(); + const { view, rtl, boundary } = app; + boundary.fs!.seedFile(LIBRARY_PHOTO, 2048); + await makeClipInChat(app, 'a girl flying a kite on a windy beach'); + await storyboard(app); + + // A photo removed before saving leaves Keep off again. + rtl.fireEvent.press(view.getByTestId('add-character')); + await view.findByTestId('character-save', {}, { timeout: 10_000 }); + rtl.fireEvent.changeText(view.getByTestId('character-name'), 'Mia'); + rtl.fireEvent.press(view.getByTestId('character-add-photos')); + rtl.fireEvent.press(await view.findByLabelText('Remove photo')); + await rtl.waitFor(() => expect(view.queryByLabelText('Remove photo')).toBeNull()); + rtl.fireEvent(view.getByTestId('character-consent'), 'valueChange', true); + expect(view.getByTestId('character-save').props.accessibilityState?.disabled).toBe(true); + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + await view.findByTestId('add-character'); + + await makeCharacter(app, 'Mia'); + expect(view.queryByTestId('add-character')).toBeNull(); + expect(view.getByText('Mia')).toBeTruthy(); + + // The film made on the computer carries Mia and her photo, with her consent. + app.stories.hold(); + rtl.fireEvent.press(view.getByTestId('film-make-real')); + await rtl.waitFor(() => expect(app.stories.submissions).toHaveLength(1), { timeout: 10_000 }); + const sent = app.stories.submissions[0] as { + characters: Array<{ name: string; photos: unknown[]; consentAt?: number }>; + photos: Record; + }; + expect(sent.characters.map(person => person.name)).toEqual(['Mia']); + expect(sent.characters[0].photos).toHaveLength(1); + expect(sent.characters[0].consentAt).toBeGreaterThan(0); + expect(Object.values(sent.photos)[0]).toMatch(/^data:image\/jpeg;base64,/); + + // Stopped, the film is a draft again and Mia can be let go. + rtl.fireEvent.press(await view.findByTestId('making-stop', {}, { timeout: 10_000 })); + await view.findByTestId('making-retry', {}, { timeout: 10_000 }); + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + rtl.fireEvent.press(await view.findByLabelText('Stop keeping Mia')); + await view.findByTestId('add-character'); + expect(view.queryByText('Mia')).toBeNull(); +}, 90_000); + +it('offers the person already made on the next film', async () => { + const app = await launchStudioApp(); + const { view, rtl, boundary } = app; + boundary.fs!.seedFile(LIBRARY_PHOTO, 2048); + await makeClipInChat(app, 'a boy skating down a quiet street'); + await storyboard(app); + await makeCharacter(app, 'Leo'); + + // Back to the chat, and a second film from the same clip. + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + await storyboard(app); + const offer = await view.findByTestId('add-character', {}, { timeout: 10_000 }); + rtl.fireEvent.press(offer); + rtl.fireEvent.press(await view.findByText('Leo', {}, { timeout: 10_000 })); + await view.findByLabelText('Stop keeping Leo', {}, { timeout: 10_000 }); +}, 90_000); diff --git a/__tests__/pro/video/clipOnComputerPreview.rendered.test.tsx b/__tests__/pro/video/clipOnComputerPreview.rendered.test.tsx new file mode 100644 index 000000000..d8648c949 --- /dev/null +++ b/__tests__/pro/video/clipOnComputerPreview.rendered.test.tsx @@ -0,0 +1,30 @@ +/** + * A clip made on the paired computer shows its first frame while it is made. + * + * SPEC (OGAM user's view): while my computer makes a clip I described in a chat, the chat shows its + * progress and, once the computer has one, a live preview of the clip; when the clip lands the + * preview gives way to it. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the RAM sensor and + * Desktop's gateway at `fetch` (its poll's `preview` and the job's /preview PNG, as Desktop serves them). + */ +jest.unmock('@react-navigation/native'); + +import { closeStudioApp, launchStudioApp, makeClipInChat } from '../../harness/videoStudioApp'; + +afterEach(() => closeStudioApp(), 90_000); + +it('shows the computer\'s live preview while the clip is made, then the clip', async () => { + const app = await launchStudioApp(); + const { view, rtl } = app; + app.jobs.showPreviews(); + app.jobs.hold(); + const making = makeClipInChat(app, 'a lighthouse beam sweeping the sea'); + await view.findByLabelText('Live preview of the clip', {}, { timeout: 15_000 }); + expect(view.getByTestId('video-progress')).toBeTruthy(); + expect(view.queryByTestId('chat-clip-card')).toBeNull(); + app.jobs.release(); + await making; + await rtl.waitFor(() => expect(view.queryByLabelText('Live preview of the clip')).toBeNull()); + expect(view.getByTestId('chat-clip-card')).toBeTruthy(); +}, 60_000); diff --git a/__tests__/pro/video/continueFromClip.rendered.test.tsx b/__tests__/pro/video/continueFromClip.rendered.test.tsx new file mode 100644 index 000000000..697f9400a --- /dev/null +++ b/__tests__/pro/video/continueFromClip.rendered.test.tsx @@ -0,0 +1,48 @@ +/** + * Continue a clip from where it ended, on the paired computer. + * + * SPEC (OGAM user's view): under a clip in chat, Continue makes the next clip start from this one's + * last frame. Once my computer has a model that can start from a picture, Continue sends that frame + * (the app's own picture, not a photo of a person) with a description that carries the action on, + * and the next clip lands in the chat. + * + * The real App runs, as a phone build with its film composer (which reads a clip's last frame). + * Device boundaries only: the llama.rn engine, the disk, the RAM sensor, the native film composer + * and Desktop's gateway at `fetch`. + */ +jest.unmock('@react-navigation/native'); + +import { closeStudioApp, launchStudioApp, makeClipInChat } from '../../harness/videoStudioApp'; + +afterEach(() => closeStudioApp(), 90_000); + +const LIGHT = 'off-grid-ai/Wan2.2-TI2V-5B-GGUF-Q4'; +const CLIP = 'a red fox trotting through fresh snow'; + +it('Continue sends the clip\'s last frame to the computer, and the next clip lands', async () => { + const app = await launchStudioApp(undefined, { phoneEngine: true }); + const { view, rtl } = app; + await makeClipInChat(app, CLIP); + app.models.install(LIGHT); + rtl.fireEvent.press(await view.findByTestId('video-continue')); + app.stories.hold(); + rtl.fireEvent.press(await view.findByTestId('start-picture-use', {}, { timeout: 10_000 })); + await view.findByText('Wan 2.2 (5B), Light is ready.', {}, { timeout: 15_000 }); + // Ready: Continue carries on by itself with the frame. + await rtl.waitFor(() => expect(app.stories.submissions).toHaveLength(1), { timeout: 15_000 }); + const sent = app.stories.submissions[0] as { + consent: boolean; + init_images: Record; + story: { shots: Array<{ prompt: string; start: string; initImageSource?: string }> }; + }; + expect(Object.values(sent.init_images)).toHaveLength(1); + expect(Object.values(sent.init_images)[0]).toMatch(/^data:image\/png;base64,/); + expect(sent.story.shots[0]).toMatchObject({ start: 'continue', initImageSource: 'generated' }); + expect(sent.story.shots[0].prompt).toContain(CLIP); + + rtl.fireEvent.press(view.getByTestId('start-picture-done')); + app.stories.release(); + // The next clip is ready in the chat, with its own Continue under it. + expect(view.getAllByTestId('video-continue')).toHaveLength(1); + await rtl.waitFor(() => expect(view.getAllByTestId('video-continue')).toHaveLength(2), { timeout: 30_000 }); +}, 90_000); diff --git a/__tests__/pro/video/filmOnComputer.rendered.test.tsx b/__tests__/pro/video/filmOnComputer.rendered.test.tsx new file mode 100644 index 000000000..136b798a4 --- /dev/null +++ b/__tests__/pro/video/filmOnComputer.rendered.test.tsx @@ -0,0 +1,93 @@ +/** + * A film made on the paired computer, from a clip in chat. + * + * SPEC (OGAM user's view): under a clip I tap Make it a story, then Make it real. A Making screen + * shows the film being made on my computer — how far it is, the time left, and Stop — and I can go + * back to the storyboard, which shows the same progress with one tile per shot. When the computer + * finishes, the film comes to the phone: the Making screen says how long it is and offers Watch. + * If I stop it, the computer is told and I can try again; if the computer fails, I see why. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the RAM sensor and + * Desktop's gateway at `fetch` (its video jobs, model manager and `/v1/stories` film jobs, in + * Desktop's own wire shapes). + */ +jest.unmock('@react-navigation/native'); + +import { closeStudioApp, launchStudioApp, makeClipInChat, type StudioApp } from '../../harness/videoStudioApp'; + +afterEach(() => closeStudioApp(), 90_000); + +const SENTENCE = 'a red fox trotting through fresh snow'; + +/** From the clip in chat to the storyboard, then Make it real, with the computer holding the film. */ +async function filmUnderway(): Promise { + const app = await launchStudioApp(); + const { view, rtl } = app; + await makeClipInChat(app, SENTENCE); + rtl.fireEvent.press(await view.findByTestId('video-make-story')); + const makeReal = await view.findByTestId('film-make-real', {}, { timeout: 15_000 }); + expect(view.queryByTestId('video-making')).toBeNull(); + app.stories.hold(); + rtl.fireEvent.press(makeReal); + await view.findByTestId('video-making', {}, { timeout: 10_000 }); + await rtl.waitFor(() => expect(app.stories.submissions).toHaveLength(1), { timeout: 10_000 }); + return app; +} + +it('shows the film being made on the computer, then plays it when it comes to the phone', async () => { + const app = await filmUnderway(); + const { view, rtl } = app; + // In flight: progress, where it runs, and Stop — no Watch yet. + expect(view.getByTestId('making-progress')).toBeTruthy(); + expect(view.getByTestId('making-stop')).toBeTruthy(); + expect(view.queryByTestId('making-watch')).toBeNull(); + + // The storyboard underneath shows the same film in progress, one tile per shot. + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + await view.findByTestId('film-progress'); + expect(view.getByTestId('shot-strip')).toBeTruthy(); + expect(view.getByLabelText('On your computer')).toBeTruthy(); + expect(view.getAllByLabelText(/^Shot 1, /).length).toBeGreaterThan(0); + + app.stories.release(); + // The film arrives: the storyboard plays it and the progress is gone. + await view.findByTestId('film-player', {}, { timeout: 30_000 }); + expect(view.queryByTestId('film-progress')).toBeNull(); +}, 60_000); + +it('Stop on the Making screen stops the film on the computer and offers Try again', async () => { + const app = await filmUnderway(); + const { view, rtl } = app; + expect(view.queryByTestId('making-retry')).toBeNull(); + rtl.fireEvent.press(view.getByTestId('making-stop')); + await view.findByTestId('making-retry', {}, { timeout: 10_000 }); + expect(view.queryByTestId('making-stop')).toBeNull(); + expect(app.stories.cancels).toHaveLength(1); +}, 60_000); + +it('a film the computer cannot make says why, and Try again sends it again', async () => { + const app = await filmUnderway(); + const { view, rtl } = app; + app.stories.failNext('Not enough memory for this film.'); + app.stories.release(); + await view.findByText('Not enough memory for this film.', {}, { timeout: 30_000 }); + rtl.fireEvent.press(view.getByTestId('making-retry')); + await rtl.waitFor(() => expect(app.stories.submissions).toHaveLength(2), { timeout: 10_000 }); + await view.findByTestId('making-watch', {}, { timeout: 30_000 }); +}, 90_000); + +it('the finished film plays under its title, made on the computer, and Delete there removes it', async () => { + const app = await filmUnderway(); + const { view, rtl } = app; + app.stories.release(); + rtl.fireEvent.press(await view.findByTestId('making-watch', {}, { timeout: 30_000 })); + await view.findByTestId('video-viewer', {}, { timeout: 10_000 }); + expect(view.getByText(/^Made with AI · on /)).toBeTruthy(); + rtl.fireEvent.press(view.getByTestId('viewer-delete')); + expect(app.dialog()?.title).toBe('Delete this video?'); + await app.answer('Delete'); + await rtl.waitFor(() => expect(view.queryByTestId('video-viewer')).toBeNull(), { timeout: 10_000 }); + // Back on Making, the film and its story are gone. + await view.findByText('This video was deleted.', {}, { timeout: 10_000 }); + expect(view.queryByTestId('making-watch')).toBeNull(); +}, 90_000); diff --git a/__tests__/pro/video/offGridMark.rendered.test.tsx b/__tests__/pro/video/offGridMark.rendered.test.tsx new file mode 100644 index 000000000..e4c131074 --- /dev/null +++ b/__tests__/pro/video/offGridMark.rendered.test.tsx @@ -0,0 +1,51 @@ +/** + * A model Off Grid AI made carries the app's mark beside its name. + * + * SPEC (OGAM user's view): on Models, Video, every build from Off Grid AI's own org shows the small + * Off Grid AI logo ("Made by Off Grid AI" to a screen reader), with no words added to the row. A build + * another uploader made does not show it. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk and the RAM sensor + * (a 12 GB Android phone). The person opens Models and its Video tab. + */ +import { launchApp, passOnboarding, restoreBuildFlavor, type AnalyticsApp } from '../../harness/analyticsApp'; + +let app: AnalyticsApp | undefined; + +afterEach(() => { + app?.view.unmount(); + app?.endpoint.restore(); + app = undefined; + restoreBuildFlavor(); +}); + +type Node = { parent: Node | null }; + +/** The model card around a name: six levels up from the name is its card, one card and no more. */ +function cardOf(current: AnalyticsApp, name: string): Node { + const { view, rtl } = current; + let node = view.getByText(name) as unknown as Node; + for (let depth = 0; depth < 6 && node.parent; depth += 1) node = node.parent; + return rtl.within(node as never) as unknown as Node; +} + +it('Off Grid AI builds carry the Off Grid AI mark on Models, Video; other uploads do not', async () => { + app = await launchApp({ release: false }); + const { view, rtl } = app; + await passOnboarding(app, false); + await require('../../harness/proHarness').installPro(); + rtl.fireEvent.press(await view.findByTestId('models-tab')); + await view.findByTestId('models-screen'); + expect(view.queryByTestId('made-by-off-grid')).toBeNull(); + rtl.fireEvent.press(view.getByText('Video')); + const light = await view.findByText('Wan 2.2 (5B), Light', {}, { timeout: 10_000 }); + expect(light).toBeTruthy(); + const marks = await view.findAllByLabelText('Made by Off Grid AI'); + expect(marks.length).toBeGreaterThan(0); + const within = cardOf(app, 'Wan 2.2 (5B), Light') as unknown as ReturnType; + expect(within.getByTestId('made-by-off-grid')).toBeTruthy(); + // The row says nothing more: the mark is the only sign of who made it. + expect(view.queryByText('Off Grid AI')).toBeNull(); + const other = cardOf(app, 'Wan 2.2 (5B), Q5_K_M') as unknown as ReturnType; + expect(other.queryByTestId('made-by-off-grid')).toBeNull(); +}, 60_000); diff --git a/__tests__/pro/video/queuedVideoIsItsOwnTurn.rendered.test.tsx b/__tests__/pro/video/queuedVideoIsItsOwnTurn.rendered.test.tsx new file mode 100644 index 000000000..a7b139598 --- /dev/null +++ b/__tests__/pro/video/queuedVideoIsItsOwnTurn.rendered.test.tsx @@ -0,0 +1,152 @@ +/** + * A queued video prompt is always its own turn; queued text follows the shared queue rule. + * + * SPEC (the user's view): while a reply is running I send a question, then switch to video and + * describe a clip, then switch back and send another question. When the reply ends, my two + * questions are answered together (the shared queue rule, as on Off Grid Desktop), and the clip is + * made from my description alone — never from my questions mixed into it. + * + * Real ChatScreen, generation service, queue and Pro's video service. Fakes only at the boundaries: the + * llama.rn engine (held so the first reply stays running) and Off Grid Desktop's video gateway at + * fetch, whose saved server record is what pairing leaves behind. + */ +import React from 'react'; +import { setupChatScreen } from '../../harness/chatHarness'; +import { + DESKTOP_VIDEO_ENDPOINT, + DESKTOP_VIDEO_MODEL, + serveDesktopVideoJobs, + type DesktopVideoJobs, +} from '../../harness/desktopVideoServer'; + +jest.mock('@react-navigation/native', () => ({ + useNavigation: () => ({ navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {} }), + useRoute: () => require('../../harness/chatHarness').routeHolder, + useFocusEffect: () => {}, useIsFocused: () => true, +})); + +const CLIP = 'a paper boat on a pond'; +const COMBINED_REPLY = 'Both questions, answered together.'; + +let desktop: DesktopVideoJobs | undefined; + +afterEach(() => { + desktop?.restore(); + desktop = undefined; + require('../../../src/stores/remoteServerStore').useRemoteServerStore.getState().clearAllServers(); +}); + +it('queued questions go together and a queued video prompt goes alone', async () => { + const h = await setupChatScreen({ engine: 'llama', platform: 'android' }); + const { rtl } = h; + desktop = serveDesktopVideoJobs(h.boundary.fs!); + + // Desktop's saved record, then its video model picked in the real remote-model picker. + const { remoteServerManager } = require('../../../src/services/remoteServerManager'); + const { RemoteModelOptionsSection } = require('../../../src/components/models/RemoteModelOptionsSection'); + const server = await remoteServerManager.addServer({ + name: 'Studio Mac', + endpoint: DESKTOP_VIDEO_ENDPOINT, + providerType: 'openai-compatible', + modelManagement: 'offgrid-desktop-v1', + modelCatalog: { video: [DESKTOP_VIDEO_MODEL] }, + }); + const rowId = `remote-video-model-${server.id}:${DESKTOP_VIDEO_MODEL.id}`; + const picker = rtl.render(React.createElement(RemoteModelOptionsSection, { category: 'video' })); + rtl.fireEvent.press(picker.getByTestId(rowId)); + await rtl.waitFor(() => expect(picker.getByTestId(rowId).props.accessibilityState?.selected).toBe(true)); + picker.unmount(); + + // Video is Pro: the app activates it before the chat opens. + await require('../../harness/proHarness').installPro(); + h.render(); + const view = h.view!; + const toggleVideoMode = async () => { + rtl.fireEvent.press(await rtl.waitFor(() => view.getByTestId('quick-settings-button'))); + rtl.fireEvent.press(await rtl.waitFor(() => view.getByTestId('quick-video-mode'))); + }; + + // The first reply holds, so everything sent next queues behind it. + h.boundary.llama!.scriptCompletion({ text: 'First reply.', holdBeforeStream: true }); + await h.tapSend('what is the weather like'); + await rtl.waitFor(() => expect(view.queryByTestId('stop-button')).not.toBeNull(), { timeout: 4000 }); + + await h.tapSend('how deep is the pond'); + await h.settle(30); + await toggleVideoMode(); + await h.tapSend(CLIP); + await h.settle(30); + // Video mode is for one message: the input is back to text after the clip is sent. + await h.tapSend('and how cold is it'); + await h.settle(50); + // Nothing has left the queue while the first reply holds. + expect(desktop.submissions).toHaveLength(0); + expect(view.queryByText(COMBINED_REPLY)).toBeNull(); + + h.boundary.llama!.scriptCompletion({ text: COMBINED_REPLY }); + h.boundary.llama!.releaseStream(); + + // The two questions are answered in one reply: one text request carries both, and not the clip. + await rtl.waitFor(() => expect(view.getAllByText(COMBINED_REPLY)).toHaveLength(1), { timeout: 8000 }); + const lastUserTurn = (call: unknown[]) => { + const messages = (call[0] as { messages?: Array<{ role: string; content?: unknown }> }).messages ?? []; + const user = messages.filter(message => message.role === 'user').pop(); + return typeof user?.content === 'string' ? user.content : JSON.stringify(user?.content ?? ''); + }; + const textTurns = h.boundary.llama!.calls.completion.map(lastUserTurn); + const combined = textTurns.filter(turn => turn.includes('how deep is the pond')); + expect(combined).toHaveLength(1); + expect(combined[0]).toContain('and how cold is it'); + expect(combined[0]).not.toContain(CLIP); + // The clip is made from the description alone. + await rtl.waitFor(() => expect(desktop!.submissions).toHaveLength(1), { timeout: 8000 }); + expect(desktop.submissions[0].prompt).toBe(CLIP); + // The clip finishes before the test ends, so no poll outlives the fake Desktop. + await rtl.waitFor(() => expect(view.getByTestId('chat-clip-card')).toBeTruthy(), { timeout: 8000 }); + expect(require('../../../src/bootstrap/videoSeam').videoSeam.hasPending()).toBe(false); +}); + +it('Send now on a queued question stops the running video and sends the question', async () => { + const h = await setupChatScreen({ engine: 'llama', platform: 'android' }); + const { rtl } = h; + desktop = serveDesktopVideoJobs(h.boundary.fs!); + const { remoteServerManager } = require('../../../src/services/remoteServerManager'); + const { RemoteModelOptionsSection } = require('../../../src/components/models/RemoteModelOptionsSection'); + const server = await remoteServerManager.addServer({ + name: 'Studio Mac', + endpoint: DESKTOP_VIDEO_ENDPOINT, + providerType: 'openai-compatible', + modelManagement: 'offgrid-desktop-v1', + modelCatalog: { video: [DESKTOP_VIDEO_MODEL] }, + }); + const rowId = `remote-video-model-${server.id}:${DESKTOP_VIDEO_MODEL.id}`; + const picker = rtl.render(React.createElement(RemoteModelOptionsSection, { category: 'video' })); + rtl.fireEvent.press(picker.getByTestId(rowId)); + await rtl.waitFor(() => expect(picker.getByTestId(rowId).props.accessibilityState?.selected).toBe(true)); + picker.unmount(); + await require('../../harness/proHarness').installPro(); + h.render(); + const view = h.view!; + const toggleVideoMode = async () => { + rtl.fireEvent.press(await rtl.waitFor(() => view.getByTestId('quick-settings-button'))); + rtl.fireEvent.press(await rtl.waitFor(() => view.getByTestId('quick-video-mode'))); + }; + + // A clip is being made on the computer, and it keeps running. + desktop.hold(); + await toggleVideoMode(); + await h.tapSend(CLIP); + await rtl.waitFor(() => expect(desktop!.submissions).toHaveLength(1), { timeout: 8000 }); + // A question sent meanwhile (video mode is for one message) waits behind the clip. + h.boundary.llama!.scriptCompletion({ text: 'About two metres.' }); + await h.tapSend('how deep is the pond'); + await rtl.waitFor(() => expect(view.getByTestId('queued-message-0')).toBeTruthy(), { timeout: 4000 }); + expect(view.queryByText('About two metres.')).toBeNull(); + + rtl.fireEvent.press(view.getByTestId('queued-send-now-0')); + + // The clip is stopped on the computer, and the question is answered now. + await rtl.waitFor(() => expect(desktop!.cancels).toEqual([String(desktop!.submissions[0].client_job_id)]), { timeout: 8000 }); + expect(await view.findByText('About two metres.', {}, { timeout: 8000 })).toBeTruthy(); + await rtl.waitFor(() => expect(require('../../../src/bootstrap/videoSeam').videoSeam.hasPending()).toBe(false)); +}, 60_000); diff --git a/__tests__/pro/video/remoteVideoAnalytics.rendered.test.tsx b/__tests__/pro/video/remoteVideoAnalytics.rendered.test.tsx new file mode 100644 index 000000000..450acd3f6 --- /dev/null +++ b/__tests__/pro/video/remoteVideoAnalytics.rendered.test.tsx @@ -0,0 +1,100 @@ +/** + * Video in opt-in analytics (docs/ANALYTICS.md: Generation, Actions). + * + * SPEC (OGAM user's view): a person who shares usage and makes a video in a chat on their Off Grid + * Desktop sends one `generation_finished` with modality `video` (engine and backend `remote`, an + * outcome and a duration) and `feature_used` `video_generation`. The prompt never leaves the phone + * for PostHog. The video itself appears in the chat. + * + * The real App runs as a store build. Device boundaries only: the llama.rn engine, the disk, the + * RAM sensor, Desktop's gateway and the PostHog endpoint at `fetch`. Desktop's saved server record + * is what pairing leaves behind; its video model is picked in the real remote-model picker. + */ +import { + launchApp, + passOnboarding, + restoreBuildFlavor, + sendToBackground, + type AnalyticsApp, +} from '../../harness/analyticsApp'; +import { + DESKTOP_VIDEO_ENDPOINT, + DESKTOP_VIDEO_MODEL, + serveDesktopVideoJobs, + type DesktopVideoJobs, +} from '../../harness/desktopVideoServer'; + +const PROMPT = 'a paper boat drifting down a rainy street'; + +let app: AnalyticsApp | undefined; +let desktop: DesktopVideoJobs | undefined; + +afterEach(() => { + app?.view.unmount(); + app?.endpoint.restore(); + desktop?.restore(); + require('../../../src/stores/remoteServerStore').useRemoteServerStore.getState().clearAllServers(); + app = undefined; + desktop = undefined; + restoreBuildFlavor(); +}); + +/** + * The paired Desktop's saved record is what pairing leaves behind (as in remoteImageServer). The + * person then picks its video model from Home's Video row; Desktop activates it. + */ +async function chooseDesktopVideoModel(current: AnalyticsApp): Promise { + const { view, rtl } = current; + const { remoteServerManager } = require('../../../src/services/remoteServerManager'); + const server = await remoteServerManager.addServer({ + name: 'Studio Mac', + endpoint: DESKTOP_VIDEO_ENDPOINT, + providerType: 'openai-compatible', + modelManagement: 'offgrid-desktop-v1', + modelCatalog: { video: [DESKTOP_VIDEO_MODEL] }, + }); + expect(view.queryByTestId('new-chat-button')).toBeNull(); + rtl.fireEvent.press(await view.findByTestId('model-summary-video-open')); + rtl.fireEvent.press(await view.findByTestId(`remote-video-model-${server.id}:${DESKTOP_VIDEO_MODEL.id}`, {}, { timeout: 5000 })); + await view.findByTestId('new-chat-button', {}, { timeout: 5000 }); +} + +it('a Desktop video in a chat sends a video generation_finished, never the prompt', async () => { + app = await launchApp({ release: true }); + const current = app; + const { view, rtl, endpoint, boundary } = current; + desktop = serveDesktopVideoJobs(boundary.fs!); + await passOnboarding(current, true); + // Video is Pro: the person activates it, as with a key. + await require('../../harness/proHarness').installPro(); + await chooseDesktopVideoModel(current); + // A video model alone is enough to start a chat. + rtl.fireEvent.press(await view.findByTestId('new-chat-button')); + await view.findByTestId('chat-input'); + + rtl.fireEvent.press(view.getByTestId('quick-settings-button')); + const videoMode = await view.findByTestId('quick-video-mode'); + expect(videoMode.props.accessibilityState?.selected).toBe(false); + rtl.fireEvent.press(videoMode); + await rtl.waitFor(() => + expect(view.getByTestId('quick-video-mode').props.accessibilityState?.selected).toBe(true), + ); + + expect(view.queryByTestId('message-bubble')).toBeNull(); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), PROMPT); + rtl.fireEvent.press(view.getByTestId('send-button')); + // The prompt reached Desktop, and the finished clip shows in the chat. + await rtl.waitFor(() => expect(desktop!.submissions).toHaveLength(1), { timeout: 10_000 }); + expect(desktop!.submissions[0].prompt).toBe(PROMPT); + // A finished clip shows as its clip card in the reply (this release's chat clip card). + expect(view.queryByTestId('chat-clip-card')).toBeNull(); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); + + await sendToBackground(current); + await rtl.waitFor(() => expect(endpoint.named('generation_finished')).toHaveLength(1), { timeout: 10_000 }); + const finished = endpoint.named('generation_finished')[0].properties; + expect(finished).toMatchObject({ modality: 'video', engine: 'remote', backend: 'remote', model: 'custom', outcome: 'done' }); + expect(typeof finished.duration_ms).toBe('number'); + expect(endpoint.named('feature_used').map(e => e.properties.feature)).toContain('video_generation'); + for (const body of endpoint.bodies) expect(body).not.toContain('paper boat'); +}); diff --git a/__tests__/pro/video/startPictureOffer.test.ts b/__tests__/pro/video/startPictureOffer.test.ts new file mode 100644 index 000000000..4c6b9065d --- /dev/null +++ b/__tests__/pro/video/startPictureOffer.test.ts @@ -0,0 +1,58 @@ +/** + * The start-picture choice, judged over the REAL shared catalog: what a phone offers when a video should start + * from a picture, given what the paired computer has. No mocks. + */ +import { chooseStartPicture, startPictureOffer } from '@offgrid/pro/video/studio/startPictureOffer'; + +const LIGHT = 'off-grid-ai/Wan2.2-TI2V-5B-GGUF-Q4'; +const FULL = 'off-grid-ai/Wan2.2-TI2V-5B-GGUF'; +const none = new Set(); + +describe('chooseStartPicture', () => { + it('uses a start-picture model the computer already has', () => { + expect(chooseStartPicture({ memoryGb: 16, installed: new Set([LIGHT]), downloading: none })).toMatchObject({ + kind: 'installed', + offer: { id: LIGHT }, + }); + }); + + it('follows a download already running on the computer, never a second one', () => { + expect(chooseStartPicture({ memoryGb: 64, installed: none, downloading: new Set([FULL]) })).toMatchObject({ + kind: 'downloading', + offer: { id: FULL }, + }); + }); + + it('resumes a download that stopped part way', () => { + expect( + chooseStartPicture({ memoryGb: 34, installed: none, downloading: none, stopped: new Map([[LIGHT, 35]]) }), + ).toEqual(expect.objectContaining({ kind: 'stopped', offer: expect.objectContaining({ id: LIGHT }), percent: 35 })); + }); + + it('offers the Light build when the computer fits it without clear room for Full quality', () => { + expect(chooseStartPicture({ memoryGb: 34, installed: none, downloading: none })).toMatchObject({ + kind: 'download', + offer: { id: LIGHT, minRamGb: 24 }, + }); + }); + + it('offers Full quality only with clear room', () => { + expect(chooseStartPicture({ memoryGb: 64, installed: none, downloading: none })).toMatchObject({ + kind: 'download', + offer: { id: FULL }, + }); + }); + + it('says the computer is too small when nothing fits', () => { + expect(chooseStartPicture({ memoryGb: 16, installed: none, downloading: none })).toEqual({ + kind: 'tooSmall', + memoryGb: 16, + needsGb: 24, + }); + }); + + it('offers the Light build to an older Desktop that does not report its memory', () => { + expect(chooseStartPicture({ installed: none, downloading: none })).toMatchObject({ kind: 'download', offer: { id: LIGHT } }); + expect(startPictureOffer()).toMatchObject({ id: LIGHT }); + }); +}); diff --git a/__tests__/pro/video/startPictureSheet.rendered.test.tsx b/__tests__/pro/video/startPictureSheet.rendered.test.tsx new file mode 100644 index 000000000..8b549fe6c --- /dev/null +++ b/__tests__/pro/video/startPictureSheet.rendered.test.tsx @@ -0,0 +1,193 @@ +/** + * Continue from a clip when the paired computer has no model that can start from a picture. + * + * SPEC (OGAM user's view): Continue stays on screen, muted. A tap opens one sheet that asks the + * computer what it has and gives one next step: + * - a start-picture build downloading there: its progress, never a second download; when it lands + * the computer switches to it and Continue turns on; + * - a build already there: Use it; + * - otherwise the Light build that fits the computer (Full quality only with clear room), with + * "Make a new clip from a description" beside it; an older Desktop that does not report its memory + * is offered the Light build too; + * - a computer too small for any: how much memory it needs and has, and a new clip from a description. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the RAM sensor and + * Desktop's gateway at `fetch` (its video jobs and its model manager). The clip is made the way a + * person makes one: video mode in a chat, then Send. + */ +// The real navigator: studio screens open from chat (the default setup mocks useNavigation to a no-op). +jest.unmock('@react-navigation/native'); + +import { + launchApp, + passOnboarding, + restoreBuildFlavor, + type AnalyticsApp, +} from '../../harness/analyticsApp'; +import { + DESKTOP_VIDEO_ENDPOINT, + DESKTOP_VIDEO_MODEL, + serveDesktopVideoJobs, + type DesktopVideoJobs, +} from '../../harness/desktopVideoServer'; +import { serveDesktopModelPulls, type DesktopModelPulls } from '../../harness/desktopModelPulls'; + +const LIGHT = 'off-grid-ai/Wan2.2-TI2V-5B-GGUF-Q4'; + +let app: AnalyticsApp | undefined; +let jobs: DesktopVideoJobs | undefined; +let models: DesktopModelPulls | undefined; + +afterEach(() => { + app?.view.unmount(); + app?.endpoint.restore(); + models?.restore(); + jobs?.restore(); + require('../../../src/stores/remoteServerStore').useRemoteServerStore.getState().clearAllServers(); + app = undefined; + jobs = undefined; + models = undefined; + restoreBuildFlavor(); +}); + +/** A clip made on the paired computer, in a chat, with Continue under it. */ +async function clipInChat(computer: Parameters[0]): Promise { + app = await launchApp({ release: false }); + const { view, rtl, boundary } = app; + jobs = serveDesktopVideoJobs(boundary.fs!); + models = serveDesktopModelPulls(computer); + await passOnboarding(app, false); + await require('../../harness/proHarness').installPro(); + // What pairing leaves behind; the person picks the computer's video model on Home. + const { remoteServerManager } = require('../../../src/services/remoteServerManager'); + const server = await remoteServerManager.addServer({ + name: 'Studio Mac', + endpoint: DESKTOP_VIDEO_ENDPOINT, + providerType: 'openai-compatible', + modelManagement: 'offgrid-desktop-v1', + modelCatalog: { video: [DESKTOP_VIDEO_MODEL] }, + }); + rtl.fireEvent.press(await view.findByTestId('model-summary-video-open')); + rtl.fireEvent.press( + await view.findByTestId(`remote-video-model-${server.id}:${DESKTOP_VIDEO_MODEL.id}`, {}, { timeout: 5000 }), + ); + rtl.fireEvent.press(await view.findByTestId('new-chat-button', {}, { timeout: 5000 })); + await view.findByTestId('chat-input'); + rtl.fireEvent.press(view.getByTestId('quick-settings-button')); + rtl.fireEvent.press(await view.findByTestId('quick-video-mode')); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), 'a red fox trotting through fresh snow'); + rtl.fireEvent.press(view.getByTestId('send-button')); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); + return app; +} + +/** Continue is on screen but off; a tap opens the sheet. */ +async function tapMutedContinue(current: AnalyticsApp) { + const { view, rtl } = current; + const continueChip = await view.findByTestId('video-continue'); + expect(continueChip.props.accessibilityState?.disabled).toBe(true); + expect(view.queryByTestId('start-picture-sheet')).toBeNull(); + rtl.fireEvent.press(continueChip); + await view.findByTestId('start-picture-sheet'); +} + +it('follows the download already running on the computer, then Continue turns on', async () => { + const current = await clipInChat({ memoryGb: 34, downloading: { id: LIGHT, percent: 40 } }); + const { view, rtl } = current; + models!.hold(); + await tapMutedContinue(current); + await view.findByText('Downloading Wan 2.2 (5B), Light on your computer', {}, { timeout: 10_000 }); + expect(view.getByText(/^40% of /)).toBeTruthy(); + expect(view.queryByTestId('start-picture-download')).toBeNull(); + models!.release(); + await view.findByText('Wan 2.2 (5B), Light is ready.', {}, { timeout: 15_000 }); + // No second download was started; the computer switched to the build that landed. + expect(models!.pulls).toEqual([]); + expect(models!.activations).toEqual([DESKTOP_VIDEO_MODEL.id, LIGHT]); + rtl.fireEvent.press(view.getByTestId('start-picture-done')); + await rtl.waitFor(() => + expect(view.getAllByTestId('video-continue')[0].props.accessibilityState?.disabled).toBe(false), + ); +}, 60_000); + +it('offers the Light build on a computer without clear room for Full quality, and downloads it once', async () => { + const current = await clipInChat({ memoryGb: 34 }); + const { view, rtl } = current; + await tapMutedContinue(current); + await view.findByText('Continue needs Wan 2.2 (5B), Light, a 7.9 GB download.', {}, { timeout: 10_000 }); + expect(view.getByTestId('start-picture-describe')).toBeTruthy(); + models!.hold(); + rtl.fireEvent.press(view.getByTestId('start-picture-download')); + await view.findByText('Downloading Wan 2.2 (5B), Light on your computer', {}, { timeout: 10_000 }); + models!.release(); + await view.findByText('Wan 2.2 (5B), Light is ready.', {}, { timeout: 15_000 }); + expect(models!.pulls).toEqual([{ id: LIGHT }]); +}, 60_000); + +it('offers Full quality on a computer with clear room', async () => { + const current = await clipInChat({ memoryGb: 64 }); + await tapMutedContinue(current); + await current.view.findByText('Continue needs Wan 2.2 (5B), a 10.4 GB download.', {}, { timeout: 10_000 }); +}, 60_000); + +it('offers the Light build to an older Desktop that does not report its memory', async () => { + const current = await clipInChat({}); + await tapMutedContinue(current); + await current.view.findByText('Continue needs Wan 2.2 (5B), Light, a 7.9 GB download.', {}, { timeout: 10_000 }); +}, 60_000); + +it('says a computer is too small, and offers a new clip from a description', async () => { + const current = await clipInChat({ memoryGb: 16 }); + const { view } = current; + await tapMutedContinue(current); + await view.findByText( + 'Starting from a picture needs 24 GB of memory. Your computer has 16 GB.', + {}, + { timeout: 10_000 }, + ); + expect(view.getByTestId('start-picture-describe')).toBeTruthy(); + expect(view.queryByTestId('start-picture-download')).toBeNull(); +}, 60_000); + +it('uses a start-picture build installed on the computer since the phone last looked', async () => { + const current = await clipInChat({ memoryGb: 34 }); + const { view, rtl } = current; + models!.install(LIGHT); + await tapMutedContinue(current); + await view.findByText('Continue needs Wan 2.2 (5B), Light. It is on your computer.', {}, { timeout: 10_000 }); + rtl.fireEvent.press(view.getByTestId('start-picture-use')); + await view.findByText('Wan 2.2 (5B), Light is ready.', {}, { timeout: 15_000 }); + expect(models!.pulls).toEqual([]); + expect(models!.activations).toEqual([DESKTOP_VIDEO_MODEL.id, LIGHT]); + rtl.fireEvent.press(view.getByTestId('start-picture-done')); + await rtl.waitFor(() => + expect(view.getAllByTestId('video-continue')[0].props.accessibilityState?.disabled).toBe(false), + ); +}, 60_000); + +it('the storyboard\'s muted Start from a photo opens the sheet, with the shot kept from its description', async () => { + const current = await clipInChat({ memoryGb: 34 }); + const { view, rtl } = current; + rtl.fireEvent.press(await view.findByTestId('video-make-story')); + const photo = await view.findByTestId('shot-photo-0', {}, { timeout: 15_000 }); + expect(photo.props.accessibilityState?.disabled).toBe(true); + expect(view.queryByTestId('start-picture-sheet')).toBeNull(); + rtl.fireEvent.press(photo); + await view.findByText( + 'Starting from a photo needs Wan 2.2 (5B), Light, a 7.9 GB download.', + {}, + { timeout: 10_000 }, + ); + expect(view.getByText('Keep this shot from its description')).toBeTruthy(); +}, 60_000); + +it('the viewer\'s muted Continue opens the same sheet', async () => { + const current = await clipInChat({ memoryGb: 34 }); + const { view, rtl } = current; + rtl.fireEvent.press(view.getByTestId('chat-clip-card')); + const continueAction = await view.findByTestId('viewer-continue', {}, { timeout: 10_000 }); + expect(continueAction.props.accessibilityState?.disabled).toBe(true); + expect(view.queryByTestId('start-picture-sheet')).toBeNull(); + rtl.fireEvent.press(continueAction); + await view.findByText('Continue needs Wan 2.2 (5B), Light, a 7.9 GB download.', {}, { timeout: 10_000 }); +}, 60_000); diff --git a/__tests__/pro/video/storyboardEditing.rendered.test.tsx b/__tests__/pro/video/storyboardEditing.rendered.test.tsx new file mode 100644 index 000000000..f0e9cffee --- /dev/null +++ b/__tests__/pro/video/storyboardEditing.rendered.test.tsx @@ -0,0 +1,151 @@ +/** + * Shaping a film on its storyboard before it is made. + * + * SPEC (OGAM user's view): a storyboard lists the film's shots. I can pick the format, add a shot, + * remove one, rewrite what happens and what is said, change a shot's length, and turn narration + * and captions on or off; the film the computer makes is the one I shaped. Tonight on my computer + * starts it in its queue, with progress on the storyboard and a stop. A shot can be redone. A shot + * can start from my photo once the computer has a model that starts from a picture, after I say the + * photo is of me or someone who agreed. Delete asks first and takes me back to the chat. With an + * image model on the phone, each new shot gets a still to look at before anything is made. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the RAM sensor, the + * photo library, the OS dialog, and Desktop's gateway at `fetch`. + */ +jest.unmock('@react-navigation/native'); + +import { Image } from 'react-native'; +import { closeStudioApp, launchStudioApp, makeClipInChat, type StudioApp } from '../../harness/videoStudioApp'; + +afterEach(() => closeStudioApp(), 90_000); + +const LIGHT = 'off-grid-ai/Wan2.2-TI2V-5B-GGUF-Q4'; + +async function storyboard(prompt: string): Promise { + const app = await launchStudioApp(); + const { view, rtl } = app; + await makeClipInChat(app, prompt); + rtl.fireEvent.press(await view.findByTestId('video-make-story')); + await view.findByTestId('film-make-real', {}, { timeout: 15_000 }); + return app; +} + +const shotCount = (app: StudioApp) => app.view.queryAllByTestId(/^shot-\d+$/).length; + +interface SentStory { + story: { format: string; voice: string | null; captions: boolean; shots: Array<{ prompt: string; narration: string; seconds: number }> }; +} + +it('the film made is the storyboard as I shaped it', async () => { + const app = await storyboard('a lighthouse keeper climbs the stairs at dusk'); + const { view, rtl } = app; + const before = shotCount(app); + expect(before).toBeGreaterThan(1); + + rtl.fireEvent.press(view.getByText('Landscape')); + rtl.fireEvent.press(view.getByTestId('add-shot')); + await rtl.waitFor(() => expect(shotCount(app)).toBe(before + 1)); + const removes = view.getAllByLabelText('Remove this shot'); + rtl.fireEvent.press(removes[removes.length - 1]); + await rtl.waitFor(() => expect(shotCount(app)).toBe(before)); + + const what = view.getByLabelText('Shot 2, what happens'); + rtl.fireEvent.changeText(what, 'The keeper lights the great lamp'); + rtl.fireEvent(what, 'endEditing'); + const said = view.getByLabelText('Shot 2, narration'); + rtl.fireEvent.changeText(said, 'Every night, the same climb.'); + rtl.fireEvent(said, 'endEditing'); + rtl.fireEvent.press(view.getAllByLabelText('7 seconds')[1]); + rtl.fireEvent.press(view.getByLabelText('Narration on')); + await view.findByLabelText('Narration off'); + rtl.fireEvent.press(view.getByLabelText('Captions on')); + await view.findByLabelText('Captions off'); + + app.stories.hold(); + rtl.fireEvent.press(view.getByTestId('film-make-real')); + await rtl.waitFor(() => expect(app.stories.submissions).toHaveLength(1), { timeout: 10_000 }); + const sent = (app.stories.submissions[0] as unknown as SentStory).story; + expect(sent.format).toBe('landscape'); + expect(sent.voice).toBeNull(); + expect(sent.captions).toBe(false); + expect(sent.shots).toHaveLength(before); + expect(sent.shots[1]).toMatchObject({ prompt: 'The keeper lights the great lamp', narration: 'Every night, the same climb.', seconds: 7 }); +}, 90_000); + +it('Tonight starts it on the computer with progress on the storyboard and a stop; a shot can be redone', async () => { + const app = await storyboard('a fox crossing a frozen river'); + const { view, rtl } = app; + expect(view.queryByTestId('film-progress')).toBeNull(); + app.stories.hold(); + rtl.fireEvent.press(view.getByTestId('film-tonight')); + await view.findByTestId('film-progress', {}, { timeout: 10_000 }); + expect(view.queryByTestId('film-make-real')).toBeNull(); + rtl.fireEvent.press(view.getByTestId('film-cancel')); + await view.findByTestId('film-make-real', {}, { timeout: 10_000 }); + expect(app.stories.cancels).toHaveLength(1); + + // The opening shot is the clip from chat: Redo makes it again on the computer. + rtl.fireEvent.press(view.getByTestId('redo-shot-0')); + await view.findByTestId('film-progress', {}, { timeout: 10_000 }); + await rtl.waitFor(() => expect(app.stories.submissions).toHaveLength(2), { timeout: 10_000 }); +}, 90_000); + +it('a shot starts from my photo once the computer can, after I say yes', async () => { + const app = await storyboard('a dog running along the beach'); + const { view, rtl, boundary } = app; + boundary.fs!.seedFile('/mock/image.jpg', 2048); + app.models.install(LIGHT); + rtl.fireEvent.press(view.getByTestId('shot-photo-1')); + rtl.fireEvent.press(await view.findByTestId('start-picture-use', {}, { timeout: 10_000 })); + await view.findByText('Wan 2.2 (5B), Light is ready.', {}, { timeout: 15_000 }); + rtl.fireEvent.press(view.getByText('Choose a photo')); + await rtl.waitFor(() => expect(app.dialog()?.title).toBe('Is this you, or someone who agreed?'), { timeout: 10_000 }); + await app.answer('Yes'); + await rtl.waitFor(() => expect(view.getByTestId('shot-photo-1').props.accessibilityState?.selected).toBe(true)); + + app.stories.hold(); + rtl.fireEvent.press(view.getByTestId('film-make-real')); + await rtl.waitFor(() => expect(app.stories.submissions).toHaveLength(1), { timeout: 10_000 }); + const sent = app.stories.submissions[0] as { consent: boolean; init_images: Record }; + expect(sent.consent).toBe(true); + expect(Object.values(sent.init_images)).toHaveLength(1); +}, 90_000); + +it('Delete asks first, then the film is gone and I am back in the chat', async () => { + const app = await storyboard('a kite over green hills'); + const { view, rtl } = app; + rtl.fireEvent.press(view.getByTestId('film-delete')); + expect(app.dialog()?.title).toBe('Delete this film?'); + await app.answer('Delete'); + await view.findByTestId('chat-input', {}, { timeout: 10_000 }); + await rtl.waitFor(() => expect(view.queryByTestId('film-make-real')).toBeNull()); +}, 90_000); + +it('with an image model on the phone, the storyboard draws a still for each new shot', async () => { + const app = await launchStudioApp(undefined, { imageModel: true }); + const { view, rtl, boundary } = app; + rtl.fireEvent.press(await view.findByTestId('new-chat-button', {}, { timeout: 5000 })); + await view.findByTestId('chat-input'); + // Image mode picks the image model on the phone; then video mode for the clip. + rtl.fireEvent.press(view.getByTestId('quick-settings-button')); + rtl.fireEvent.press(await view.findByTestId('quick-image-mode')); + rtl.fireEvent.press(view.getByTestId('quick-settings-button')); + rtl.fireEvent.press(await view.findByTestId('quick-video-mode')); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), 'a lighthouse keeper climbs the stairs at dusk'); + rtl.fireEvent.press(view.getByTestId('send-button')); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); + const stillsBefore = boundary.diffusion.calls.generateImage.length; + + rtl.fireEvent.press(await view.findByTestId('video-make-story')); + await view.findByTestId('film-make-real', {}, { timeout: 15_000 }); + const newShots = view.queryAllByTestId(/^shot-\d+$/).length - 1; + await rtl.waitFor( + () => expect(boundary.diffusion.calls.generateImage.length - stillsBefore).toBe(newShots), + { timeout: 20_000 }, + ); + // Each new shot shows its still; the opening shot is the clip itself. + await rtl.waitFor(() => { + const stills = view.UNSAFE_getAllByType(Image).filter(image => /generated_images/.test(String(image.props.source?.uri))); + expect(stills.length).toBeGreaterThanOrEqual(newShots); + }, { timeout: 10_000 }); +}, 90_000); diff --git a/__tests__/pro/video/videoAfterRelaunch.rendered.test.tsx b/__tests__/pro/video/videoAfterRelaunch.rendered.test.tsx new file mode 100644 index 000000000..0af98797f --- /dev/null +++ b/__tests__/pro/video/videoAfterRelaunch.rendered.test.tsx @@ -0,0 +1,88 @@ +/** + * Video work on the paired computer survives the app closing. + * + * SPEC (OGAM user's view): if I close the app while my computer is making a clip, the chat says the + * connection was interrupted when I come back, and Retry checks the same job on the computer — no + * second clip is started — and the clip lands in the chat, or the computer's reason it could not. A film the computer was making when I + * closed the app carries on by itself, and arrives. + * + * The real App runs, closed and opened again (storage kept). Device boundaries only: the llama.rn + * engine, the disk, the RAM sensor and Desktop's gateway at `fetch`, which keeps its jobs running + * while the phone is closed. + */ +jest.unmock('@react-navigation/native'); + +import { closeStudioApp, launchStudioApp, makeClipInChat, relaunchStudioApp } from '../../harness/videoStudioApp'; + +afterEach(() => closeStudioApp(), 90_000); + +const CLIP = 'a paper boat drifting on a pond'; + +it('a clip interrupted by closing the app is picked up again with Retry, from the same job', async () => { + const first = await launchStudioApp(); + first.jobs.hold(); + const making = makeClipInChat(first, CLIP).catch(() => undefined); + await first.rtl.waitFor(() => expect(first.jobs.submissions).toHaveLength(1), { timeout: 15_000 }); + await first.view.findByTestId('video-progress', {}, { timeout: 10_000 }); + + const app = await relaunchStudioApp(); + await making; + const { view, rtl } = app; + app.jobs.release(); + rtl.fireEvent.press(await view.findByTestId('chats-tab')); + rtl.fireEvent.press(await view.findByText(CLIP, {}, { timeout: 10_000 })); + await view.findByText(/The connection was interrupted/, {}, { timeout: 10_000 }); + expect(view.queryByTestId('chat-clip-card')).toBeNull(); + rtl.fireEvent.press(view.getByText('Retry')); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); + // The same job on the computer (its client job id), not a second clip. + expect(new Set(app.jobs.submissions.map(sent => sent.client_job_id)).size).toBe(1); +}, 90_000); + +it('a film the computer was making when the app closed carries on and arrives', async () => { + const first = await launchStudioApp(); + await makeClipInChat(first, CLIP); + first.rtl.fireEvent.press(await first.view.findByTestId('video-make-story')); + first.stories.hold(); + first.rtl.fireEvent.press(await first.view.findByTestId('film-make-real', {}, { timeout: 15_000 })); + await first.rtl.waitFor(() => expect(first.stories.submissions).toHaveLength(1), { timeout: 10_000 }); + + const app = await relaunchStudioApp(); + const { view, rtl } = app; + app.stories.release(); + rtl.fireEvent.press(await view.findByText('Image Gallery', {}, { timeout: 10_000 })); + rtl.fireEvent.press(await view.findByText(/^Videos \(/)); + await view.findByTestId('library-grid', {}, { timeout: 10_000 }); + rtl.fireEvent.press(view.getByTestId('library-tab-stories')); + rtl.fireEvent.press((await view.findAllByTestId(/^library-item-/, {}, { timeout: 10_000 }))[0]); + // Opened while it is still being made, then Watch; or straight to the film once it is there. + await rtl.waitFor( + () => expect(view.queryByTestId('making-watch') ?? view.queryByTestId('video-viewer')).not.toBeNull(), + { timeout: 30_000 }, + ); + const watch = view.queryByTestId('making-watch'); + if (watch) rtl.fireEvent.press(watch); + await view.findByTestId('video-viewer', {}, { timeout: 10_000 }); + // The film the computer was already making, not a second one. + expect(new Set(app.stories.submissions.map(sent => (sent as { story: { id: string } }).story.id)).size).toBe(1); +}, 120_000); + +it('Retry on an interrupted clip the computer could not make says the computer\'s reason', async () => { + const first = await launchStudioApp(); + first.jobs.hold(); + const making = makeClipInChat(first, CLIP).catch(() => undefined); + await first.rtl.waitFor(() => expect(first.jobs.submissions).toHaveLength(1), { timeout: 15_000 }); + await first.view.findByTestId('video-progress', {}, { timeout: 10_000 }); + + const app = await relaunchStudioApp(); + await making; + const { view, rtl } = app; + rtl.fireEvent.press(await view.findByTestId('chats-tab')); + rtl.fireEvent.press(await view.findByText(CLIP, {}, { timeout: 10_000 })); + await view.findByText(/The connection was interrupted/, {}, { timeout: 10_000 }); + app.jobs.failRunning('The video model ran out of memory on this computer.'); + app.jobs.release(); + rtl.fireEvent.press(view.getByText('Retry')); + await view.findByText(/The video model ran out of memory on this computer\./, {}, { timeout: 15_000 }); + expect(view.queryByTestId('chat-clip-card')).toBeNull(); +}, 90_000); diff --git a/__tests__/pro/video/videoModelSearch.rendered.test.tsx b/__tests__/pro/video/videoModelSearch.rendered.test.tsx new file mode 100644 index 000000000..67309d1ce --- /dev/null +++ b/__tests__/pro/video/videoModelSearch.rendered.test.tsx @@ -0,0 +1,98 @@ +/** + * Finding a video model in Models › Video: search, a model's versions, and their downloads. + * + * SPEC (OGAM user's view): the Video tab lists the video models this phone can run. I can search + * Hugging Face; a failed search says why. Opening a model + * shows its memory suggestion against this phone and the versions this phone can run, the standard + * one first; downloading a version shows it downloading and can be cancelled. Back returns to the list. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the RAM sensor, the + * native background downloads, and Hugging Face's API at `fetch` (its /api/models search and a + * repo's ?blobs=true listing, in Hugging Face's own shapes). + */ +jest.unmock('@react-navigation/native'); + +import { launchApp, passOnboarding, restoreBuildFlavor, type AnalyticsApp } from '../../harness/analyticsApp'; + +const REPO = 'off-grid-ai/Wan2.1-T2V-1.3B-GGUF'; +const REPO_NAME = 'Wan 2.1 (1.3B)'; +const HF = 'https://huggingface.co/api'; + +let app: AnalyticsApp | undefined; +let original: typeof globalThis.fetch | undefined; + +afterEach(() => { + if (original) globalThis.fetch = original; + original = undefined; + app?.view.unmount(); + app?.endpoint.restore(); + app = undefined; + restoreBuildFlavor(); +}); + +/** Hugging Face: a search answers `results`, or fails with `status`; the repo lists two weights. */ +function serveHuggingFace(search: { results?: Array<{ id: string; downloads: number }>; status?: number }) { + original = globalThis.fetch; + const inner = original; + const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const href = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + if (href.startsWith(`${HF}/models?`)) { + if (search.status) return json({ error: 'Service unavailable' }, search.status); + return json(search.results ?? []); + } + if (href === `${HF}/models/${REPO}?blobs=true`) { + return json({ + id: REPO, + siblings: [ + { rfilename: 'README.md', size: 2048 }, + { rfilename: 'Wan2.1-T2V-1.3B-Q8_0.gguf', size: 1519435360, lfs: { size: 1519435360, oid: 'a'.repeat(64) } }, + { rfilename: 'Wan2.1-T2V-1.3B-calibrated-Q4_K_S.gguf', size: 816061024, lfs: { size: 816061024, oid: 'b'.repeat(64) } }, + ], + }); + } + return inner(input, init); + }) as typeof globalThis.fetch; +} + +async function videoTab(): Promise { + app = await launchApp({ release: false, download: true }); + await passOnboarding(app, false); + await require('../../harness/proHarness').installPro(); + const { view, rtl } = app; + rtl.fireEvent.press(await view.findByTestId('models-tab')); + rtl.fireEvent.press(await view.findByText('Video')); + await view.findByText(REPO_NAME, {}, { timeout: 10_000 }); + return app; +} + +it('a failed search says why', async () => { + serveHuggingFace({ status: 503 }); + const { view, rtl } = await videoTab(); + rtl.fireEvent.changeText(view.getByLabelText('Search video models on Hugging Face'), 'wan'); + await view.findByText('Hugging Face search failed: HTTP 503', {}, { timeout: 10_000 }); +}, 60_000); + +it('a model shows its versions, standard first; a version downloads and can be cancelled', async () => { + serveHuggingFace({}); + const { view, rtl } = await videoTab(); + expect(view.queryByText('Available versions')).toBeNull(); + rtl.fireEvent.press(view.getByText(REPO_NAME)); + await view.findByText('Available versions', {}, { timeout: 10_000 }); + // 16 GB suggested against this 12 GB phone. + expect(view.getByText('16 GB memory suggested · This phone: 12 GB')).toBeTruthy(); + expect(view.getByText('Local generation may exceed available memory.')).toBeTruthy(); + const standard = await view.findByText('Wan 2.1 - Standard', {}, { timeout: 10_000 }); + const compact = view.getByText('Wan 2.1 - Compact Q4_K_S'); + expect(standard).toBeTruthy(); + expect(compact).toBeTruthy(); + + rtl.fireEvent.press(view.getByLabelText('Download Wan 2.1 - Compact Q4_K_S')); + rtl.fireEvent.press(await view.findByLabelText('Cancel download', {}, { timeout: 10_000 })); + await view.findByLabelText('Download Wan 2.1 - Compact Q4_K_S', {}, { timeout: 10_000 }); + + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + await rtl.waitFor(() => expect(view.queryByText('Available versions')).toBeNull()); + expect(view.getByText(REPO_NAME)).toBeTruthy(); +}, 60_000); diff --git a/__tests__/pro/video/videoOnThisPhone.rendered.test.tsx b/__tests__/pro/video/videoOnThisPhone.rendered.test.tsx new file mode 100644 index 000000000..96fb7c264 --- /dev/null +++ b/__tests__/pro/video/videoOnThisPhone.rendered.test.tsx @@ -0,0 +1,421 @@ +/** + * A video model downloaded to the phone, and clips made on the phone with it. + * + * SPEC (OGAM user's view): in Models › Video I find a video model, see its size and the memory it + * suggests, and download it: every file of it, with progress, until the card says it is on the + * phone. I pick it on Home and describe a clip in a chat in video mode: the clip is made on this + * phone, shows its progress while it runs, and lands as a clip card. I can stop a clip midway. A whole film + * is made here too, shot by shot, then joined. A download can be paused, resumed or cancelled, and + * an installed model deleted; a download the network drops says so and can be retried or removed, and one cut off by the app + * closing says so and can be retried. A clip still being made when the app opens again carries on; one the phone stopped with the + * app says so and Retry makes it again; one the worker could not finish shows why. While a clip is + * made, the engine's first frame shows; a phone short of free + * memory says so before it starts and can run it anyway. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the RAM sensor, the + * native background downloads, the phone's video engine and film composer (nativeVideoEngine). + */ +jest.unmock('@react-navigation/native'); + +import { GB } from '../../harness/nativeBoundary'; +import { launchApp, passOnboarding, restoreBuildFlavor, type AnalyticsApp } from '../../harness/analyticsApp'; +import { installNativeVideoEngine, phoneDownloads, type NativeVideoEngine, type PhoneDownloads } from '../../harness/nativeVideoEngine'; + +/** Wan 2.1 (1.3B), FP16: the catalog build Android runs whose files carry no published checksum. */ +const MODEL = { id: 'Comfy-Org/Wan_2.1_ComfyUI_repackaged', name: 'Wan 2.1 (1.3B), FP16', files: 3 }; + +let app: AnalyticsApp | undefined; + +afterEach(() => { + // What ending Pro does: stop every render and the charger watch, so no timer outlives the test. + if (app) require('../../../pro/video/studio/filmRenderer').stopAllRenders(); + app?.view.unmount(); + app?.endpoint.restore(); + app = undefined; + restoreBuildFlavor(); +}); + +interface PhoneVideoApp extends AnalyticsApp { + engine: NativeVideoEngine; + downloads: PhoneDownloads; +} + +/** By default a phone with room for a 6.3 GB video model: 24 GB, 20 GB of it free (the RAM sensor). */ +async function launch( + options: { + ram?: { totalBytes: number; availBytes: number }; + relaunch?: boolean; + /** The video model a finished download left on the phone's disk, which a relaunch keeps. */ + installedModel?: boolean; + carryOver?: NativeVideoEngine; + } = {}, +): Promise { + let engine!: NativeVideoEngine; + const ram = options.ram ?? { totalBytes: 24 * GB, availBytes: 20 * GB }; + app = await launchApp({ release: false, download: true, relaunch: options.relaunch, beforeApp: boundary => { + boundary.setRam({ platform: 'android', ...ram }); + // A relaunch keeps the phone's disk; this harness's disk is new per launch, so the video model a + // finished download left there is put back, file by file, at its real size. + if (options.installedModel) { + const { videoModelDirectory } = require('../../../src/services/videoModelFiles'); + const entry = require('@offgrid/models').CATALOG.find((model: { id: string }) => model.id === MODEL.id); + for (const file of entry.files) boundary.fs!.seedFile(`${videoModelDirectory(MODEL.id)}/${file.name}`, file.sizeBytes); + } + engine = installNativeVideoEngine(boundary, options.carryOver); + } }); + if (options.relaunch) await app.view.findByTestId('home-screen', {}, { timeout: 10_000 }); + else await passOnboarding(app, false); + await require('../../harness/proHarness').installPro(); + return { ...app, engine, downloads: phoneDownloads(app.boundary, app.rtl.act) }; +} + +/** Models tab › Video › Download, then every file of the pack finishes. */ +async function downloadVideoModel(current: PhoneVideoApp): Promise { + const { view, rtl } = current; + rtl.fireEvent.press(await view.findByTestId('models-tab')); + rtl.fireEvent.press(await view.findByText('Video')); + await view.findByText(MODEL.name, {}, { timeout: 10_000 }); + rtl.fireEvent.press(view.getByLabelText(`Download ${MODEL.name}`)); + let finished = 0; + await rtl.waitFor(async () => { + finished += await current.downloads.finishRunning(); + expect(finished).toBe(MODEL.files); + }, { timeout: 15_000 }); + await rtl.waitFor(() => expect(view.queryByLabelText(`Download ${MODEL.name}`)).toBeNull()); + await rtl.waitFor(() => expect(view.getAllByLabelText('Delete model').length).toBeGreaterThan(0), { timeout: 10_000 }); +} + +/** Home's video summary › the downloaded model, then a new chat in video mode, and Send. */ +async function describeClipOnPhone(current: PhoneVideoApp, prompt: string): Promise { + const { view, rtl } = current; + rtl.fireEvent.press(await view.findByTestId('home-tab')); + rtl.fireEvent.press(await view.findByTestId('model-summary-video-open')); + rtl.fireEvent.press(await view.findByTestId(`video-model-row-${MODEL.id}-select`, {}, { timeout: 5000 })); + rtl.fireEvent.press(await view.findByTestId('new-chat-button', {}, { timeout: 5000 })); + await view.findByTestId('chat-input'); + rtl.fireEvent.press(view.getByTestId('quick-settings-button')); + rtl.fireEvent.press(await view.findByTestId('quick-video-mode')); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), prompt); + rtl.fireEvent.press(view.getByTestId('send-button')); +} + +it('downloads every file of a video model, then makes a clip on the phone', async () => { + const current = await launch(); + const { view, rtl } = current; + await downloadVideoModel(current); + current.engine.hold(); + await describeClipOnPhone(current, 'a hummingbird hovering over a red flower'); + // In flight on the phone: the engine's step count, and no clip card yet. + await view.findByText(/Step 1 of \d+/, {}, { timeout: 15_000 }); + expect(view.queryByTestId('chat-clip-card')).toBeNull(); + current.engine.release(); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); + await rtl.waitFor(() => expect(view.queryByTestId('video-progress')).toBeNull()); + expect(current.engine.generations).toHaveLength(1); + expect(current.engine.generations[0].prompt).toBe('a hummingbird hovering over a red flower'); +}, 90_000); + +it('stops a clip midway', async () => { + const current = await launch(); + const { view, rtl } = current; + await downloadVideoModel(current); + current.engine.hold(); + await describeClipOnPhone(current, 'waves rolling onto a sandy beach'); + await view.findByText(/Step 1 of \d+/, {}, { timeout: 15_000 }); + rtl.fireEvent.press(view.getByTestId('stop-button')); + await rtl.waitFor(() => expect(view.queryByTestId('video-progress')).toBeNull(), { timeout: 10_000 }); + expect(view.queryByTestId('chat-clip-card')).toBeNull(); + +}, 90_000); + +it('makes a whole film on the phone: each shot, then the shots joined', async () => { + const current = await launch(); + const { view, rtl } = current; + await downloadVideoModel(current); + await describeClipOnPhone(current, 'a cat chasing a butterfly through a garden'); + rtl.fireEvent.press(await view.findByTestId('video-make-story', {}, { timeout: 15_000 })); + await view.findByText('Renders on this phone, also with the screen off.', {}, { timeout: 15_000 }); + current.engine.hold(); + rtl.fireEvent.press(view.getByTestId('film-make-real')); + await view.findByTestId('video-making', {}, { timeout: 10_000 }); + // The film is being made here: a shot in flight, nothing joined yet. + await view.findByText(/Shot \d+ of \d+/, {}, { timeout: 15_000 }); + expect(view.getByText('You can leave the app. We send a notification when it is ready.')).toBeTruthy(); + expect(current.engine.films).toHaveLength(0); + current.engine.release(); + rtl.fireEvent.press(await view.findByTestId('making-watch', {}, { timeout: 30_000 })); + await view.findByTestId('video-viewer', {}, { timeout: 10_000 }); + // Every shot was made on the phone and the composer joined them into one film. + expect(current.engine.films).toHaveLength(1); + expect(current.engine.films[0].clips.length).toBeGreaterThan(1); +}, 90_000); + +it('pauses, resumes and cancels a video model download, and deletes an installed one', async () => { + const current = await launch(); + const { view, rtl } = current; + rtl.fireEvent.press(await view.findByTestId('models-tab')); + rtl.fireEvent.press(await view.findByText('Video')); + await view.findByText(MODEL.name, {}, { timeout: 10_000 }); + rtl.fireEvent.press(view.getByLabelText(`Download ${MODEL.name}`)); + rtl.fireEvent.press(await view.findByLabelText('Pause download', {}, { timeout: 10_000 })); + rtl.fireEvent.press(await view.findByLabelText('Resume download', {}, { timeout: 10_000 })); + rtl.fireEvent.press(await view.findByLabelText('Cancel download', {}, { timeout: 10_000 })); + await view.findByLabelText(`Download ${MODEL.name}`, {}, { timeout: 10_000 }); + expect(view.queryByLabelText('Cancel download')).toBeNull(); + + await downloadVideoModel(current); + rtl.fireEvent.press(view.getAllByLabelText('Delete model')[0]); + await view.findByLabelText(`Download ${MODEL.name}`, {}, { timeout: 10_000 }); +}, 90_000); + +/** A clip on the phone, then Make it a story: the storyboard of a film made here. */ +async function phoneStoryboard(current: PhoneVideoApp, prompt: string): Promise { + const { view, rtl } = current; + await downloadVideoModel(current); + await describeClipOnPhone(current, prompt); + rtl.fireEvent.press(await view.findByTestId('video-make-story', {}, { timeout: 15_000 })); + await view.findByText('Renders on this phone, also with the screen off.', {}, { timeout: 15_000 }); +} + +it('a shot the engine cannot make stops the film with its reason; Try again finishes it', async () => { + const current = await launch(); + const { view, rtl } = current; + await phoneStoryboard(current, 'a sailboat crossing a calm bay'); + current.engine.failNext('The video engine ran out of memory.'); + rtl.fireEvent.press(view.getByTestId('film-make-real')); + await view.findByTestId('video-making', {}, { timeout: 10_000 }); + await view.findByText('The video engine ran out of memory.', {}, { timeout: 15_000 }); + expect(view.queryByTestId('making-watch')).toBeNull(); + // The storyboard names the shot that failed. + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + await rtl.waitFor(() => expect(view.getAllByText('The video engine ran out of memory.').length).toBeGreaterThan(0)); + rtl.fireEvent.press(view.getByTestId('film-make-real')); + rtl.fireEvent.press(await view.findByTestId('making-watch', {}, { timeout: 30_000 })); + await view.findByTestId('video-viewer', {}, { timeout: 10_000 }); + expect(current.engine.films).toHaveLength(1); +}, 90_000); + +it('Tonight on the phone, off the charger, waits for it and can be stopped', async () => { + const current = await launch(); + const { view, rtl } = current; + await phoneStoryboard(current, 'fireflies over a summer field'); + expect(view.queryByText('Waiting for the charger')).toBeNull(); + rtl.fireEvent.press(view.getByTestId('film-tonight')); + await view.findByText('Waiting for the charger', {}, { timeout: 10_000 }); + // Nothing renders while it waits. + expect(current.engine.generations).toHaveLength(1); + rtl.fireEvent.press(view.getByTestId('film-cancel')); + await view.findByTestId('film-make-real', {}, { timeout: 10_000 }); + expect(view.queryByText('Waiting for the charger')).toBeNull(); +}, 90_000); + +it('the film\'s card in the chat opens its storyboard as a draft, and Making while it is made', async () => { + const current = await launch(); + const { view, rtl } = current; + await phoneStoryboard(current, 'a hot air balloon over the desert'); + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + const card = (await view.findAllByTestId('film-card', {}, { timeout: 10_000 })).pop()!; + rtl.fireEvent.press(rtl.within(card).getAllByRole('button')[0]); + await view.findByTestId('film-make-real', {}, { timeout: 10_000 }); + current.engine.hold(); + rtl.fireEvent.press(view.getByTestId('film-make-real')); + await view.findByTestId('video-making', {}, { timeout: 10_000 }); + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + const working = (await view.findAllByTestId('film-card', {}, { timeout: 10_000 })).pop()!; + rtl.fireEvent.press(rtl.within(working).getAllByRole('button')[0]); + await view.findByTestId('making-stop', {}, { timeout: 10_000 }); + rtl.fireEvent.press(view.getByTestId('making-stop')); + await view.findByTestId('making-retry', {}, { timeout: 10_000 }); +}, 90_000); + +it('a video model download the network drops says so; Retry finishes it, and Remove clears it', async () => { + const current = await launch(); + const { view, rtl, boundary } = current; + rtl.fireEvent.press(await view.findByTestId('models-tab')); + rtl.fireEvent.press(await view.findByText('Video')); + await view.findByText(MODEL.name, {}, { timeout: 10_000 }); + const dropNetwork = async () => { + await rtl.waitFor(() => expect(boundary.download!.active().some(row => row.status === 'running')).toBe(true)); + const row = boundary.download!.active().filter(item => item.status === 'running').pop()!; + await rtl.act(async () => { + boundary.download!.events.emit('DownloadError', { + downloadId: row.downloadId, fileName: row.fileName, modelId: row.modelId, + status: 'failed', reason: 'Network connection lost', + }); + }); + }; + + rtl.fireEvent.press(view.getByLabelText(`Download ${MODEL.name}`)); + await dropNetwork(); + await view.findByText(/Network connection lost/, {}, { timeout: 10_000 }); + rtl.fireEvent.press(await view.findByLabelText('Remove download', {}, { timeout: 10_000 })); + await view.findByLabelText(`Download ${MODEL.name}`, {}, { timeout: 10_000 }); + expect(view.queryByLabelText('Retry download')).toBeNull(); + + rtl.fireEvent.press(view.getByLabelText(`Download ${MODEL.name}`)); + await dropNetwork(); + rtl.fireEvent.press(await view.findByLabelText('Retry download', {}, { timeout: 10_000 })); + let finished = 0; + await rtl.waitFor(async () => { + finished += await current.downloads.finishRunning(); + expect(view.queryAllByLabelText('Delete model').length).toBeGreaterThan(0); + }, { timeout: 15_000 }); + // The retry really downloaded the files, not just cleared the error. + expect(finished).toBeGreaterThan(0); + expect(view.queryByLabelText('Retry download')).toBeNull(); +}, 90_000); + +it('shows the engine\'s first frame while the clip is still being made', async () => { + const current = await launch(); + const { view, rtl } = current; + await downloadVideoModel(current); + current.engine.showPreviews(); + current.engine.hold(); + await describeClipOnPhone(current, 'a red kite over a hillside'); + await view.findByLabelText('Live preview of the clip', {}, { timeout: 15_000 }); + expect(view.queryByTestId('chat-clip-card')).toBeNull(); + current.engine.release(); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); + await rtl.waitFor(() => expect(view.queryByLabelText('Live preview of the clip')).toBeNull()); +}, 90_000); + +it('on a phone short of free memory, says so before it starts, and Run anyway makes the clip', async () => { + const current = await launch({ ram: { totalBytes: 12 * GB, availBytes: 8 * GB } }); + const { view } = current; + await downloadVideoModel(current); + await describeClipOnPhone(current, 'a hummingbird hovering over a red flower'); + const runAnyway = await view.findByText('Run anyway', {}, { timeout: 15_000 }); + expect(current.engine.generations).toHaveLength(0); + current.rtl.fireEvent.press(runAnyway); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); + expect(current.engine.generations).toHaveLength(1); +}, 90_000); + +it('a video model download cut off by the app closing says so, and Retry finishes it', async () => { + const first = await launch(); + first.rtl.fireEvent.press(await first.view.findByTestId('models-tab')); + first.rtl.fireEvent.press(await first.view.findByText('Video')); + await first.view.findByText(MODEL.name, {}, { timeout: 10_000 }); + first.rtl.fireEvent.press(first.view.getByLabelText(`Download ${MODEL.name}`)); + await first.view.findByLabelText('Cancel download', {}, { timeout: 10_000 }); + first.view.unmount(); + first.endpoint.restore(); + + const current = await launch({ relaunch: true }); + const { view, rtl } = current; + rtl.fireEvent.press(await view.findByTestId('models-tab')); + rtl.fireEvent.press(await view.findByText('Video')); + await view.findByText(/Interrupted — app closed\./, {}, { timeout: 10_000 }); + rtl.fireEvent.press(view.getByLabelText('Retry download')); + await rtl.waitFor(async () => { + await current.downloads.finishRunning(); + expect(view.queryAllByLabelText('Delete model').length).toBeGreaterThan(0); + }, { timeout: 15_000 }); + expect(view.queryByLabelText('Retry download')).toBeNull(); +}, 90_000); + +it('a video model too big for the space left says how much to free, and starts nothing', async () => { + const current = await launch(); + const { view, rtl, boundary } = current; + boundary.fs!.setFreeBytes(2 * GB); + rtl.fireEvent.press(await view.findByTestId('models-tab')); + rtl.fireEvent.press(await view.findByText('Video')); + await view.findByText(MODEL.name, {}, { timeout: 10_000 }); + rtl.fireEvent.press(view.getByLabelText(`Download ${MODEL.name}`)); + await view.findByText(/^Not enough space for this video model\. Free about \d+ GB and try again\.$/, {}, { timeout: 10_000 }); + expect(boundary.download!.active()).toHaveLength(0); + expect(view.getByLabelText(`Download ${MODEL.name}`)).toBeTruthy(); +}, 90_000); + +it('Retry after a dropped download keeps the files already on the phone', async () => { + const current = await launch(); + const { view, rtl, boundary } = current; + rtl.fireEvent.press(await view.findByTestId('models-tab')); + rtl.fireEvent.press(await view.findByText('Video')); + await view.findByText(MODEL.name, {}, { timeout: 10_000 }); + rtl.fireEvent.press(view.getByLabelText(`Download ${MODEL.name}`)); + // The first file lands; the network drops during the second. + await rtl.waitFor(async () => expect(await current.downloads.finishRunning()).toBe(1), { timeout: 10_000 }); + await rtl.waitFor(() => expect(boundary.download!.active().filter(row => row.status === 'running')).toHaveLength(2)); + const second = boundary.download!.active().filter(row => row.status === 'running').pop()!; + await rtl.act(async () => { + boundary.download!.events.emit('DownloadError', { + downloadId: second.downloadId, fileName: second.fileName, modelId: second.modelId, + status: 'failed', reason: 'Network connection lost', + }); + }); + const startedBefore = boundary.download!.module.startDownload.mock.calls.length; + rtl.fireEvent.press(await view.findByLabelText('Retry download', {}, { timeout: 10_000 })); + let finished = 0; + await rtl.waitFor(async () => { + finished += await current.downloads.finishRunning(); + expect(view.queryAllByLabelText('Delete model').length).toBeGreaterThan(0); + }, { timeout: 15_000 }); + // The retry really finished downloads, and only the two files not yet on the phone were fetched again. + expect(finished).toBeGreaterThan(0); + expect(boundary.download!.module.startDownload.mock.calls.length - startedBefore).toBe(MODEL.files - 1); +}, 90_000); + +it('a clip still being made when the app is opened again carries on and lands in its chat', async () => { + const first = await launch(); + await downloadVideoModel(first); + first.engine.hold(); + await describeClipOnPhone(first, 'a lantern floating down a dark river'); + await first.view.findByText(/Step 1 of \d+/, {}, { timeout: 15_000 }); + first.view.unmount(); + first.endpoint.restore(); + + // The app's JS starts again; Android's video worker kept making the clip. + const current = await launch({ relaunch: true, installedModel: true, carryOver: first.engine }); + const { view, rtl } = current; + rtl.fireEvent.press(await view.findByTestId('chats-tab')); + rtl.fireEvent.press(await view.findByText('a lantern floating down a dark river', {}, { timeout: 10_000 })); + await view.findByTestId('video-progress', {}, { timeout: 10_000 }); + expect(view.queryByTestId('chat-clip-card')).toBeNull(); + current.engine.release(); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); + // The same job, found again: no second clip was started. + expect(current.engine.generations).toHaveLength(1); +}, 90_000); + +it('a clip lost with the app (the phone stopped the worker) says so, and Retry makes it again', async () => { + const first = await launch(); + await downloadVideoModel(first); + first.engine.hold(); + await describeClipOnPhone(first, 'a paper plane over the rooftops'); + await first.view.findByText(/Step 1 of \d+/, {}, { timeout: 15_000 }); + first.view.unmount(); + first.endpoint.restore(); + + // Opened again after the phone ended the whole app, video worker included: a fresh engine knows no job. + const current = await launch({ relaunch: true, installedModel: true }); + const { view, rtl } = current; + rtl.fireEvent.press(await view.findByTestId('chats-tab')); + rtl.fireEvent.press(await view.findByText('a paper plane over the rooftops', {}, { timeout: 10_000 })); + await view.findByText(/Video generation was interrupted\. Retry to start it again\./, {}, { timeout: 10_000 }); + expect(current.engine.generations).toHaveLength(0); + rtl.fireEvent.press(view.getByText('Retry')); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); + expect(current.engine.generations).toHaveLength(1); + expect(current.engine.generations[0].prompt).toBe('a paper plane over the rooftops'); +}, 90_000); + +it('a clip the worker could not finish while the app was away shows the engine\'s reason', async () => { + const first = await launch(); + await downloadVideoModel(first); + first.engine.hold(); + await describeClipOnPhone(first, 'a whale breaching at sunset'); + await first.view.findByText(/Step 1 of \d+/, {}, { timeout: 15_000 }); + first.view.unmount(); + first.endpoint.restore(); + + const current = await launch({ relaunch: true, installedModel: true, carryOver: first.engine }); + const { view, rtl } = current; + current.engine.failNext('The video engine ran out of memory.'); + current.engine.release(); + rtl.fireEvent.press(await view.findByTestId('chats-tab')); + rtl.fireEvent.press(await view.findByText('a whale breaching at sunset', {}, { timeout: 10_000 })); + await view.findByText(/The video engine ran out of memory\./, {}, { timeout: 15_000 }); + expect(view.queryByTestId('chat-clip-card')).toBeNull(); +}, 90_000); diff --git a/__tests__/pro/video/videoRefusals.rendered.test.tsx b/__tests__/pro/video/videoRefusals.rendered.test.tsx new file mode 100644 index 000000000..3b3e9bfed --- /dev/null +++ b/__tests__/pro/video/videoRefusals.rendered.test.tsx @@ -0,0 +1,51 @@ +/** + * A clip that cannot be made says why. + * + * SPEC (OGAM user's view): when I describe a clip in video mode and the app will not make it (the + * description breaks the video rules), I am told why, and nothing is sent to my computer. When my + * computer cannot make a clip, the chat says the computer's reason once, on its failure card. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the RAM sensor and + * Desktop's gateway at `fetch`. + */ +jest.unmock('@react-navigation/native'); + +import { closeStudioApp, launchStudioApp } from '../../harness/videoStudioApp'; + +afterEach(() => closeStudioApp(), 90_000); + +async function videoChat(app: Awaited>): Promise { + const { view, rtl } = app; + rtl.fireEvent.press(await view.findByTestId('new-chat-button', {}, { timeout: 5000 })); + await view.findByTestId('chat-input'); + rtl.fireEvent.press(view.getByTestId('quick-settings-button')); + rtl.fireEvent.press(await view.findByTestId('quick-video-mode')); +} + +it('a clip the computer cannot make shows the computer\'s reason once, on its failure card', async () => { + const app = await launchStudioApp(); + const { view, rtl } = app; + await videoChat(app); + app.jobs.hold(); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), 'a lighthouse in a storm'); + rtl.fireEvent.press(view.getByTestId('send-button')); + await rtl.waitFor(() => expect(app.jobs.submissions).toHaveLength(1), { timeout: 15_000 }); + app.jobs.failRunning('The video model ran out of memory on this computer.'); + app.jobs.release(); + await view.findByText(/The video model ran out of memory on this computer\./, {}, { timeout: 15_000 }); + expect(view.getAllByText(/The video model ran out of memory on this computer\./)).toHaveLength(1); + expect(view.queryByText('Could not make the video')).toBeNull(); + expect(view.queryByTestId('chat-clip-card')).toBeNull(); +}, 90_000); + +it('a description the video rules refuse says why, and nothing reaches the computer', async () => { + const app = await launchStudioApp(); + const { view, rtl } = app; + await videoChat(app); + expect(view.queryByText("This video can't be made.")).toBeNull(); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), 'a deepfake porn clip of a celebrity'); + rtl.fireEvent.press(view.getByTestId('send-button')); + await view.findByText("This video can't be made.", {}, { timeout: 10_000 }); + expect(app.jobs.submissions).toHaveLength(0); + expect(view.queryByTestId('chat-clip-card')).toBeNull(); +}, 90_000); diff --git a/__tests__/pro/video/videoSettings.rendered.test.tsx b/__tests__/pro/video/videoSettings.rendered.test.tsx new file mode 100644 index 000000000..4a87ca671 --- /dev/null +++ b/__tests__/pro/video/videoSettings.rendered.test.tsx @@ -0,0 +1,70 @@ +/** + * Video settings in a chat shape the next clip. + * + * SPEC (OGAM user's view): a chat's settings have a Video generation section, named for the video + * model in use (the name the computer gives it, not its id). A fixed seed and what to avoid that I set there are what my next clip is + * made with on my computer. A seed that is not a whole + * number is not kept. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the RAM sensor and Desktop's gateway at `fetch`. + */ +jest.unmock('@react-navigation/native'); + +import { closeStudioApp, launchStudioApp, type StudioApp } from '../../harness/videoStudioApp'; +import { DESKTOP_VIDEO_MODEL } from '../../harness/desktopVideoServer'; + +afterEach(() => closeStudioApp(), 90_000); + +const CLIP = 'a red fox trotting through fresh snow'; + +/** A new chat, its settings, then the Video generation section. */ +async function videoSettings(): Promise { + const app = await launchStudioApp(); + const { view, rtl } = app; + rtl.fireEvent.press(await view.findByTestId('new-chat-button', {}, { timeout: 5000 })); + await view.findByTestId('chat-input'); + rtl.fireEvent.press(view.getByTestId('chat-settings-icon')); + expect(view.queryByLabelText('Video seed')).toBeNull(); + rtl.fireEvent.press(await view.findByLabelText('Video generation')); + await view.findByLabelText('Video seed'); + return app; +} + +async function sendClip(app: StudioApp, prompt: string): Promise { + const { view, rtl } = app; + const close = view.queryAllByTestId('app-sheet-close'); + if (close.length) rtl.fireEvent.press(close[close.length - 1]); + rtl.fireEvent.press(await view.findByTestId('quick-settings-button')); + rtl.fireEvent.press(await view.findByTestId('quick-video-mode')); + rtl.fireEvent.changeText(view.getByTestId('chat-input'), prompt); + rtl.fireEvent.press(view.getByTestId('send-button')); + await view.findByTestId('chat-clip-card', {}, { timeout: 15_000 }); +} + +it('a fixed seed and what to avoid reach the computer with the next clip', async () => { + const app = await videoSettings(); + const { view, rtl } = app; + const seed = view.getByLabelText('Video seed'); + rtl.fireEvent.changeText(seed, '42'); + rtl.fireEvent(seed, 'endEditing'); + rtl.fireEvent.changeText(view.getByLabelText('Video negative prompt'), 'blurry, text'); + await sendClip(app, CLIP); + expect(app.jobs.submissions[0]).toMatchObject({ prompt: CLIP, seed: 42, negativePrompt: 'blurry, text' }); +}, 90_000); + +it('is named for the computer\'s video model, never its id', async () => { + const app = await videoSettings(); + const { view, rtl } = app; + await rtl.waitFor(() => expect(view.getByText(DESKTOP_VIDEO_MODEL.name)).toBeTruthy()); + expect(view.queryByText(DESKTOP_VIDEO_MODEL.id)).toBeNull(); +}, 90_000); + +it('a seed that is not a whole number is not kept', async () => { + const app = await videoSettings(); + const { view, rtl } = app; + const seed = view.getByLabelText('Video seed'); + rtl.fireEvent.changeText(seed, '4.5'); + rtl.fireEvent(seed, 'endEditing'); + await rtl.waitFor(() => expect(view.getByLabelText('Video seed').props.value).toBe('')); +}, 90_000); + diff --git a/__tests__/pro/video/videoWithoutAModel.rendered.test.tsx b/__tests__/pro/video/videoWithoutAModel.rendered.test.tsx new file mode 100644 index 000000000..f425f2255 --- /dev/null +++ b/__tests__/pro/video/videoWithoutAModel.rendered.test.tsx @@ -0,0 +1,62 @@ +/** + * Turning on video in a chat when this phone has no video model. + * + * SPEC (OGAM user's view): I never meet a dead end. + * - A paired computer that makes videos: video turns on and the clip is made there. No question. + * - No computer paired: a sheet says videos are made on a computer and offers "Find my computer", + * which opens the screen where I pair one. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk and the RAM sensor. + */ +jest.unmock('@react-navigation/native'); + +import { launchApp, openNewChat, passOnboarding, restoreBuildFlavor, type AnalyticsApp } from '../../harness/analyticsApp'; +import { DESKTOP_VIDEO_ENDPOINT, DESKTOP_VIDEO_MODEL } from '../../harness/desktopVideoServer'; + +let app: AnalyticsApp | undefined; + +afterEach(() => { + app?.view.unmount(); + app?.endpoint.restore(); + require('../../../src/stores/remoteServerStore').useRemoteServerStore.getState().clearAllServers(); + app = undefined; + restoreBuildFlavor(); +}); + +async function chatWithVideoOff(paired: boolean): Promise { + app = await launchApp({ release: false }); + const { view, rtl } = app; + await passOnboarding(app, false); + await require('../../harness/proHarness').installPro(); + if (paired) { + const { remoteServerManager } = require('../../../src/services/remoteServerManager'); + await remoteServerManager.addServer({ + name: 'Studio Mac', + endpoint: DESKTOP_VIDEO_ENDPOINT, + providerType: 'openai-compatible', + modelManagement: 'offgrid-desktop-v1', + modelCatalog: { video: [DESKTOP_VIDEO_MODEL] }, + mediaModels: { video: DESKTOP_VIDEO_MODEL.id }, + }); + } + await openNewChat(app); + rtl.fireEvent.press(view.getByTestId('quick-settings-button')); + expect((await view.findByTestId('quick-video-mode')).props.accessibilityState.selected).toBe(false); + return app; +} + +it('with no computer paired, video offers "Find my computer", which opens where I pair one', async () => { + const { view, rtl } = await chatWithVideoOff(false); + rtl.fireEvent.press(view.getByTestId('quick-video-mode')); + await view.findByText('Videos are made on your computer'); + expect(view.queryByText(/Run models this phone cannot hold/)).toBeNull(); + rtl.fireEvent.press(view.getByText('Find my computer')); + await view.findByText(/Run models this phone cannot hold/); +}, 90_000); + +it('with a computer that makes videos paired, video turns on with no question', async () => { + const { view, rtl } = await chatWithVideoOff(true); + rtl.fireEvent.press(view.getByTestId('quick-video-mode')); + await rtl.waitFor(() => expect(view.getByTestId('quick-video-mode').props.accessibilityState.selected).toBe(true)); + expect(view.queryByText('Videos are made on your computer')).toBeNull(); +}, 90_000); diff --git a/__tests__/pro/video/yourVideosLibrary.rendered.test.tsx b/__tests__/pro/video/yourVideosLibrary.rendered.test.tsx new file mode 100644 index 000000000..939e0a30e --- /dev/null +++ b/__tests__/pro/video/yourVideosLibrary.rendered.test.tsx @@ -0,0 +1,128 @@ +/** + * Your videos: the Library behind the Gallery's Videos tab. + * + * SPEC (OGAM user's view): with Pro, the Gallery's Videos tab opens "Your videos": every story and + * clip I made, newest first, with All / Stories / Clips. A tab with nothing says so. Opening a clip + * plays it full screen; New shot there makes the next clip on my computer and shows it being made, + * then Watch plays it. Delete asks first and removes the clip from the list. The + starts a new + * video in a chat. The Studio sits at its top: one sentence starts a film, and New character opens + * the character screen. + * + * The real App runs. Device boundaries only: the llama.rn engine, the disk, the RAM sensor, the OS + * dialog, and Desktop's gateway at `fetch`. + */ +jest.unmock('@react-navigation/native'); + +import { closeStudioApp, launchStudioApp, makeClipInChat, type StudioApp } from '../../harness/videoStudioApp'; + +afterEach(() => closeStudioApp(), 90_000); + +const CLIP = 'a paper boat drifting on a pond'; + +/** From the chat: back Home, the Gallery card, then its Videos tab. */ +async function openLibrary(app: StudioApp): Promise { + const { view, rtl } = app; + rtl.fireEvent.press(view.getAllByLabelText('Back').pop()!); + rtl.fireEvent.press(await view.findByText('Image Gallery', {}, { timeout: 10_000 })); + await view.findByTestId('gallery-screen'); + rtl.fireEvent.press(view.getByText(/^Videos \(/)); + await view.findByTestId('library-grid', {}, { timeout: 10_000 }); +} + +it('lists the clip under All and Clips, not Stories, and plays it full screen', async () => { + const app = await launchStudioApp(); + const { view, rtl } = app; + await makeClipInChat(app, CLIP); + await openLibrary(app); + + expect(view.getByText(CLIP)).toBeTruthy(); + rtl.fireEvent.press(view.getByTestId('library-tab-stories')); + await view.findByText('No stories yet.'); + expect(view.queryByText(CLIP)).toBeNull(); + rtl.fireEvent.press(view.getByTestId('library-tab-clips')); + await view.findByText(CLIP); + expect(view.queryByText('No stories yet.')).toBeNull(); + + const item = view.getAllByTestId(/^library-item-/); + expect(item).toHaveLength(1); + rtl.fireEvent.press(item[0]); + await view.findByTestId('video-viewer', {}, { timeout: 10_000 }); + expect(view.getByTestId('viewer-new-shot')).toBeTruthy(); +}, 60_000); + +it('New shot from the viewer is made on the computer, then Watch plays it', async () => { + const app = await launchStudioApp(); + const { view, rtl } = app; + await makeClipInChat(app, CLIP); + await openLibrary(app); + rtl.fireEvent.press(view.getAllByTestId(/^library-item-/)[0]); + await view.findByTestId('video-viewer', {}, { timeout: 10_000 }); + + app.stories.hold(); + rtl.fireEvent.press(view.getByTestId('viewer-new-shot')); + await view.findByTestId('video-making', {}, { timeout: 10_000 }); + // The next clip is a one-shot film on the computer, from this clip's description. + await rtl.waitFor(() => expect(app.stories.submissions).toHaveLength(1), { timeout: 10_000 }); + expect(JSON.stringify(app.stories.submissions[0])).toContain(CLIP); + expect(view.queryByTestId('making-watch')).toBeNull(); + app.stories.release(); + rtl.fireEvent.press(await view.findByTestId('making-watch', {}, { timeout: 20_000 })); + // Watch goes back to the viewer, now on the new clip. + await rtl.waitFor(() => expect(view.queryByTestId('video-making')).toBeNull()); + expect(view.getByTestId('video-viewer')).toBeTruthy(); + expect(view.getByText(/new camera angle/i)).toBeTruthy(); +}, 60_000); + +it('Delete asks first, then removes the clip; + opens a chat', async () => { + const app = await launchStudioApp(); + const { view, rtl } = app; + await makeClipInChat(app, CLIP); + await openLibrary(app); + rtl.fireEvent.press(view.getAllByTestId(/^library-item-/)[0]); + rtl.fireEvent.press(await view.findByTestId('viewer-delete', {}, { timeout: 10_000 })); + expect(app.dialog()?.title).toBe('Delete this video?'); + await app.answer('Delete'); + await view.findByText('Describe a video in a chat to make your first one.', {}, { timeout: 10_000 }); + expect(view.queryByText(CLIP)).toBeNull(); + + rtl.fireEvent.press(view.getByTestId('library-new')); + await view.findByTestId('chat-input', {}, { timeout: 10_000 }); +}, 60_000); + +it('the viewer pauses on a tap and saves the clip to Downloads', async () => { + const app = await launchStudioApp(); + const { view, rtl } = app; + // Android's Downloads, at the native boundary (MediaStore through SyncDownloadsModule). + const saved: Array<{ name: string; mime: string }> = []; + require('react-native').NativeModules.SyncDownloadsModule = { + saveFileToDownloads: async (_source: string, name: string, mime: string) => { + saved.push({ name, mime }); + return true; + }, + }; + await makeClipInChat(app, CLIP); + await openLibrary(app); + rtl.fireEvent.press(view.getAllByTestId(/^library-item-/)[0]); + const toggle = await view.findByTestId('viewer-toggle', {}, { timeout: 10_000 }); + expect(toggle.props.accessibilityLabel).toBe('Pause'); + rtl.fireEvent.press(toggle); + await rtl.waitFor(() => expect(view.getByTestId('viewer-toggle').props.accessibilityLabel).toBe('Play')); + + expect(saved).toHaveLength(0); + rtl.fireEvent.press(view.getByTestId('viewer-save')); + await rtl.waitFor(() => expect(app.dialog()?.title).toBe('Video saved')); + expect(app.dialog()?.message).toBe('The video is in Downloads.'); + expect(saved).toEqual([{ name: expect.stringMatching(/\.mp4$/), mime: 'video/mp4' }]); +}, 60_000); + +it('the Studio is at the top: one sentence starts a film, and New character opens a character', async () => { + const app = await launchStudioApp(); + const { view, rtl } = app; + await makeClipInChat(app, CLIP); + await openLibrary(app); + + expect(view.getByTestId('new-film-sentence')).toBeTruthy(); + expect(view.getByTestId('new-film-go')).toBeTruthy(); + rtl.fireEvent.press(view.getByLabelText('New character')); + await view.findByTestId('character-name', {}, { timeout: 10_000 }); +}, 60_000); diff --git a/__tests__/rntl/components/GenerationSettingsModal.test.tsx b/__tests__/rntl/components/GenerationSettingsModal.test.tsx index b7cd52c88..56c869552 100644 --- a/__tests__/rntl/components/GenerationSettingsModal.test.tsx +++ b/__tests__/rntl/components/GenerationSettingsModal.test.tsx @@ -110,6 +110,8 @@ describe('GenerationSettingsModal', () => { downloadedModels: [], downloadedImageModels: [], activeImageModelId: null, + downloadedVideoModels: [], + activeVideoModelId: null, }; }); @@ -574,15 +576,15 @@ describe('GenerationSettingsModal', () => { // NEW TESTS: Performance section details // ============================================================================ - it('shows generation details toggle in text generation section', () => { + it('shows generation details toggle in the General section', () => { const { getByText } = render( , ); - fireEvent.press(getByText('TEXT GENERATION')); + fireEvent.press(getByText('GENERAL')); expect(getByText('Show Generation Details')).toBeTruthy(); - expect(getByText('Show context use, model, speed, and image settings below each reply')).toBeTruthy(); + expect(getByText('Applies to all chats. Show context use, model, speed, and image settings below each reply.')).toBeTruthy(); }); it('calls updateSettings to enable show generation details', () => { @@ -590,9 +592,9 @@ describe('GenerationSettingsModal', () => { , ); - fireEvent.press(getByText('TEXT GENERATION')); + fireEvent.press(getByText('GENERAL')); - // Find the "On" buttons in text generation section + // Find the "On" buttons in the General section const onButtons = getAllByText('On'); // The last "On" is for show generation details fireEvent.press(onButtons[onButtons.length - 1]); @@ -882,19 +884,17 @@ describe('GenerationSettingsModal', () => { it('renders GPU layers slider with gpuLayersEffective when backend is OpenCL', () => { mockStoreValues.settings = { ...defaultSettings, inferenceBackend: 'opencl' as const, gpuLayers: 8, flashAttn: false }; const { getByText, getByTestId } = render(); - fireEvent.press(getByText('TEXT GENERATION')); - fireEvent.press(getByTestId('modal-text-advanced-toggle')); - expect(getByText('8')).toBeTruthy(); + fireEvent.press(getByText('GENERAL')); + expect(getByTestId('gpu-layers-stepper-value').props.children).toBe('8'); }); it('shows GPU layers at full value when flash attention is On (no clamping)', () => { // Flash attention no longer caps GPU layers — gpuLayersMax is always 99 mockStoreValues.settings = { ...defaultSettings, inferenceBackend: 'opencl' as const, gpuLayers: 8, flashAttn: true }; const { getByText, getByTestId } = render(); - fireEvent.press(getByText('TEXT GENERATION')); - fireEvent.press(getByTestId('modal-text-advanced-toggle')); + fireEvent.press(getByText('GENERAL')); // gpuLayersEffective = Math.min(8, 99) = 8 - expect(getByText('8')).toBeTruthy(); + expect(getByTestId('gpu-layers-stepper-value').props.children).toBe('8'); }); it('uses default gpuLayers value of 1 when gpuLayers is undefined (covers ?? fallback)', () => { @@ -905,8 +905,7 @@ describe('GenerationSettingsModal', () => { flashAttn: false, }; const { getByText, getByTestId } = render(); - fireEvent.press(getByText('TEXT GENERATION')); - fireEvent.press(getByTestId('modal-text-advanced-toggle')); + fireEvent.press(getByText('GENERAL')); // gpuLayersEffective = Math.min(undefined ?? 1, 99) = 1 expect(getByTestId('gpu-layers-stepper-value').props.children).toBe('1'); }); @@ -959,8 +958,7 @@ describe('GenerationSettingsModal', () => { it('calls updateSettings with inferenceBackend: cpu when CPU button pressed', () => { mockStoreValues.settings = { ...defaultSettings, inferenceBackend: 'opencl' as const }; const { getByText, getByTestId } = render(); - fireEvent.press(getByText('TEXT GENERATION')); - fireEvent.press(getByTestId('modal-text-advanced-toggle')); + fireEvent.press(getByText('GENERAL')); mockUpdateSettings.mockClear(); fireEvent.press(getByTestId('backend-cpu-button')); @@ -971,8 +969,7 @@ describe('GenerationSettingsModal', () => { it('calls updateSettings with inferenceBackend: opencl when OpenCL button pressed on Android', () => { mockStoreValues.settings = { ...defaultSettings, inferenceBackend: 'cpu' as const }; const { getByText, getByTestId } = render(); - fireEvent.press(getByText('TEXT GENERATION')); - fireEvent.press(getByTestId('modal-text-advanced-toggle')); + fireEvent.press(getByText('GENERAL')); mockUpdateSettings.mockClear(); fireEvent.press(getByTestId('backend-opencl-button')); @@ -983,8 +980,7 @@ describe('GenerationSettingsModal', () => { it('calls updateSettings with gpuLayers value from GPU layers slider', () => { mockStoreValues.settings = { ...defaultSettings, inferenceBackend: 'opencl' as const, gpuLayers: 6, flashAttn: false }; const { getByText, getByTestId } = render(); - fireEvent.press(getByText('TEXT GENERATION')); - fireEvent.press(getByTestId('modal-text-advanced-toggle')); + fireEvent.press(getByText('GENERAL')); mockUpdateSettings.mockClear(); fireEvent(getByTestId('gpu-layers-stepper-slider'), 'slidingComplete', 7); @@ -1012,7 +1008,7 @@ describe('GenerationSettingsModal', () => { , ); - fireEvent.press(getByText('TEXT GENERATION')); + fireEvent.press(getByText('GENERAL')); mockUpdateSettings.mockClear(); // Find and press the Off button that sets showGenerationDetails diff --git a/__tests__/rntl/components/ModelsManagerSheet.test.tsx b/__tests__/rntl/components/ModelsManagerSheet.test.tsx index 50777e963..1e42c44c8 100644 --- a/__tests__/rntl/components/ModelsManagerSheet.test.tsx +++ b/__tests__/rntl/components/ModelsManagerSheet.test.tsx @@ -21,7 +21,7 @@ jest.mock('../../../src/components/AnimatedPressable', () => { import { ModelsManagerSheet } from '../../../src/components/models/ModelsManagerSheet'; -const labels = { text: 'Qwen3.5', image: '—', voice: 'Kokoro', speech: 'Base' }; +const labels = { text: 'Qwen3.5', image: '—', video: '—', voice: 'Kokoro', speech: 'Base' }; const baseProps = { visible: true, onClose: jest.fn(), diff --git a/__tests__/rntl/components/ModelsSummaryRow.test.tsx b/__tests__/rntl/components/ModelsSummaryRow.test.tsx index cdb1d01ba..72eeb7fb6 100644 --- a/__tests__/rntl/components/ModelsSummaryRow.test.tsx +++ b/__tests__/rntl/components/ModelsSummaryRow.test.tsx @@ -16,7 +16,7 @@ jest.mock('../../../src/components/AnimatedPressable', () => { import { ModelsSummaryRow } from '../../../src/components/models/ModelsSummaryRow'; -const labels = { text: 'Qwen3.5', image: '—', voice: 'Kokoro · Warm', speech: 'Base' }; +const labels = { text: 'Qwen3.5', image: '—', video: '—', voice: 'Kokoro · Warm', speech: 'Base' }; describe('ModelsSummaryRow', () => { it('renders the four model-type captions and the label', () => { diff --git a/__tests__/rntl/components/VoiceModelsPanel.test.tsx b/__tests__/rntl/components/VoiceModelsPanel.test.tsx index b0febee37..dac8501b6 100644 --- a/__tests__/rntl/components/VoiceModelsPanel.test.tsx +++ b/__tests__/rntl/components/VoiceModelsPanel.test.tsx @@ -93,7 +93,7 @@ describe('VoiceModelsPanel', () => { it('shows the RAM privacy banner', async () => { const { getByText } = await renderPanel(); - expect(getByText(/nothing is sent anywhere/)).toBeTruthy(); + expect(getByText(/your voice never leaves it/)).toBeTruthy(); }); it('shows the installed model without speaker choices', async () => { diff --git a/__tests__/rntl/components/companionToolsSection.test.tsx b/__tests__/rntl/components/companionToolsSection.test.tsx deleted file mode 100644 index 6a083d199..000000000 --- a/__tests__/rntl/components/companionToolsSection.test.tsx +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Integration (RNTL): CompanionToolsSection - the single home for desktop tools. - * - * Proves the moved grant: paired desktops (only desktops) appear in Pro tools with a - * switch that reflects whether their tools are connected here (grantedByDeviceId), and - * flipping one calls requestTools(deviceId, next) - the same mesh request the old - * Devices-row toggle sent. Loaded via a computed path so it skips where pro/ is absent. - */ - -import React from 'react'; -import { render, fireEvent } from '@testing-library/react-native'; - -jest.mock('react-native-vector-icons/Feather', () => { - const { Text } = require('react-native'); - return ({ name, ...props }: any) => {name}; -}); - -jest.mock('../../../src/theme', () => ({ - useTheme: () => ({ - colors: { - text: '#000', textMuted: '#999', primary: '#1DB954', surface: '#F5F5F5', border: '#E0E0E0', - }, - }), -})); - -const mockState: { knownDevices: unknown[]; servers: unknown[] } = { knownDevices: [], servers: [] }; -const mockRequestTools = jest.fn(); - -jest.mock('../../../pro/sync/syncStore', () => ({ - useSyncStore: (selector: (s: unknown) => unknown) => - selector({ knownDevices: mockState.knownDevices }), -})); -jest.mock('../../../pro/mcp/mcpStore', () => ({ - useMcpStore: (selector: (s: unknown) => unknown) => selector({ servers: mockState.servers }), -})); -jest.mock('../../../pro/mcp/mcpToolGrantService', () => ({ - requestTools: (...args: unknown[]) => mockRequestTools(...args), -})); - -type Mod = typeof import('../../../pro/ui/CompanionToolsSection'); -function load(): Mod | null { - try { - return require(['..', '..', '..', 'pro', 'ui', 'CompanionToolsSection'].join('/')); - } catch { - return null; - } -} - -const mod = load(); -const maybe = mod ? describe : describe.skip; - -maybe('CompanionToolsSection', () => { - const { CompanionToolsSection } = mod!; - - beforeEach(() => { - mockState.knownDevices = []; - mockState.servers = []; - mockRequestTools.mockClear(); - }); - - it('lists only desktop peers, reflects the grant, and toggles via requestTools', () => { - mockState.knownDevices = [ - { id: 'mac1', name: 'My Mac', platform: 'macos' }, - { id: 'phone1', name: 'My Phone', platform: 'ios' }, - ]; - mockState.servers = [{ id: 's1', grantedByDeviceId: 'mac1' }]; - - const { getByTestId, queryByTestId } = render(); - // Desktop shows; a phone peer (serves no tools) is filtered out. - expect(getByTestId('companion-tools-mac1')).toBeTruthy(); - expect(queryByTestId('companion-tools-phone1')).toBeNull(); - - const sw = getByTestId('companion-tools-switch-mac1'); - expect(sw.props.value).toBe(true); // granted -> on - fireEvent(sw, 'valueChange', false); - expect(mockRequestTools).toHaveBeenCalledWith('mac1', false); - }); - - it('renders nothing when there are no paired desktops', () => { - mockState.knownDevices = [{ id: 'phone1', name: 'Phone', platform: 'android' }]; - const { toJSON } = render(); - expect(toJSON()).toBeNull(); - }); -}); diff --git a/__tests__/rntl/screens/DownloadManagerScreen.test.tsx b/__tests__/rntl/screens/DownloadManagerScreen.test.tsx index f7306e982..9a8acdf21 100644 --- a/__tests__/rntl/screens/DownloadManagerScreen.test.tsx +++ b/__tests__/rntl/screens/DownloadManagerScreen.test.tsx @@ -240,6 +240,8 @@ const createDefaultState = (overrides: any = {}) => ({ downloadedImageModels: [], setDownloadedImageModels: jest.fn(), removeDownloadedImageModel: jest.fn(), + downloadedVideoModels: [], + removeDownloadedVideoModel: jest.fn(), themeMode: 'system', ...overrides, }); diff --git a/__tests__/rntl/screens/GalleryScreen.test.tsx b/__tests__/rntl/screens/GalleryScreen.test.tsx index 0c307b240..9047b2bad 100644 --- a/__tests__/rntl/screens/GalleryScreen.test.tsx +++ b/__tests__/rntl/screens/GalleryScreen.test.tsx @@ -313,7 +313,7 @@ describe('GalleryScreen', () => { } }); - it('shows "Chat Images" title when conversationId is provided', () => { + it('shows "Chat Media" title when conversationId is provided', () => { mockRouteParams = { conversationId: 'conv-123' }; mockGeneratedImages.push({ ...sampleImages[0], @@ -321,7 +321,7 @@ describe('GalleryScreen', () => { }); const { getByText } = render(); - expect(getByText('Chat Images')).toBeTruthy(); + expect(getByText('Chat Media')).toBeTruthy(); }); it('shows chat-specific empty state when no images match conversation', () => { @@ -452,18 +452,9 @@ describe('GalleryScreen', () => { fireEvent.press(result.getByText('All')); expect(result.getByText('3 selected')).toBeTruthy(); - // In select mode, the header touchables (non-grid) are: - // [X close button, "All" text button, trash icon button] - // The trash button is the one with disabled={false} (items selected) - // and is NOT the All button or X button. - const allTouchables = result.UNSAFE_getAllByType(TouchableOpacity); - const nonGridTouchables = allTouchables.filter((t: any) => t.props.activeOpacity !== 0.8); - // The last non-grid touchable before grid items should be the trash button - // Try pressing from the last non-grid touchable backwards until handleDeleteSelected fires - for (let i = nonGridTouchables.length - 1; i >= 0; i--) { - fireEvent.press(nonGridTouchables[i]); - if (mockShowAlert.mock.calls.length > 0) break; - } + // The header's trash button, by its label (pressing every header button in turn would also + // press the Videos tab, which ends image select mode). + fireEvent.press(result.getByLabelText('Delete selected images')); expect(mockShowAlert).toHaveBeenCalledWith( 'Delete Images', diff --git a/__tests__/rntl/screens/ModelSettingsScreen.test.tsx b/__tests__/rntl/screens/ModelSettingsScreen.test.tsx index c25aa4852..1eeccfc34 100644 --- a/__tests__/rntl/screens/ModelSettingsScreen.test.tsx +++ b/__tests__/rntl/screens/ModelSettingsScreen.test.tsx @@ -59,6 +59,9 @@ const renderWithSections = (...sections: ('prompt' | 'image' | 'text')[]) => { text: 'text-advanced-toggle', }; for (const section of sections) { + if (section === 'text') { + fireEvent.press(result.getByTestId('general-generation-accordion')); + } fireEvent.press(result.getByTestId(testIDMap[section])); if (advancedMap[section]) { fireEvent.press(result.getByTestId(advancedMap[section])); @@ -179,7 +182,7 @@ describe('ModelSettingsScreen', () => { expect(getByText('Show Generation Details')).toBeTruthy(); expect( getByText( - 'Show context use, model, speed, and image settings below each reply', + 'Applies to all chats. Show context use, model, speed, and image settings below each reply.', ), ).toBeTruthy(); }); @@ -618,15 +621,15 @@ describe('ModelSettingsScreen', () => { // ============================================================================ describe('GPU settings', () => { // Platform.OS is 'ios' in the test environment; Metal is the default backend, GPU Layers hidden when cpu - it('shows Inference Backend section on iOS', () => { + it('shows Text Model Backend section on iOS', () => { const { getByText } = renderWithSections('text'); - expect(getByText('Inference Backend')).toBeTruthy(); + expect(getByText('Text Model Backend')).toBeTruthy(); }); it('does not show GPU Layers on iOS when backend is cpu', () => { useAppStore.getState().updateSettings({ inferenceBackend: 'cpu' }); - const { queryByText } = renderWithSections('text'); - expect(queryByText('GPU Layers')).toBeNull(); + const { queryByTestId } = renderWithSections('text'); + expect(queryByTestId('gpu-layers-stepper-slider')).toBeNull(); }); // Android-specific backend tests: mock Platform.OS before each, restore after @@ -649,12 +652,12 @@ describe('ModelSettingsScreen', () => { }); }); - it('shows Inference Backend section and GPU Layers slider when backend is OpenCL', () => { + it('shows Text Model Backend section and GPU Layers slider when backend is OpenCL', () => { useAppStore .getState() .updateSettings({ inferenceBackend: 'opencl', gpuLayers: 6 }); const { getByText, getByTestId } = renderWithSections('text'); - expect(getByText('Inference Backend')).toBeTruthy(); + expect(getByText('Text Model Backend')).toBeTruthy(); // Label now names the backend: "GPU Layers (OpenCL)". Assert the slider by its // SliderSetting child testID (SliderSetting suffixes testID with -slider/-value). expect(getByTestId('gpu-layers-stepper-slider')).toBeTruthy(); @@ -941,6 +944,14 @@ describe('ModelSettingsScreen', () => { liteRTTemperature: undefined as any, liteRTTopP: undefined as any, liteRTMaxTokens: undefined as any, + analyticsUsage: false, + analyticsCrashes: false, + analyticsAsked: false, + analyticsLastPromptAt: null, + videoParams: undefined as any, + videoSeed: undefined as any, + videoNegative: undefined as any, + enhanceVideoPrompts: undefined as any, }, }); diff --git a/__tests__/rntl/screens/SettingsScreen.test.tsx b/__tests__/rntl/screens/SettingsScreen.test.tsx index 1930ba0e2..f2edba3d4 100644 --- a/__tests__/rntl/screens/SettingsScreen.test.tsx +++ b/__tests__/rntl/screens/SettingsScreen.test.tsx @@ -105,7 +105,7 @@ describe('SettingsScreen', () => { it('shows the Pro upsell banner when Pro is not active and not dismissed', () => { const { getByText } = render(); expect( - getByText('Your private AI stays current across your devices with live sync.'), + getByText('Keep your work in context with memory, Sync, and actions you approve.'), ).toBeTruthy(); }); @@ -184,12 +184,13 @@ describe('SettingsScreen', () => { // The three theme options are rendered - pressing one calls setThemeMode }); - it('renders Privacy First section', () => { - const { getByText } = render(); - expect(getByText('Privacy First')).toBeTruthy(); - expect( - getByText(/All your data stays on this device/), - ).toBeTruthy(); + it('renders the Analytics section with both consent toggles off by default', () => { + const { getByText, getByLabelText } = render(); + expect(getByText('Analytics')).toBeTruthy(); + expect(Boolean(getByLabelText('Usage data').props.on ?? getByLabelText('Usage data').props.value)).toBe(false); + expect(Boolean(getByLabelText('Crash reports').props.on ?? getByLabelText('Crash reports').props.value)).toBe(false); + expect(getByText('What we share')).toBeTruthy(); + expect(getByText('Report a problem')).toBeTruthy(); }); it('renders about section text', () => { diff --git a/__tests__/rntl/screens/StorageSettingsScreen.test.tsx b/__tests__/rntl/screens/StorageSettingsScreen.test.tsx index a447ee308..a74e161b7 100644 --- a/__tests__/rntl/screens/StorageSettingsScreen.test.tsx +++ b/__tests__/rntl/screens/StorageSettingsScreen.test.tsx @@ -87,13 +87,24 @@ let mockDownloadedImageModels: any[] = []; let mockConversations: any[] = []; let mockStaleDownloadStoreEntries: any[] = []; const mockRemoveFromStore = jest.fn(); +const mockNoVideos: any[] = []; +const mockNoImages: any[] = []; +const mockAppState = () => ({ + downloadedModels: mockDownloadedModels, + downloadedImageModels: mockDownloadedImageModels, + downloadedVideoModels: mockNoVideos, + generatedImages: mockNoImages, + generatedVideos: mockNoVideos, +}); jest.mock('../../../src/stores', () => ({ - useAppStore: jest.fn(() => ({ - downloadedModels: mockDownloadedModels, - downloadedImageModels: mockDownloadedImageModels, - generatedImages: [], - })), + // The screen reads the store whole, through selectors and through getState (the video count and + // the storage breakdown, 8bc6073e3), as the real store allows. Empty lists keep one identity, as + // the real store's do between renders. + useAppStore: Object.assign( + jest.fn((selector?: any) => (selector ? selector(mockAppState()) : mockAppState())), + { getState: () => mockAppState() }, + ), useChatStore: jest.fn((selector?: any) => { const state = { conversations: mockConversations }; return selector ? selector(state) : state; diff --git a/__tests__/rntl/screens/ToolsScreen.switches.test.tsx b/__tests__/rntl/screens/ToolsScreen.switches.test.tsx index b5d013bb1..01a780c90 100644 --- a/__tests__/rntl/screens/ToolsScreen.switches.test.tsx +++ b/__tests__/rntl/screens/ToolsScreen.switches.test.tsx @@ -12,8 +12,11 @@ import { render, fireEvent, screen } from '@testing-library/react-native'; import { ToolsScreen } from '../../../src/screens/ToolsScreen'; import { AVAILABLE_TOOLS } from '../../../src/services/tools/registry'; +// Video tools are Pro (d51e111c1, a3f6bd8cf): without Pro video the screen offers the free tools. +const FREE_TOOLS = AVAILABLE_TOOLS.filter(tool => !['generate_video', 'make_film'].includes(tool.id)); + function expectEverySwitchMatchesItsLabel() { - for (const tool of AVAILABLE_TOOLS) { + for (const tool of FREE_TOOLS) { const toggle = screen.getByTestId(`tool-picker-toggle-${tool.id}`); const label: string = toggle.props.accessibilityLabel; expect(label === `${tool.displayName}, ON` || label === `${tool.displayName}, OFF`).toBe(true); diff --git a/__tests__/rntl/screens/ToolsScreen.test.tsx b/__tests__/rntl/screens/ToolsScreen.test.tsx index 757d0858b..8313a234b 100644 --- a/__tests__/rntl/screens/ToolsScreen.test.tsx +++ b/__tests__/rntl/screens/ToolsScreen.test.tsx @@ -13,6 +13,9 @@ import React from 'react'; import { render, fireEvent } from '@testing-library/react-native'; import { ToolsScreen } from '../../../src/screens/ToolsScreen'; import { AVAILABLE_TOOLS } from '../../../src/services/tools/registry'; + +const PRO_VIDEO_TOOL_IDS = ['generate_video', 'make_film']; +const FREE_TOOLS = AVAILABLE_TOOLS.filter(tool => !PRO_VIDEO_TOOL_IDS.includes(tool.id)); import { registerScreen, _clearScreensForTesting } from '../../../src/navigation/screenRegistry'; import { PRO_TOOLS_SCREEN } from '../../../src/hooks/useIsProActive'; import { useAppStore } from '../../../src/stores/appStore'; @@ -82,11 +85,15 @@ describe('ToolsScreen', () => { }); it('renders the page header and every free tool row', () => { - const { getByText, getByTestId } = render(); + const { getByText, getByTestId, queryByTestId } = render(); expect(getByText('Tools')).toBeTruthy(); - for (const tool of AVAILABLE_TOOLS) { + for (const tool of FREE_TOOLS) { expect(getByTestId(`tool-picker-row-${tool.id}`)).toBeTruthy(); } + // Video tools are Pro (d51e111c1, a3f6bd8cf): without Pro video they are not offered. + for (const id of PRO_VIDEO_TOOL_IDS) { + expect(queryByTestId(`tool-picker-row-${id}`)).toBeNull(); + } }); it('shows the Pro Tools row to everyone (even free users)', () => { @@ -115,10 +122,9 @@ describe('ToolsScreen', () => { }); it('toggles a tool through the core settings store', () => { - const { getAllByRole } = render(); - const switches = getAllByRole('switch'); + const { getByTestId } = render(); // web_search + calculator are enabled, get_current_datetime is not. - fireEvent(switches[2], 'valueChange', true); + fireEvent(getByTestId('tool-picker-toggle-get_current_datetime'), 'valueChange', true); expect(mockUpdateSettings).toHaveBeenCalledWith({ enabledTools: ['web_search', 'calculator', 'get_current_datetime'], }); diff --git a/__tests__/services/autoSetupCatalog.test.ts b/__tests__/services/autoSetupCatalog.test.ts index edc49f213..e20dbaf6f 100644 --- a/__tests__/services/autoSetupCatalog.test.ts +++ b/__tests__/services/autoSetupCatalog.test.ts @@ -1,35 +1,25 @@ -import { RECOMMENDED_MODELS } from '../../src/constants'; -import { buildAutoSetupTextCandidates } from '../../src/services/autoSetupCatalog'; +import { Platform } from 'react-native'; +import { loadAutoSetupCompatibleCatalog } from '../../src/services/autoSetupCatalog'; import { selectAutoSetupPlans } from '../../src/services/autoSetupPlan'; -import { recommendedModelsForDevice } from '../../src/utils/recommendedModels'; -const GB = 1024 ** 3; - -// The 9B with its projector (about 6.2 GB) is past the phone fit ceiling: on this phone the app had -// 6.0 GB free and a 4.64 GB model needed 6.4 GB to load. The phone sees two plans, never one twice. -// The catalog offers Qwen 3.5 4B (the shared catalog is the list now): it is the 4B choice, ahead of -// Gemma 4 E4B. -test('an iPhone 17 Pro Max gets 2B, Qwen 3.5 4B and 9B setup choices, each model once', () => { - const files = Object.fromEntries(RECOMMENDED_MODELS.map(model => [model.id, [{ - name: `${model.params}B-Q4_K_M.gguf`, - size: model.params * 0.6 * GB, - quantization: 'Q4_K_M', - downloadUrl: `https://boundary.test/${model.id}`, - mmProjFile: { - name: 'mmproj-F16.gguf', - size: 0.8 * GB, - downloadUrl: `https://boundary.test/${model.id}/mmproj`, - }, - }]])); - - const catalog = buildAutoSetupTextCandidates(recommendedModelsForDevice(12), files, 12); +// Setup's Lean, Balanced and Extreme are this phone's Fast, Balanced and Best picks from the catalog +// (37759c684), ranked by quality alone (shared afdb05f) on the iPhone budget (shared ddaa6c1): on a +// 12 GB iPhone 17 Pro Max the 9B is the Best pick, and no plan repeats another's chat model. +test('an iPhone 17 Pro Max gets its Fast, Balanced and Best picks as setup choices, each model once', async () => { + expect(Platform.OS).toBe('ios'); + const catalog = await loadAutoSetupCompatibleCatalog({ + totalMemoryGB: () => 12, + fetchTextFiles: async () => ({}), + imageRecommendation: async () => ({ compatibleBackends: ['coreml'], recommendedBackend: 'coreml', bannerText: '' }), + imageModels: async () => [], + }); const image = [{ id: 'image', name: 'Image', kind: 'image' as const, sizeBytes: 1, fitScore: 0, payload: {} as never }]; const stt = [{ id: 'stt', name: 'Speech', kind: 'stt' as const, sizeBytes: 1, fitScore: 0, payload: { modelId: 'stt' } }]; - const plans = selectAutoSetupPlans({ text: catalog, image, stt }); + const plans = selectAutoSetupPlans({ text: catalog.text, image, stt }); expect(plans.map(plan => plan.items[0].name)).toEqual([ - 'Qwen 3.5 2B', - 'Qwen 3.5 4B', + 'LFM2.5 8B A1B', + 'MiniCPM5 2B', 'Qwen 3.5 9B', ]); }); diff --git a/__tests__/services/autoSetupPlan.test.ts b/__tests__/services/autoSetupPlan.test.ts index 4cdaaaa1b..58d14ae84 100644 --- a/__tests__/services/autoSetupPlan.test.ts +++ b/__tests__/services/autoSetupPlan.test.ts @@ -9,6 +9,7 @@ const candidate = ( sizeBytes: number, fitScore: number, parameterCountB?: number, + pick?: 'fast' | 'balanced' | 'best', ) => ({ id: `${kind}-${name}`, name, @@ -16,6 +17,7 @@ const candidate = ( sizeBytes, fitScore, parameterCountB, + pick, payload: kind === 'text' ? { @@ -43,9 +45,9 @@ const candidate = ( test('Lean, Balanced, and Extreme select only from the compatible device catalog', () => { const catalog = { text: [ - candidate('text', 'small', 100, 4, 2), - candidate('text', 'fit', 200, 0, 4), - candidate('text', 'large', 300, 2, 9), + candidate('text', 'small', 100, 4, 2, 'fast'), + candidate('text', 'fit', 200, 0, 4, 'balanced'), + candidate('text', 'large', 300, 2, 9, 'best'), ], image: [ candidate('image', 'small', 10, 4), @@ -70,13 +72,13 @@ test('Lean, Balanced, and Extreme select only from the compatible device catalog expect(plans[1].totalBytes).toBe(222); }); -test('text plans target 2B, 4B, and 9B instead of file-size order', () => { +test("text plans are the phone's Fast, Balanced and Best picks, not file-size order", () => { const catalog = { text: [ candidate('text', 'Qwen 0.8B', 80, 0, 0.8), - candidate('text', 'Qwen 9B', 500, 3, 9), - candidate('text', 'Gemma 4B', 900, 2, 4), - candidate('text', 'Gemma 2B', 700, 1, 2), + candidate('text', 'Qwen 9B', 500, 3, 9, 'best'), + candidate('text', 'Gemma 4B', 900, 2, 4, 'balanced'), + candidate('text', 'Gemma 2B', 700, 1, 2, 'fast'), ], image: [candidate('image', 'image', 10, 0)], stt: [candidate('stt', 'speech', 1, 0)], diff --git a/__tests__/unit/analytics/scrub.test.ts b/__tests__/unit/analytics/scrub.test.ts new file mode 100644 index 000000000..87d5263f0 --- /dev/null +++ b/__tests__/unit/analytics/scrub.test.ts @@ -0,0 +1,98 @@ +/** + * Crash report scrubbing (docs/ANALYTICS.md, Crashes): a path keeps only an app code file name, + * a person's file becomes ``, quoted text, URLs and IP addresses are dropped, and anything + * after a colon goes. Type, file, line and function stay. + */ +import { fileName, scrubExceptionProperties, scrubMessage } from '../../../src/services/analytics/scrub'; + +describe('crash report scrubbing', () => { + it.each([ + ['Cannot read /var/mobile/Containers/Data/notes/secret.txt', 'Cannot read '], + ['Bad token in /data/app/index.android.bundle', 'Bad token in index.android.bundle'], + ['Could not open Taxes2025.pdf now', 'Could not open now'], + ['Could not open Taxes2025.pdf, then retried', 'Could not open , then retried'], + ['Failed to load qwen-private-finetune.gguf', 'Failed to load '], + ['Failed to parse "my diary entry" here', 'Failed to parse here'], + ["TypeError: undefined is not a function", 'TypeError'], + ['fetch https://my.server.lan/v1/chat?q=private failed', 'fetch failed'], + ['connect 192.168.1.20:8080 refused', 'connect refused'], + ['Plain message', 'Plain message'], + ])('%p → %p', (raw, clean) => { + expect(scrubMessage(raw)).toBe(clean); + }); + + it('only scrubs text', () => { + expect(scrubMessage(undefined)).toBeUndefined(); + expect(fileName('/a/b/c.js?x=1#y')).toBe('c.js'); + expect(fileName('C:\\Users\\me\\app.js')).toBe('app.js'); + expect(fileName('')).toBeUndefined(); + expect(fileName('/a/b/')).toBeUndefined(); + expect(fileName(42)).toBeUndefined(); + }); + + it('keeps type, file name, line and function, and drops every other $exception property', () => { + const scrubbed = scrubExceptionProperties({ + platform: 'android', + $exception_level: 'fatal', + $exception_message: 'raw "secret" text', + $exception_steps: [{ message: 'typed something' }], + $exception_list: [ + { + type: 'Error', + value: 'Could not open "/Users/me/Taxes 2025.pdf": denied', + mechanism: { handled: false }, + stacktrace: { + type: 'raw', + frames: [ + { + platform: 'hermes', + abs_path: '/data/app/index.android.bundle', + function: 'openFile', + lineno: 12, + colno: 4, + in_app: true, + context_line: 'const secret = "x"', + vars: { prompt: 'hello' }, + }, + ], + }, + }, + { type: 'TypeError', value: 'x is undefined' }, + ], + }); + expect(scrubbed).toEqual({ + platform: 'android', + $exception_level: 'fatal', + $exception_list: [ + { + type: 'Error', + value: 'Could not open', + mechanism: { handled: false }, + stacktrace: { + type: 'raw', + frames: [ + { platform: 'hermes', filename: 'index.android.bundle', function: 'openFile', lineno: 12, colno: 4, in_app: true }, + ], + }, + }, + { type: 'TypeError', value: 'x is undefined', mechanism: undefined }, + ], + }); + }); + + it('a frame list without a type is still raw, and no list stays absent', () => { + expect( + scrubExceptionProperties({ $exception_list: [{ type: 'E', stacktrace: { frames: [{ filename: 'a/b.js' }] } }] }), + ).toEqual({ + $exception_list: [ + { + type: 'E', + value: undefined, + mechanism: undefined, + stacktrace: { type: 'raw', frames: [{ platform: undefined, filename: 'b.js', function: undefined, lineno: undefined, colno: undefined, in_app: undefined }] }, + }, + ], + }); + expect(scrubExceptionProperties({ a: 1 })).toEqual({ a: 1 }); + }); +}); diff --git a/__tests__/unit/analytics/uncleanExit.test.ts b/__tests__/unit/analytics/uncleanExit.test.ts new file mode 100644 index 000000000..dde93c70b --- /dev/null +++ b/__tests__/unit/analytics/uncleanExit.test.ts @@ -0,0 +1,35 @@ +/** + * The unclean-exit marker never reports this launch's own work as the previous launch's death. + */ +import AsyncStorage from '@react-native-async-storage/async-storage'; +import { markStageEnded, markStageStarted, takeUncleanExit } from '../../../src/services/analytics/uncleanExit'; + +it("a load already running in this launch is not the last launch's death, and its marker stays", async () => { + markStageStarted({ stage: 'load', model: 'custom', engine: 'llama.cpp', backend: 'cpu', startedAt: Date.now(), appVersion: '1.0.0' }); + await new Promise(resolve => setTimeout(resolve, 0)); + expect(await takeUncleanExit('1.0.0')).toBeNull(); + expect(JSON.parse((await AsyncStorage.getItem('offgrid.analytics.uncleanExit'))!).stage).toBe('load'); + markStageEnded('load'); + await new Promise(resolve => setTimeout(resolve, 0)); + expect(await AsyncStorage.getItem('offgrid.analytics.uncleanExit')).toBeNull(); +}); + +it('a marker from an earlier version is cleared and not reported', async () => { + await AsyncStorage.setItem( + 'offgrid.analytics.uncleanExit', + JSON.stringify({ stage: 'generate', model: 'custom', engine: 'llama.cpp', backend: 'cpu', startedAt: 1, appVersion: '0.9.0', foreground: true, launch: 'earlier' }), + ); + // An update happened in between: a different version's death is not reported. + expect(await takeUncleanExit('1.0.0')).toBeNull(); + expect(await AsyncStorage.getItem('offgrid.analytics.uncleanExit')).toBeNull(); +}); + +it("a read that races this launch's own marker write never deletes the new marker", async () => { + const earlier = JSON.stringify({ stage: 'load', model: 'custom', engine: 'llama.cpp', backend: 'cpu', startedAt: 1, appVersion: '1.0.0', foreground: true, launch: 'earlier' }); + markStageStarted({ stage: 'generate', model: 'custom', engine: 'llama.cpp', backend: 'cpu', startedAt: Date.now(), appVersion: '1.0.0' }); + // The disk read lands before this launch's write: it still holds the earlier launch's marker. + (AsyncStorage.getItem as jest.Mock).mockResolvedValueOnce(earlier); + expect(await takeUncleanExit('1.0.0')).toMatchObject({ stage: 'load', launch: 'earlier' }); + expect(JSON.parse((await AsyncStorage.getItem('offgrid.analytics.uncleanExit'))!).stage).toBe('generate'); + markStageEnded('generate'); +}); diff --git a/__tests__/unit/analytics/values.test.ts b/__tests__/unit/analytics/values.test.ts new file mode 100644 index 000000000..cecf94349 --- /dev/null +++ b/__tests__/unit/analytics/values.test.ts @@ -0,0 +1,117 @@ +/** + * The spec's value rules (docs/ANALYTICS.md, "Shared value rules"): pure functions, checked as + * input → output tables. + */ +import { + batteryPct, + bytesToGb, + computeBackend, + contextBucket, + errorCode, + fileExtension, + ignore, + languageOnly, + modelRef, + ms, + oneDecimal, + releaseChannel, + sizeBucket, + toolFeature, +} from '../../../src/services/analytics/values'; + +const GB = 1024 * 1024 * 1024; + +describe('analytics value rules', () => { + it.each([ + ['owner/repo/file.gguf', undefined, 'owner/repo'], + ['anything', 'unsloth/Qwen3-GGUF', 'unsloth/Qwen3-GGUF'], + ['m', 'unsloth/Qwen3-GGUF/sub/path', 'unsloth/Qwen3-GGUF'], + ['local_import/my secret notes.gguf', undefined, 'custom'], + ['local_my_zip_123', undefined, 'custom'], + ['recovered_family.gguf', undefined, 'custom'], + ['anythingv5_cpu', undefined, 'anythingv5_cpu'], + ['image:anythingv5_cpu', undefined, 'anythingv5_cpu'], + ['image:local_my_zip_123', undefined, 'custom'], + ['has spaces in it', undefined, 'custom'], + [null, undefined, 'custom'], + [undefined, 'not a repo', 'custom'], + ])('model %p (origin %p) is sent as %p', (id, repo, expected) => { + expect(modelRef(id as string | null | undefined, repo)).toBe(expected); + }); + + it.each([ + [undefined, undefined], + [0, undefined], + [0.5 * GB, '<1GB'], + [1.5 * GB, '1-2GB'], + [3 * GB, '2-4GB'], + [6 * GB, '4-8GB'], + [9 * GB, '>8GB'], + ])('size %p is bucketed as %p', (bytes, bucket) => { + expect(sizeBucket(bytes)).toBe(bucket); + }); + + it.each([ + [undefined, undefined], + [0, undefined], + [1000, '<2k'], + [4000, '2-8k'], + [16000, '8-32k'], + [64000, '>32k'], + ])('context of %p tokens is bucketed as %p', (tokens, bucket) => { + expect(contextBucket(tokens)).toBe(bucket); + }); + + it('rounds durations, rates, sizes and battery as the spec says', () => { + expect(ms(12.6)).toBe(13); + expect(ms(undefined)).toBeUndefined(); + expect(ms(Number.NaN)).toBeUndefined(); + expect(oneDecimal(12.34)).toBe(12.3); + expect(oneDecimal(null)).toBeUndefined(); + expect(bytesToGb(1.26 * GB)).toBe(1.3); + expect(bytesToGb(undefined)).toBeUndefined(); + expect(batteryPct(0.64)).toBe(60); + expect(batteryPct(-1)).toBeUndefined(); + expect(batteryPct(undefined)).toBeUndefined(); + }); + + it('error codes come from our error classes, never a message', () => { + expect(errorCode(Object.assign(new Error('Not enough space for /Users/me/x'), { code: 'disk_full' }))).toBe('disk_full'); + class OverridableMemoryError extends Error { + name = 'OverridableMemoryError'; + } + expect(errorCode(new OverridableMemoryError('my private prompt'))).toBe('OverridableMemoryError'); + expect(errorCode(new Error('my private prompt'))).toBe('error'); + expect(errorCode({ code: 'has spaces: and text' })).toBe('error'); + expect(errorCode('a raw message')).toBe('error'); + expect(errorCode(undefined)).toBe('error'); + }); + + it('keeps the language only, the channel, the extension and the feature', () => { + expect(languageOnly('en-GB')).toBe('en'); + expect(languageOnly('pt_BR')).toBe('pt'); + expect(languageOnly(undefined)).toBe('unknown'); + expect(releaseChannel('0.0.117')).toBe('stable'); + expect(releaseChannel('0.0.117-beta.1')).toBe('nightly'); + expect(fileExtension('Quarterly Plan.PDF')).toBe('pdf'); + expect(fileExtension('no-extension')).toBeUndefined(); + expect(fileExtension(undefined)).toBeUndefined(); + expect(toolFeature('web_search')).toBe('web_search'); + expect(toolFeature('read_url')).toBe('web_search'); + expect(toolFeature('clipboard_read')).toBe('clipboard'); + expect(toolFeature('browser_open')).toBe('browser_action'); + expect(toolFeature('calculator')).toBe('tools'); + }); + + it.each([ + ['htp', 'npu'], ['npu', 'npu'], ['qnn', 'npu'], + ['metal', 'gpu'], ['opencl', 'gpu'], ['gpu', 'gpu'], ['coreml', 'gpu'], + ['cpu', 'cpu'], ['mnn', 'cpu'], [undefined, 'cpu'], + ])('backend setting %p is %p', (setting, backend) => { + expect(computeBackend(setting)).toBe(backend); + }); + + it('a dropped failure resolves to nothing', () => { + expect(ignore()).toBeUndefined(); + }); +}); diff --git a/__tests__/unit/navigation/useProExpiryRedirect.test.tsx b/__tests__/unit/navigation/useProExpiryRedirect.test.tsx index dcb02a4ab..bc0af07e2 100644 --- a/__tests__/unit/navigation/useProExpiryRedirect.test.tsx +++ b/__tests__/unit/navigation/useProExpiryRedirect.test.tsx @@ -29,7 +29,7 @@ describe('the expired Pro purchase route', () => { await waitFor(() => expect(navigation.resetRoot).toHaveBeenCalledWith({ index: 1, - routes: [{ name: 'Main' }, { name: 'ProDetail' }], + routes: [{ name: 'Main', params: { screen: 'HomeTab' } }, { name: 'ProDetail' }], }), ); }); @@ -45,7 +45,7 @@ describe('the expired Pro purchase route', () => { await waitFor(() => expect(navigation.resetRoot).toHaveBeenCalledWith({ index: 1, - routes: [{ name: 'Main' }, { name: 'ProDetail' }], + routes: [{ name: 'Main', params: { screen: 'HomeTab' } }, { name: 'ProDetail' }], }), ); }); diff --git a/__tests__/unit/screens/DownloadManagerScreen/useDownloadManager.branches.test.ts b/__tests__/unit/screens/DownloadManagerScreen/useDownloadManager.branches.test.ts index 0ccd02f67..c0941437e 100644 --- a/__tests__/unit/screens/DownloadManagerScreen/useDownloadManager.branches.test.ts +++ b/__tests__/unit/screens/DownloadManagerScreen/useDownloadManager.branches.test.ts @@ -93,7 +93,7 @@ const setDownloadedModels = jest.fn(); function configureStores() { appState = { - downloadedModels: [], setDownloadedModels, downloadedImageModels: [], + downloadedModels: [], setDownloadedModels, downloadedImageModels: [], downloadedVideoModels: [], addDownloadedImageModel: jest.fn(), activeImageModelId: null, setActiveImageModelId: jest.fn(), onboardingChecklist: { triedImageGen: false }, }; diff --git a/__tests__/unit/screens/ModelsScreen/imageDownloadActions.test.ts b/__tests__/unit/screens/ModelsScreen/imageDownloadActions.test.ts index 1279671b2..b95007c7b 100644 --- a/__tests__/unit/screens/ModelsScreen/imageDownloadActions.test.ts +++ b/__tests__/unit/screens/ModelsScreen/imageDownloadActions.test.ts @@ -46,7 +46,8 @@ jest.mock('react-native-zip-archive', () => ({ // Integrity is a boundary here (these tests cover download orchestration, not the // completeness rule — that has its own tests). Default it to "complete" so the mocked // RNFS fixtures don't trip the post-unzip gate. -jest.mock('../../../../src/utils/imageModelIntegrity', () => ({ +jest.mock('../../../../src/services/imageModelIntegrity', () => ({ + ...jest.requireActual('../../../../src/services/imageModelIntegrity'), validateImageModelDir: jest.fn(async () => ({ complete: true, missing: [] })), ensureImageExtractionComplete: jest.fn(async () => {}), })); diff --git a/__tests__/unit/screens/ModelsScreen/imageDownloadResume.test.ts b/__tests__/unit/screens/ModelsScreen/imageDownloadResume.test.ts index a8a25eea7..9376370a0 100644 --- a/__tests__/unit/screens/ModelsScreen/imageDownloadResume.test.ts +++ b/__tests__/unit/screens/ModelsScreen/imageDownloadResume.test.ts @@ -20,7 +20,8 @@ jest.mock('react-native-zip-archive', () => ({ // Integrity is a boundary here (resume orchestration is under test, not the completeness // rule). Default to "complete" so mocked RNFS fixtures don't trip the post-unzip gate. -jest.mock('../../../../src/utils/imageModelIntegrity', () => ({ +jest.mock('../../../../src/services/imageModelIntegrity', () => ({ + ...jest.requireActual('../../../../src/services/imageModelIntegrity'), validateImageModelDir: jest.fn(async () => ({ complete: true, missing: [] })), ensureImageExtractionComplete: jest.fn(async () => {}), })); @@ -46,6 +47,9 @@ jest.mock('../../../../src/stores/downloadStore', () => ({ jest.mock('../../../../src/services/imageDownloadActions', () => ({ registerAndNotify: jest.fn(), + // No download is running in this process, so resume owns the model. + isMultifileImageDownloadActive: () => false, + validateMultifileComplete: jest.requireActual('../../../../src/services/imageModelIntegrity').validateMultifileComplete, })); const mockedRNFS = RNFS as jest.Mocked; @@ -238,9 +242,12 @@ describe('resumeImageDownload', () => { imageModelDescription: 'desc', imageModelSize: 1000, imageModelBackend: 'mnn', + imageModelHuggingFaceFiles: [{ path: 'unet.mnn', size: 1000 }], }), }); existingPaths.add(modelDir); + existingPaths.add(`${modelDir}/unet.mnn`); + statSizes[`${modelDir}/unet.mnn`] = 1000; await resumeImageDownload(entry, makeDeps() as any); diff --git a/__tests__/unit/screens/ModelsScreen/trendingSelection.test.ts b/__tests__/unit/screens/ModelsScreen/trendingSelection.test.ts index 985aac60e..1bb94154f 100644 --- a/__tests__/unit/screens/ModelsScreen/trendingSelection.test.ts +++ b/__tests__/unit/screens/ModelsScreen/trendingSelection.test.ts @@ -71,7 +71,7 @@ describe('curated models offered by what loads on the phone', () => { jest.clearAllMocks(); }); - // A 12 GB iPhone reports about 11.2 GB. The shared phone budget loads a download up to half of it. + // A 12 GB iPhone reports about 11.2 GB. The shared iPhone budget loads a download up to 0.70 of it. const TWELVE_GB_IPHONE = 11.2; it('offers a 12 GB iPhone the 9B class it loads, above any parameter cap, and hides what it cannot', () => { @@ -84,8 +84,10 @@ describe('curated models offered by what loads on the phone', () => { expect(offered).toContain('unsloth/Qwen3.5-9B-GGUF'); expect(offered).toContain('unsloth/gemma-4-E4B-it-GGUF'); expect(offered).toContain('LiquidAI/LFM2.5-8B-A1B-GGUF'); - // Gemma 4 12B is 7.3 GB with its projector: past 0.62 of this phone's RAM. - expect(offered).not.toContain('unsloth/gemma-4-12b-it-GGUF'); + // The iPhone loads up to 0.70 of its RAM (ddaa6c1): Gemma 4 12B, 7.3 GB with its projector, + // loads; Gemma 4 26B A4B is past it. + expect(offered).toContain('unsloth/gemma-4-12b-it-GGUF'); + expect(offered).not.toContain('unsloth/gemma-4-26B-A4B-it-GGUF'); }); it('picks the trending model per family from what loads on a 12 GB iPhone', () => { diff --git a/__tests__/unit/screens/ModelsScreen/useImageModels.branches.test.ts b/__tests__/unit/screens/ModelsScreen/useImageModels.branches.test.ts index bedc98948..267f6d342 100644 --- a/__tests__/unit/screens/ModelsScreen/useImageModels.branches.test.ts +++ b/__tests__/unit/screens/ModelsScreen/useImageModels.branches.test.ts @@ -26,6 +26,8 @@ const mockGetImageModelRecommendation = jest.fn(); const mockFetchAvailableModels = jest.fn(); const mockFetchAvailableCoreMLModels = jest.fn(); const mockGuessStyle = jest.fn(); +// The SD catalog is listed ahead of fetched packs; these tests cover the fetched packs. +let mockSDImageModels: any[] = []; const mockMatchesSdVersionFilter = jest.fn(); const mockCancelSynthetic = jest.fn(); const mockStoreRemove = jest.fn(); @@ -47,6 +49,8 @@ jest.mock('../../../../src/services', () => ({ }, hardwareService: { getSoCInfo: (...a: any[]) => mockGetSoCInfo(...a), + // The phone's memory picks which sd.cpp models are recommended (9061db452). + getTotalMemoryGB: () => 8, getImageModelRecommendation: (...a: any[]) => mockGetImageModelRecommendation(...a), }, backgroundDownloadService: { @@ -61,6 +65,10 @@ jest.mock('../../../../src/screens/ModelsScreen/imageDownloadResume', () => ({ jest.mock('../../../../src/services/huggingFaceModelBrowser', () => ({ fetchAvailableModels: (...a: any[]) => mockFetchAvailableModels(...a), guessStyle: (...a: any[]) => mockGuessStyle(...a), + getOfferedSDImageModels: () => mockSDImageModels, + // The sd.cpp models whose memory need fits the phone (9061db452). + getRecommendedSDImageModels: (ramGB: number) => + mockSDImageModels.filter((model: any) => !model.minRamGb || model.minRamGb <= ramGB), })); jest.mock('../../../../src/services/coreMLModelBrowser', () => ({ @@ -156,6 +164,17 @@ describe('loadHFModels', () => { expect(result.current.hfModelsError).toBeNull(); }); + it('lists the SD catalog ahead of the fetched packs', async () => { + mockSDImageModels = [{ id: 'sd-1', name: 'SD', backend: 'sd' }]; + mockGetSoCInfo.mockResolvedValueOnce({ hasNPU: true }); + mockFetchAvailableModels.mockResolvedValueOnce([{ id: 'mnn-1', name: 'MNN', backend: 'mnn' }]); + const { result } = renderHook(() => useImageModels(setAlertState)); + await act(async () => { await result.current.loadHFModels(); }); + mockSDImageModels = []; + + expect(result.current.availableHFModels.map((m: any) => m.id)).toEqual(['sd-1', 'mnn-1']); + }); + it('captures error.message into hfModelsError', async () => { mockGetSoCInfo.mockRejectedValueOnce(new Error('soc failed')); const { result } = renderHook(() => useImageModels(setAlertState)); diff --git a/__tests__/unit/screens/ModelsScreen/useImageModels.processing.test.ts b/__tests__/unit/screens/ModelsScreen/useImageModels.processing.test.ts index a0f27e316..f1fe0f012 100644 --- a/__tests__/unit/screens/ModelsScreen/useImageModels.processing.test.ts +++ b/__tests__/unit/screens/ModelsScreen/useImageModels.processing.test.ts @@ -23,6 +23,8 @@ jest.mock('../../../../src/services', () => ({ }, hardwareService: { getSoCInfo: jest.fn().mockResolvedValue({ hasNPU: true }), + // The phone's memory picks which sd.cpp models are recommended (9061db452). + getTotalMemoryGB: () => 8, getImageModelRecommendation: jest.fn().mockResolvedValue({ recommendedBackend: 'all', bannerText: 'ok', @@ -41,6 +43,8 @@ jest.mock('../../../../src/screens/ModelsScreen/imageDownloadResume', () => ({ jest.mock('../../../../src/services/huggingFaceModelBrowser', () => ({ fetchAvailableModels: jest.fn().mockResolvedValue([]), guessStyle: jest.fn(() => 'all'), + // No sd.cpp catalog models in this suite (9061db452 reads the recommended ones). + getRecommendedSDImageModels: jest.fn(() => []), })); jest.mock('../../../../src/services/coreMLModelBrowser', () => ({ diff --git a/__tests__/unit/screens/ModelsScreen/useModelsScreen.test.ts b/__tests__/unit/screens/ModelsScreen/useModelsScreen.test.ts index 34974ad42..4760c126e 100644 --- a/__tests__/unit/screens/ModelsScreen/useModelsScreen.test.ts +++ b/__tests__/unit/screens/ModelsScreen/useModelsScreen.test.ts @@ -724,6 +724,8 @@ describe('useModelsScreen', () => { useImageModels.mockReturnValue({ downloadedImageModels: [], availableHFModels: [{ id: 'existing-model' }], + // The engine catalog was fetched (848a7f0c0: sd.cpp entries alone no longer count as loaded). + hfCatalogLoaded: true, hfModelsLoading: false, loadHFModels: mockLoadHFModels, setImageFiltersVisible: jest.fn(), diff --git a/__tests__/unit/services/generationService.test.ts b/__tests__/unit/services/generationService.test.ts index 5920509d4..e174dca47 100644 --- a/__tests__/unit/services/generationService.test.ts +++ b/__tests__/unit/services/generationService.test.ts @@ -1097,6 +1097,7 @@ describe('generationService', () => { mockedRunToolLoop.mockImplementation(async ({ onStream, onThinkingDone }: any) => { onThinkingDone?.(); onStream?.({ content: 'result', reasoningContent: '' }); + return { interrupted: false }; }); await generationService.generateWithTools(convId, [ @@ -1115,6 +1116,7 @@ describe('generationService', () => { onStream?.({ content: 'before reset' }); onStreamReset?.(); onStream?.({ content: 'after reset' }); + return { interrupted: false }; }); await generationService.generateWithTools(convId, [ @@ -1130,6 +1132,7 @@ describe('generationService', () => { mockedRunToolLoop.mockImplementation(async ({ onFinalResponse }: any) => { onFinalResponse?.('final answer'); + return { interrupted: false }; }); await generationService.generateWithTools(convId, [ @@ -1317,7 +1320,7 @@ describe('generationService', () => { it('routes generateWithTools to generateRemoteWithTools and calls runToolLoop with forceRemote', async () => { const convId = setupWithConversation(); - mockedRunToolLoop.mockResolvedValue(undefined); + mockedRunToolLoop.mockResolvedValue({ interrupted: false }); await generationService.generateWithTools(convId, [ createMessage({ role: 'user', content: 'use tools' }), @@ -1345,6 +1348,7 @@ describe('generationService', () => { const convId = setupWithConversation(); mockedRunToolLoop.mockImplementation(async ({ onFinalResponse }: any) => { onFinalResponse?.('remote result'); + return { interrupted: false }; }); await generationService.generateWithTools(convId, [ diff --git a/__tests__/unit/services/generationServiceHelpers.branches.test.ts b/__tests__/unit/services/generationServiceHelpers.branches.test.ts index bf9b50f0b..87eba23fa 100644 --- a/__tests__/unit/services/generationServiceHelpers.branches.test.ts +++ b/__tests__/unit/services/generationServiceHelpers.branches.test.ts @@ -74,7 +74,7 @@ jest.mock('../../../src/stores/debugLogsStore', () => ({ })); jest.mock('../../../src/services/generationToolLoop', () => ({ - runToolLoop: jest.fn(() => Promise.resolve()), + runToolLoop: jest.fn(() => Promise.resolve({ interrupted: false })), buildLiteRTHistory: jest.fn(() => []), })); @@ -136,6 +136,7 @@ function makeServiceSvc(overrides: any = {}) { forceFlushTokens: jest.fn(), flushTokenBuffer: jest.fn(), checkSharePrompt: jest.fn(), + drainQueue: jest.fn(), isUsingRemoteProvider: () => false, getCurrentProvider: () => null, ...rest, @@ -451,6 +452,8 @@ describe('generateRemoteWithToolsImpl', () => { expect(store.finalizeStreamingMessage).toHaveBeenCalled(); expect(svc.checkSharePrompt).toHaveBeenCalled(); expect(svc.resetState).toHaveBeenCalled(); + // The next queued send starts only after this turn has finished. + expect(svc.drainQueue).toHaveBeenCalled(); }); it('skips finalize when the generation was aborted during the tool loop', async () => { diff --git a/__tests__/unit/services/imageGenerationHelpers.test.ts b/__tests__/unit/services/imageGenerationHelpers.test.ts index e80b0020d..bdcc96f13 100644 --- a/__tests__/unit/services/imageGenerationHelpers.test.ts +++ b/__tests__/unit/services/imageGenerationHelpers.test.ts @@ -168,7 +168,7 @@ describe('buildImageGenMeta', () => { (Platform as any).OS = 'ios'; const meta = buildImageGenMeta(baseModel, baseOpts); expect(meta.gpu).toBe(true); - expect(meta.gpuBackend).toBe('Core ML (ANE)'); + expect(meta.gpuBackend).toBe('Core ML'); }); it('includes model name, steps, guidanceScale, resolution', () => { diff --git a/__tests__/unit/services/llmSafetyChecks.test.ts b/__tests__/unit/services/llmSafetyChecks.test.ts index b6483c9d0..5ee2b2e2a 100644 --- a/__tests__/unit/services/llmSafetyChecks.test.ts +++ b/__tests__/unit/services/llmSafetyChecks.test.ts @@ -142,10 +142,11 @@ describe('checkMemoryForModel', () => { expect((await checkMemoryForModel({ ...args, projectorFileSize: 0.92e9 })).safe).toBe(true); }); - it('refuses mapped weights past what the fit rule allows on this RAM (Gemma 4 12B on 12 GB)', async () => { + it('refuses mapped weights past what the fit rule allows on this RAM (8.5 GB on 12 GB)', async () => { + // The iPhone's chat ceiling is 70% of 11.2 GB (7.8 GB): Gemma 4 12B (7.3 GB) loads, 8.5 GB does not. mockGetMemory.mockResolvedValue(IPHONE_12GB); const result = await checkMemoryForModel({ - modelFileSize: 7.3e9, contextLength: 4096, getAvailableMemory: mockGetMemory, quantizedCache: true, weightsMapped: true, + modelFileSize: 8.5e9, contextLength: 4096, getAvailableMemory: mockGetMemory, quantizedCache: true, weightsMapped: true, }); expect(result.safe).toBe(false); }); diff --git a/__tests__/unit/services/memoryBudget.test.ts b/__tests__/unit/services/memoryBudget.test.ts index 6f6fa60c1..7656a24f4 100644 --- a/__tests__/unit/services/memoryBudget.test.ts +++ b/__tests__/unit/services/memoryBudget.test.ts @@ -156,11 +156,12 @@ describe('awaitMemoryReclaim — unload does not return until native memory is f }); describe('fit budgets: an iPhone maps GGUF chat weights, nothing else', () => { - // A 12 GB iPhone reports ~11.2 GB. A 9B Q4_K_M with its vision file is 6.6 GB. + // A 12 GB iPhone reports ~11.2 GB. Gemma 4 12B Q4 is 7.3 GB: under the iPhone's 70% chat + // ceiling (7.8 GB), past the 62% everything else keeps (6.9 GB). const RAM = 11.2; - const BYTES = 6.6e9; + const BYTES = 7.3e9; - it('fits a 6.6 GB chat model on a 12 GB iPhone with the text budget', () => { + it('fits a 7.3 GB chat model on a 12 GB iPhone with the text budget', () => { expect(fileExceedsBudget(BYTES, RAM, textModelBudget())).toBe(false); }); diff --git a/__tests__/unit/services/modelManager.test.ts b/__tests__/unit/services/modelManager.test.ts index 61425f79c..937013571 100644 --- a/__tests__/unit/services/modelManager.test.ts +++ b/__tests__/unit/services/modelManager.test.ts @@ -70,6 +70,12 @@ function setupLiteRTImportMocks() { } describe('ModelManager', () => { + // Some groups pin Platform with a getter spy; resetAllMocks would leave that getter answering + // undefined for every later test, so each test starts from the real Platform. + afterEach(() => { + jest.restoreAllMocks(); + }); + beforeEach(() => { jest.resetAllMocks(); @@ -140,7 +146,7 @@ describe('ModelManager', () => { await modelManager.initialize(); - expect(mockedBackgroundDownloadService.excludeFromBackup).toHaveBeenCalledTimes(3); + expect(mockedBackgroundDownloadService.excludeFromBackup).toHaveBeenCalledTimes(4); expect(mockedBackgroundDownloadService.excludeFromBackup).toHaveBeenCalledWith( expect.stringContaining('/models'), ); @@ -150,6 +156,9 @@ describe('ModelManager', () => { expect(mockedBackgroundDownloadService.excludeFromBackup).toHaveBeenCalledWith( expect.stringContaining('/whisper-models'), ); + expect(mockedBackgroundDownloadService.excludeFromBackup).toHaveBeenCalledWith( + expect.stringContaining('/video-models'), + ); }); }); diff --git a/__tests__/unit/services/offGridDesktopRoom.test.ts b/__tests__/unit/services/offGridDesktopRoom.test.ts new file mode 100644 index 000000000..62d8e99b7 --- /dev/null +++ b/__tests__/unit/services/offGridDesktopRoom.test.ts @@ -0,0 +1,41 @@ +/** + * Reading what a paired Off Grid AI Desktop can take, over its gateway. Boundary faked: fetch only. + */ +import { readOffGridDesktopRoom } from '../../../src/services/offGridDesktopModels'; +import type { RemoteServer } from '../../../src/types'; + +const server = { id: 's', name: 'Mac', endpoint: 'http://192.168.1.34:7878/', apiKey: 'key' } as RemoteServer; +const reply = (status: number, body: unknown) => ({ ok: status < 400, status, json: async () => body }); + +function gateway(catalog: [number, unknown], installed: [number, unknown]) { + global.fetch = jest.fn(async (url: string) => + url.endsWith('/v1/models/catalog') ? reply(...catalog) : reply(...installed), + ) as unknown as typeof fetch; +} + +describe('readOffGridDesktopRoom', () => { + it('reads the computer memory and its installed models', async () => { + gateway([200, { memoryGb: 34, kinds: [], models: [] }], [200, { installed: ['a', ' ', 'b'] }]); + await expect(readOffGridDesktopRoom(server)).resolves.toEqual({ memoryGb: 34, installed: ['a', 'b'] }); + }); + + it('leaves the memory unknown for an older Desktop', async () => { + gateway([200, { kinds: [], models: [] }], [200, { installed: [] }]); + await expect(readOffGridDesktopRoom(server)).resolves.toEqual({ installed: [] }); + }); + + it('says why when the Desktop refuses the key', async () => { + gateway([401, { error: 'nope' }], [401, null]); + await expect(readOffGridDesktopRoom(server)).rejects.toThrow('Desktop did not accept this API key'); + }); + + it('passes on the Desktop error when the installed list fails', async () => { + gateway([200, { memoryGb: 8 }], [500, { error: 'Models are loading.' }]); + await expect(readOffGridDesktopRoom(server)).rejects.toThrow('Models are loading.'); + }); + + it('falls back to a plain reason when the Desktop gives none', async () => { + gateway([500, null], [200, { installed: [] }]); + await expect(readOffGridDesktopRoom(server)).rejects.toThrow('Desktop did not say which models it has.'); + }); +}); diff --git a/__tests__/unit/services/remoteMediaRuntime.test.ts b/__tests__/unit/services/remoteMediaRuntime.test.ts index d6c79a0e3..9161f0bcb 100644 --- a/__tests__/unit/services/remoteMediaRuntime.test.ts +++ b/__tests__/unit/services/remoteMediaRuntime.test.ts @@ -58,12 +58,17 @@ describe('remoteMediaRuntime', () => { expect(remoteServerCapabilities(server)).toEqual({ imageGeneration: true, transcription: true, + videoGeneration: false, voice: true, }); + expect( + remoteServerCapabilities({ mediaModels: { video: 'wan2.1' } }).videoGeneration, + ).toBe(true); expect(remoteServerCapabilities({ mediaModels: { image: ' ' } })).toEqual( { imageGeneration: false, transcription: false, + videoGeneration: false, voice: false, }, ); diff --git a/__tests__/unit/services/remoteServerManager.test.ts b/__tests__/unit/services/remoteServerManager.test.ts index 9ca5f47db..ab35e78b5 100644 --- a/__tests__/unit/services/remoteServerManager.test.ts +++ b/__tests__/unit/services/remoteServerManager.test.ts @@ -417,9 +417,13 @@ describe('remoteServerManager', () => { expect( useRemoteServerStore.getState().setActiveRemoteImageModelId, ).toHaveBeenCalledWith(null); - expect( - useRemoteServerStore.getState().setActiveRemoteMediaServerId, - ).toHaveBeenCalledTimes(3); + const clearedMedia = ( + useRemoteServerStore.getState().setActiveRemoteMediaServerId as jest.Mock + ).mock.calls; + expect(clearedMedia.map(([category]) => category).sort()).toEqual( + ['image', 'transcription', 'video', 'voice'], + ); + expect(clearedMedia.every(([, id]) => id === null)).toBe(true); expect(providerRegistry.setActiveProvider).toHaveBeenCalledWith('local'); }); }); diff --git a/__tests__/unit/services/toolEmbeddingRouter.test.ts b/__tests__/unit/services/toolEmbeddingRouter.test.ts index 0145248e6..3b2716a03 100644 --- a/__tests__/unit/services/toolEmbeddingRouter.test.ts +++ b/__tests__/unit/services/toolEmbeddingRouter.test.ts @@ -9,6 +9,8 @@ const mockEmbed = jest.fn(); const mockLoad = jest.fn(); jest.mock('../../../src/services/rag/embedding', () => ({ embeddingService: { + runExclusive: (operation: () => Promise) => operation(), + getModelId: () => 'bundled:all-MiniLM-L6-v2-Q8_0', load: (...a: unknown[]) => mockLoad(...a), embed: (...a: unknown[]) => mockEmbed(...a), }, diff --git a/__tests__/unit/services/tools/registry.test.ts b/__tests__/unit/services/tools/registry.test.ts index 767cc679e..776dd10b7 100644 --- a/__tests__/unit/services/tools/registry.test.ts +++ b/__tests__/unit/services/tools/registry.test.ts @@ -16,13 +16,15 @@ describe('Tool Registry', () => { // AVAILABLE_TOOLS // ======================================================================== describe('AVAILABLE_TOOLS', () => { - it('has exactly 6 tools with correct IDs', () => { + it('has exactly 8 tools with correct IDs', () => { // Email + calendar tools are pro-gated and live in the pro package // (EmailCalendarExtension), so they are not part of the core registry. - expect(AVAILABLE_TOOLS).toHaveLength(6); + expect(AVAILABLE_TOOLS).toHaveLength(8); const ids = AVAILABLE_TOOLS.map(t => t.id); expect(ids).toEqual([ + 'generate_video', + 'make_film', 'web_search', 'calculator', 'get_current_datetime', diff --git a/__tests__/unit/sync/sharedFileMaterializer.test.ts b/__tests__/unit/sync/sharedFileMaterializer.test.ts index f98508349..d9192f26c 100644 --- a/__tests__/unit/sync/sharedFileMaterializer.test.ts +++ b/__tests__/unit/sync/sharedFileMaterializer.test.ts @@ -65,10 +65,36 @@ describe('making a transferred file appear in the app', () => { ?.messages.find(({ uuid }) => uuid === messageUuid)?.attachments ?? []; beforeEach(() => { - useAppStore.setState({ generatedImages: [] }); + useAppStore.setState({ generatedImages: [], generatedVideos: [] }); useChatStore.setState({ conversations: [] }); }); + describe('a video generated on another device', () => { + const video = () => record({ + mimeType: 'video/mp4', name: 'lake.mp4', localPath: '/docs/shared_files/lake.mp4', + conversationId: 'chat-7', messageId: 'message-1', width: 320, height: 192, + durationSeconds: 2.125, + metadataJson: JSON.stringify({ prompt: 'A moving lake', steps: 20, guidance: 6, seed: 42, modelId: 'wan', fps: 8, frames: 17 }), + }); + it('appears once in the video gallery and on its message after repeated delivery', () => { + useChatStore.setState({ conversations: [conversation()] }); + materializeSharedFile(video()); + materializeSharedFile(video()); + expect(gallery()).toEqual([]); + expect(useAppStore.getState().generatedVideos).toHaveLength(1); + expect(useAppStore.getState().generatedVideos[0]).toMatchObject({ prompt: 'A moving lake', durationSeconds: 2.125, fps: 8, seed: 42 }); + expect(attachmentsOn('chat-7', 'message-1')).toHaveLength(1); + expect(attachmentsOn('chat-7', 'message-1')[0]).toMatchObject({ type: 'video', mimeType: 'video/mp4' }); + }); + it('removes the gallery entry and chat attachment when the shared video is deleted', () => { + useChatStore.setState({ conversations: [conversation()] }); + materializeSharedFile(video()); + removeMaterializedSharedFile(video()); + expect(useAppStore.getState().generatedVideos).toEqual([]); + expect(attachmentsOn('chat-7', 'message-1')).toEqual([]); + }); + }); + describe('a picture generated on another device', () => { it('appears in the gallery, with the prompt that made it', () => { materializeSharedFile( diff --git a/__tests__/unit/utils/imageModelIntegrity.io.test.ts b/__tests__/unit/utils/imageModelIntegrity.io.test.ts index 3a6545435..a58adb63c 100644 --- a/__tests__/unit/utils/imageModelIntegrity.io.test.ts +++ b/__tests__/unit/utils/imageModelIntegrity.io.test.ts @@ -27,7 +27,7 @@ jest.mock('../../../src/utils/logger', () => ({ __esModule: true, default: { log import { resolveImageModelDir, validateImageModelDir, ensureImageExtractionComplete, -} from '../../../src/utils/imageModelIntegrity'; +} from '../../../src/services/imageModelIntegrity'; import { ImageModelIncompleteError } from '../../../src/services/modelLoadErrors'; const COMPLETE_MNN = [ diff --git a/__tests__/unit/utils/imageModelIntegrity.sd.test.ts b/__tests__/unit/utils/imageModelIntegrity.sd.test.ts new file mode 100644 index 000000000..37e40ab0f --- /dev/null +++ b/__tests__/unit/utils/imageModelIntegrity.sd.test.ts @@ -0,0 +1,143 @@ +/** + * imageModelIntegrity for stable-diffusion.cpp packs and multi-file downloads. Files live in an + * in-memory disk behind the react-native-fs boundary; the SD pack rules, checkpoint header check + * and completeness rules run for real. + */ +import { createHash } from 'crypto'; + +const mockFiles = new Map(); + +function mockList(dir: string) { + const children = new Map(); + for (const [path, bytes] of mockFiles) { + if (!path.startsWith(`${dir}/`)) continue; + const [name, ...rest] = path.slice(dir.length + 1).split('/'); + children.set(name, rest.length ? { size: 0, isFile: false } : { size: bytes.length, isFile: true }); + } + if (children.size === 0) throw new Error(`ENOENT ${dir}`); + return [...children].map(([name, entry]) => ({ + name, + size: entry.size, + path: `${dir}/${name}`, + isFile: () => entry.isFile, + isDirectory: () => !entry.isFile, + })); +} + +jest.mock('react-native-fs', () => ({ + DocumentDirectoryPath: '/docs', + exists: async (path: string) => mockFiles.has(path), + readDir: async (dir: string) => mockList(dir), + stat: async (path: string) => ({ size: mockFiles.get(path)?.length ?? 0 }), + read: async (path: string, length: number, position: number) => + mockFiles.get(path)!.subarray(position, position + length).toString('base64'), + hash: async (path: string) => + require('crypto').createHash('sha256').update(mockFiles.get(path)!).digest('hex'), +})); + +jest.mock('react-native-zip-archive', () => ({ unzip: jest.fn() })); + +import { + checkImageModelFiles, + resolveImageModelDir, + validateImageModelDir, + validateMultifileComplete, +} from '../../../src/services/imageModelIntegrity'; + +/** A safetensors file whose header names the given tensors. */ +function safetensors(tensors: string[]): Buffer { + const header = Buffer.from(JSON.stringify(Object.fromEntries(tensors.map(name => [name, {}])))); + const length = Buffer.alloc(8); + length.writeUInt32LE(header.length, 0); + return Buffer.concat([length, header, Buffer.alloc(16)]); +} + +const FULL_CHECKPOINT = safetensors([ + 'model.diffusion_model.input_blocks.0.0.weight', + 'first_stage_model.decoder.conv_in.weight', + 'cond_stage_model.transformer.text_model.embeddings.token_embedding.weight', +]); +const UNET_ONLY = safetensors(['model.diffusion_model.input_blocks.0.0.weight']); + +beforeEach(() => mockFiles.clear()); + +describe('checkImageModelFiles for SD packs', () => { + it('needs exactly one diffusion weight to recognise a pack', () => { + expect(checkImageModelFiles([{ name: 'notes.txt', size: 5, isFile: true }], 'sd')).toEqual({ + complete: false, + missing: ['image model pack'], + }); + }); + + it('accepts a non-empty single checkpoint and names an empty one', () => { + expect(checkImageModelFiles([{ name: 'dreamshaper_8.safetensors', size: 10, isFile: true }], 'sd')).toEqual({ + complete: true, + missing: [], + }); + expect(checkImageModelFiles([{ name: 'dreamshaper_8.safetensors', size: 0, isFile: true }], 'sd')).toEqual({ + complete: false, + missing: ['dreamshaper_8.safetensors'], + }); + }); + + it('lists the Qwen-Image companions that are not on disk', () => { + const result = checkImageModelFiles([{ name: 'qwen_image_2.1-Q4_K_M.gguf', size: 10, isFile: true }], 'sd'); + expect(result.complete).toBe(false); + expect(result.missing.length).toBeGreaterThan(0); + expect(result.missing).not.toContain('qwen_image_2.1-Q4_K_M.gguf'); + }); +}); + +describe('SD pack directories on disk', () => { + it('finds the pack directly or one folder down', async () => { + mockFiles.set('/m/dreamshaper_8.safetensors', FULL_CHECKPOINT); + expect(await resolveImageModelDir('/m', 'sd')).toBe('/m'); + mockFiles.clear(); + mockFiles.set('/m/pack/dreamshaper_8.safetensors', FULL_CHECKPOINT); + expect(await resolveImageModelDir('/m', 'sd')).toBe('/m/pack'); + }); + + it('reports a missing pack when no weight is present', async () => { + mockFiles.set('/m/readme.txt', Buffer.from('x')); + expect(await validateImageModelDir('/m', 'sd')).toEqual({ complete: false, missing: ['image model pack'] }); + }); + + it('accepts a complete checkpoint and rejects a UNet-only export', async () => { + mockFiles.set('/m/dreamshaper_8.safetensors', FULL_CHECKPOINT); + expect(await validateImageModelDir('/m', 'sd')).toEqual({ complete: true, missing: [] }); + mockFiles.set('/m/dreamshaper_8.safetensors', UNET_ONLY); + expect(await validateImageModelDir('/m', 'sd')).toEqual({ + complete: false, + missing: ['complete SD checkpoint with image model, text encoder, and VAE'], + }); + }); +}); + +describe('validateMultifileComplete', () => { + const part = Buffer.from('model bytes'); + const sha256 = createHash('sha256').update(part).digest('hex'); + + it('refuses a download with no file list', async () => { + await expect(validateMultifileComplete('/m', [])).rejects.toThrow('Download file list missing'); + }); + + it('accepts parts that are present, the expected size, and hash-matched in any case', async () => { + mockFiles.set('/m/unet.mnn', part); + await expect( + validateMultifileComplete('/m', [{ relativePath: 'unet.mnn', expectedSize: part.length, sha256: sha256.toUpperCase() }]), + ).resolves.toBeUndefined(); + }); + + it('names a missing, short, or damaged part', async () => { + await expect(validateMultifileComplete('/m', [{ relativePath: 'unet.mnn' }])).rejects.toThrow( + 'Downloaded file missing or empty: unet.mnn', + ); + mockFiles.set('/m/unet.mnn', part); + await expect( + validateMultifileComplete('/m', [{ relativePath: 'unet.mnn', expectedSize: part.length + 1 }]), + ).rejects.toThrow('Downloaded file is incomplete: unet.mnn'); + await expect( + validateMultifileComplete('/m', [{ relativePath: 'unet.mnn', sha256: '0'.repeat(64) }]), + ).rejects.toThrow('Downloaded file is damaged: unet.mnn'); + }); +}); diff --git a/__tests__/unit/utils/imageModelIntegrity.test.ts b/__tests__/unit/utils/imageModelIntegrity.test.ts index bc30fba7f..1fec6b49c 100644 --- a/__tests__/unit/utils/imageModelIntegrity.test.ts +++ b/__tests__/unit/utils/imageModelIntegrity.test.ts @@ -6,7 +6,7 @@ * Reproduces the EXACT on-device defect: the mnn model extracted missing `pos_emb.bin` * AND `clip_v2.mnn.weight` while every other file was present, yet nothing flagged it. */ -import { checkImageModelFiles, type ImageDirEntry } from '../../../src/utils/imageModelIntegrity'; +import { checkImageModelFiles, type ImageDirEntry } from '../../../src/services/imageModelIntegrity'; // The complete, correct mnn model file set (from the verified xororz/sd-mnn zip). const COMPLETE_MNN: ImageDirEntry[] = [ diff --git a/__tests__/utils/testHelpers.ts b/__tests__/utils/testHelpers.ts index 30c42eb91..c2928f888 100644 --- a/__tests__/utils/testHelpers.ts +++ b/__tests__/utils/testHelpers.ts @@ -83,6 +83,14 @@ export const resetStores = (): void => { liteRTTemperature: 0.7, liteRTTopP: 0.9, liteRTMaxTokens: 4096, + analyticsUsage: false, + analyticsCrashes: false, + analyticsAsked: false, + analyticsLastPromptAt: null, + videoParams: {}, + videoSeed: -1, + videoNegative: '', + enhanceVideoPrompts: false, }, downloadedImageModels: [], activeImageModelId: null, diff --git a/altstore-source.json b/altstore-source.json index 0308e5e59..ac501cac8 100644 --- a/altstore-source.json +++ b/altstore-source.json @@ -13,7 +13,7 @@ "bundleIdentifier": "ai.offgridmobile", "developerName": "Off Grid", "subtitle": "On-device AI", - "localizedDescription": "Run large language models, generate images with Stable Diffusion, and transcribe speech with Whisper — all on-device, fully offline, no cloud required.", + "localizedDescription": "Run large language models, generate images with Stable Diffusion, and transcribe speech with Whisper — all on-device, fully offline, no cloud required. Your chats and files never leave your device; anonymous usage data and crash reports only if you turn them on.", "iconURL": "https://raw.githubusercontent.com/alichherawalla/offline-mobile-llm-manager/main/ios/OffgridMobile/Images.xcassets/AppIcon.appiconset/icon.png", "tintColor": "000000", "versions": [ diff --git a/android/app/build.gradle b/android/app/build.gradle index 3bbbaff34..d7472143d 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -89,6 +89,8 @@ android { coreLibraryDesugaringEnabled true } + externalNativeBuild { cmake { path "src/main/cpp/CMakeLists.txt"; version "3.22.1" } } + sourceSets.main.assets.srcDir("../../.video-build/hexagon-assets") defaultConfig { applicationId "ai.offgridmobile" minSdkVersion rootProject.ext.minSdkVersion @@ -195,3 +197,25 @@ dependencies { testImplementation("org.mockito.kotlin:mockito-kotlin:5.4.0") testImplementation("androidx.test:core:1.6.1") } + +// Native video setup is shared by command-line, IDE, and CI builds. +// Configure tasks need the source tree before CMake evaluates it. +def videoRoot = rootProject.projectDir.parentFile +def prepareVideoRuntime = tasks.register("prepareVideoRuntime", Exec) { + workingDir videoRoot + commandLine "bash", new File(videoRoot, "scripts/prepare-video-runtime.sh").absolutePath, "--android" + inputs.files(fileTree(new File(videoRoot, "scripts/video")), new File(videoRoot, "scripts/prepare-video-runtime.sh")) + outputs.files(new File(videoRoot, ".video-build/runtime-ready"), new File(videoRoot, ".video-build/shaderc/bin/glslc"), + new File(videoRoot, ".video-build/hexagon-runtime/htp_iface_stub.c"), new File(videoRoot, ".video-build/hexagon-runtime/htp_iface.h")) + outputs.dir(new File(videoRoot, ".video-build/hexagon-assets/video-hexagon")) +} +// JVM unit tests never load the native video engine. Building it needs the shader compiler and +// the Hexagon SDK, so a unit-test-only invocation (CI's Android test step) skips it. +def unitTestsOnly = !gradle.startParameter.taskNames.isEmpty() && + gradle.startParameter.taskNames.every { it.toLowerCase().contains("unittest") } +tasks.configureEach { task -> + if (unitTestsOnly) return + if (task.name.startsWith("configureCMake") || (task.name.startsWith("merge") && task.name.endsWith("Assets"))) task.dependsOn(prepareVideoRuntime) + // The release JS bundle reads the project root, where the video runtime writes .video-build. + if (task.name.startsWith("createBundle") && task.name.endsWith("JsAndAssets")) task.mustRunAfter(prepareVideoRuntime) +} diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 04fef1ba4..41b474c28 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -47,6 +47,8 @@ + + @@ -78,6 +80,7 @@ + + + diff --git a/android/app/src/main/cpp/CMakeLists.txt b/android/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 000000000..848be61ae --- /dev/null +++ b/android/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,54 @@ +cmake_minimum_required(VERSION 3.22) +project(appmodules LANGUAGES C CXX) +include(${REACT_ANDROID_DIR}/cmake-utils/ReactNative-application.cmake) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_C_VISIBILITY_PRESET hidden) +set(CMAKE_CXX_VISIBILITY_PRESET hidden) +set(SD_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(SD_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) +set(SD_BUILD_SHARED_GGML_LIB OFF CACHE BOOL "" FORCE) +set(GGML_NATIVE OFF CACHE BOOL "" FORCE) +set(SD_WEBP OFF CACHE BOOL "" FORCE) +set(SD_WEBM OFF CACHE BOOL "" FORCE) +set(SD_METAL OFF CACHE BOOL "" FORCE) +set(SD_VULKAN ON CACHE BOOL "" FORCE) +set(SD_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../.video-build/source") +set(GGML_HEXAGON ON CACHE BOOL "" FORCE) +set(OFFGRID_VIDEO_HEXAGON_CMAKE "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../scripts/video/hexagon-prebuilt.cmake") +if(NOT EXISTS "${SD_SOURCE}/CMakeLists.txt") + message(FATAL_ERROR "Run scripts/prepare-video-runtime.sh before building Android.") +endif() +set(SPIRV-Headers_DIR "${SD_SOURCE}/../vulkan-header-prefix/share/cmake/SPIRV-Headers") +set(Vulkan_INCLUDE_DIR "${SD_SOURCE}/../vulkan-header-prefix/include" CACHE PATH "Pinned Vulkan headers" FORCE) +string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" VIDEO_HOST_OS) +# The NDK shader compiler can be too old for the pinned runtime. Use the host +# shaderc installation, or an explicit -DVulkan_GLSLC_EXECUTABLE path. +find_program(Vulkan_GLSLC_EXECUTABLE NAMES glslc glslc.exe + HINTS "${SD_SOURCE}/../shaderc/bin" NO_CMAKE_FIND_ROOT_PATH REQUIRED) +if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") + # The NDK prepends its compiler directory even for host-program discovery. + # Its clang cannot compile against the current macOS SDK. Use Xcode's host + # toolchain for the shader generator; model libraries still use the NDK. + execute_process(COMMAND xcrun --sdk macosx --find clang + OUTPUT_VARIABLE VIDEO_HOST_CC OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY) + execute_process(COMMAND xcrun --sdk macosx --find clang++ + OUTPUT_VARIABLE VIDEO_HOST_CXX OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY) +else() + find_program(VIDEO_HOST_CC NAMES clang gcc cl NO_CMAKE_FIND_ROOT_PATH REQUIRED) + find_program(VIDEO_HOST_CXX NAMES clang++ g++ cl NO_CMAKE_FIND_ROOT_PATH REQUIRED) +endif() +set(GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "${CMAKE_BINARY_DIR}/video-host-toolchain.cmake") +file(WRITE "${GGML_VULKAN_SHADERS_GEN_TOOLCHAIN}" + "set(CMAKE_C_COMPILER \"${VIDEO_HOST_CC}\")\nset(CMAKE_CXX_COMPILER \"${VIDEO_HOST_CXX}\")\nset(CMAKE_MAKE_PROGRAM \"${CMAKE_MAKE_PROGRAM}\" CACHE FILEPATH \"Host build tool\" FORCE)\n") +add_subdirectory("${SD_SOURCE}" runtime) +# Keep model execution representative in debug apps. Unoptimized tensor kernels +# make CPU conditioning take minutes even though the app remains responsive. +foreach(VIDEO_RUNTIME_TARGET stable-diffusion ggml-base ggml-cpu ggml-vulkan ggml-hexagon) + if(TARGET ${VIDEO_RUNTIME_TARGET}) + target_compile_options(${VIDEO_RUNTIME_TARGET} PRIVATE $<$:-O3>) + endif() +endforeach() +add_library(offgrid_video SHARED video/video-jni.cpp) +target_include_directories(offgrid_video PRIVATE "${SD_SOURCE}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../native/video/common") +target_link_libraries(offgrid_video PRIVATE stable-diffusion android log) +target_link_options(offgrid_video PRIVATE "-Wl,--exclude-libs,ALL" "-Wl,-z,max-page-size=16384") diff --git a/android/app/src/main/cpp/video/video-jni.cpp b/android/app/src/main/cpp/video/video-jni.cpp new file mode 100644 index 000000000..c5a4c7b33 --- /dev/null +++ b/android/app/src/main/cpp/video/video-jni.cpp @@ -0,0 +1,246 @@ +#include +#include +#include +#include "VideoRuntime.hpp" + +static offgrid::VideoRuntime runtime; + +class AttachedEnv { + JavaVM *vm; + bool attached = false; +public: + JNIEnv *env = nullptr; + explicit AttachedEnv(JavaVM *value) : vm(value) { + if (vm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6) == JNI_EDETACHED) { + attached = vm->AttachCurrentThread(&env, nullptr) == JNI_OK; + if (!attached) env = nullptr; + } + } + ~AttachedEnv() { if (attached) vm->DetachCurrentThread(); } +}; + +static std::string string(JNIEnv *env, jstring value) { + if (!value) throw std::runtime_error("Missing video argument."); + const char *bytes = env->GetStringUTFChars(value, nullptr); + if (!bytes) throw std::runtime_error("Missing video argument."); + std::string result(bytes); + env->ReleaseStringUTFChars(value, bytes); + return result; +} + +extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativePrepare(JNIEnv *, jobject) { +#ifndef NDEBUG + sd_set_log_callback([](sd_log_level_t level, const char *text, void *) { + const int priority = level == SD_LOG_ERROR ? ANDROID_LOG_ERROR : + level == SD_LOG_WARN ? ANDROID_LOG_WARN : ANDROID_LOG_DEBUG; + __android_log_write(priority, "OffgridVideo", text); + }, nullptr); +#endif + runtime.cancelled.store(false); +} +extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeCancel(JNIEnv *, jobject) { runtime.cancel(); } +extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeSetRuntimeDirectory(JNIEnv *env, jobject, jstring path) { + try { + auto directory = string(env, path); + const char *existing = getenv("ADSP_LIBRARY_PATH"); + std::string search = directory + ";" + (existing ? existing : "/vendor/lib/rfsa/adsp;/vendor/dsp;/system/lib/rfsa/adsp;/dsp"); + if (setenv("ADSP_LIBRARY_PATH", search.c_str(), 1) != 0 || + setenv("OFFGRID_VIDEO_HTP_DIR", directory.c_str(), 1) != 0) + throw std::runtime_error("Could not configure the NPU runtime."); + } catch (const std::exception &error) { + if (!env->ExceptionCheck()) { + auto klass = env->FindClass("java/lang/IllegalStateException"); + if (klass) { env->ThrowNew(klass, error.what()); env->DeleteLocalRef(klass); } + } + } +} +extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeGenerate( + JNIEnv *env, jobject self, jstring weight, jstring vae, jstring encoder, jstring prompt, jstring negative, + jint width, jint height, jint frames, jint fps, jint steps, jdouble guidance, jdouble seed, jstring llm, jstring embeddings, jstring audioVae, jdouble flowShift, jstring diagnosticBackend, jbyteArray initImage) { + jobject owner = nullptr; + try { + offgrid::VideoRequest request{string(env, weight), string(env, vae), string(env, encoder), string(env, prompt), string(env, negative), width, height, frames, fps, steps, (float)guidance, (int64_t)seed}; + // Optional start image: RGB at the clip size, prepared by the host (Animate this photo, Continue). + if (initImage && env->GetArrayLength(initImage) == width * height * 3) { + request.initImage.resize(static_cast(width) * height * 3); + env->GetByteArrayRegion(initImage, 0, width * height * 3, reinterpret_cast(request.initImage.data())); + } + request.llm = string(env, llm); + request.embeddings = string(env, embeddings); + request.audioVae = string(env, audioVae); + request.flowShift = (float)flowShift; + const auto backend = string(env, diagnosticBackend); + request.cpuOnly = backend == "cpu"; + request.skipNpu = backend == "gpu"; + JavaVM *vm = nullptr; + if (env->GetJavaVM(&vm) != JNI_OK) throw std::runtime_error("Could not access the video host."); + owner = env->NewGlobalRef(self); + if (!owner) throw std::runtime_error("Not enough memory to start video generation."); + auto klass = env->GetObjectClass(self); + auto progress = env->GetMethodID(klass, "progress", "(II)V"); + auto conditioning = env->GetMethodID(klass, "conditioning", "(Ljava/lang/String;)V"); + auto frame = env->GetMethodID(klass, "frame", "([BIII)V"); + auto decodeProgress = env->GetMethodID(klass, "decoding", "(II)V"); + auto preview = env->GetMethodID(klass, "preview", "([BII)V"); + env->DeleteLocalRef(klass); + if (!progress || !frame || !conditioning || !decodeProgress || !preview) throw std::runtime_error("Missing video host callbacks."); + std::atomic_bool callbackFailed{false}; + runtime.run(request, [&](int step, int total) { + // The engine may report progress from a worker thread. JNI environments + // are thread-local; never retain the caller's environment in this callback. + AttachedEnv thread(vm); + if (!thread.env) { callbackFailed.store(true); runtime.cancel(); return; } + thread.env->CallVoidMethod(owner, progress, step, total); + if (thread.env->ExceptionCheck()) { + thread.env->ExceptionClear(); + callbackFailed.store(true); + runtime.cancel(); + } + }, [&](sd_image_t *images, int count, int) { + if (callbackFailed.load()) throw std::runtime_error("Could not report video progress."); + for (int i = 0; i < count; ++i) { + if (runtime.cancelled.load()) throw std::runtime_error("Video generation stopped."); + auto &image = images[i]; + const uint64_t size = uint64_t(image.width) * image.height * image.channel; + if (!image.data || image.width != uint32_t(width) || image.height != uint32_t(height) || + image.channel != 3 || size > uint64_t(std::numeric_limits::max())) + throw std::runtime_error("The video engine returned an invalid frame."); + auto bytes = env->NewByteArray(static_cast(size)); + if (!bytes) throw std::runtime_error("Not enough memory to encode video."); + env->SetByteArrayRegion(bytes, 0, static_cast(size), reinterpret_cast(image.data)); + if (!env->ExceptionCheck()) env->CallVoidMethod(owner, frame, bytes, image.width, image.height, image.channel); + env->DeleteLocalRef(bytes); + if (env->ExceptionCheck()) throw std::runtime_error("Video encoder failed."); + } + }, [&](const char *backend) { + auto hardware = env->NewStringUTF(backend); + env->CallVoidMethod(owner, conditioning, hardware); + env->DeleteLocalRef(hardware); + if (env->ExceptionCheck()) throw std::runtime_error("Could not report prompt processing."); + }, [&](int completed, int total) { + AttachedEnv thread(vm); + if (!thread.env) return; + thread.env->CallVoidMethod(owner, decodeProgress, completed, total); + if (thread.env->ExceptionCheck()) thread.env->ExceptionClear(); + }, [&](const sd_image_t &image) { + // A preview is optional. Its failure must not discard a completed decode. + const uint64_t size = uint64_t(image.width) * image.height * image.channel; + if (!image.data || image.channel != 3 || size > uint64_t(std::numeric_limits::max())) return; + AttachedEnv thread(vm); + if (!thread.env || runtime.cancelled.load()) return; + auto bytes = thread.env->NewByteArray(static_cast(size)); + if (bytes) { + thread.env->SetByteArrayRegion(bytes, 0, static_cast(size), reinterpret_cast(image.data)); + if (!thread.env->ExceptionCheck()) thread.env->CallVoidMethod(owner, preview, bytes, image.width, image.height); + thread.env->DeleteLocalRef(bytes); + } + if (thread.env->ExceptionCheck()) thread.env->ExceptionClear(); + }); + } catch (const std::exception &error) { + if (!env->ExceptionCheck()) { + auto klass = env->FindClass("java/lang/IllegalStateException"); + if (klass) { env->ThrowNew(klass, error.what()); env->DeleteLocalRef(klass); } + } + } + if (owner) env->DeleteGlobalRef(owner); +} + +extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeLoadImage( + JNIEnv *env, jobject, jstring path, jstring weight, jstring vae, jstring llm, jint threads, jboolean cpuOnly, jstring family, jstring sampler, jstring scheduler) { + try { + offgrid::VideoRequest request{}; + request.weight = string(env, weight); request.vae = string(env, vae); request.llm = string(env, llm); + request.threads = threads; request.cpuOnly = cpuOnly; + request.imageFamily = string(env, family); request.imageSampler = string(env, sampler); request.imageScheduler = string(env, scheduler); + runtime.loadImage(request, string(env, path)); + } catch (const std::exception &error) { + auto klass = env->FindClass("java/lang/IllegalStateException"); + if (klass) { env->ThrowNew(klass, error.what()); env->DeleteLocalRef(klass); } + } +} +extern "C" JNIEXPORT jstring JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeImagePath(JNIEnv *env, jobject) { + return env->NewStringUTF(runtime.loadedImagePath().c_str()); +} +extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeUnloadImage(JNIEnv *env, jobject) { + try { runtime.unloadImage(); } + catch (const std::exception &error) { + auto klass = env->FindClass("java/lang/IllegalStateException"); + if (klass) { env->ThrowNew(klass, error.what()); env->DeleteLocalRef(klass); } + } +} +// sd.cpp's in-process ESRGAN upscaler. One context per clip; frames go through it one by one. +extern "C" JNIEXPORT jlong JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeUpscalerOpen( + JNIEnv *env, jobject, jstring modelPath, jint threads) { + try { + return reinterpret_cast(new_upscaler_ctx(string(env, modelPath).c_str(), false, threads, 128, "cpu", nullptr)); + } catch (...) { return 0; } +} +extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeUpscalerClose(JNIEnv *, jobject, jlong handle) { + if (handle) free_upscaler_ctx(reinterpret_cast(handle)); +} +extern "C" JNIEXPORT jbyteArray JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeUpscale( + JNIEnv *env, jobject, jlong handle, jbyteArray rgb, jint width, jint height, jint factor) { + if (!handle || !rgb || env->GetArrayLength(rgb) != width * height * 3) return nullptr; + std::vector pixels(static_cast(width) * height * 3); + env->GetByteArrayRegion(rgb, 0, width * height * 3, reinterpret_cast(pixels.data())); + sd_image_t input{static_cast(width), static_cast(height), 3, pixels.data()}; + sd_image_t *out = nullptr; int count = 0; + if (!upscale(reinterpret_cast(handle), input, static_cast(factor), &out, &count) || !out || count < 1) return nullptr; + const uint64_t size = uint64_t(out[0].width) * out[0].height * out[0].channel; + jbyteArray result = out[0].channel == 3 && size <= INT32_MAX ? env->NewByteArray(static_cast(size)) : nullptr; + if (result) env->SetByteArrayRegion(result, 0, static_cast(size), reinterpret_cast(out[0].data)); + free_sd_images(out, count); + return result; +} + +extern "C" JNIEXPORT jbyteArray JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeGenerateImage( + JNIEnv *env, jobject self, jstring prompt, jstring negative, jint width, jint height, jint steps, jdouble guidance, jdouble seed, jint previewInterval) { + jbyteArray output = nullptr; + jobject owner = nullptr; + try { + offgrid::VideoRequest request{}; + request.prompt = string(env, prompt); request.negative = string(env, negative); + request.width = width; request.height = height; request.steps = steps; request.guidance = guidance; request.seed = seed; + JavaVM *vm = nullptr; + if (env->GetJavaVM(&vm) != JNI_OK) throw std::runtime_error("Could not access the image host."); + owner = env->NewGlobalRef(self); + if (!owner) throw std::runtime_error("Could not retain the image host."); + auto klass = env->GetObjectClass(self); + auto progress = env->GetMethodID(klass, "imageProgress", "(II)V"); + auto preview = env->GetMethodID(klass, "imagePreview", "([BIIII)V"); + env->DeleteLocalRef(klass); + if (!progress || !preview) throw std::runtime_error("Missing image progress callback."); + runtime.image(request, [&](int step, int total) { + AttachedEnv thread(vm); + if (!thread.env) { runtime.cancel(); return; } + thread.env->CallVoidMethod(owner, progress, step, total); + if (thread.env->ExceptionCheck()) { thread.env->ExceptionClear(); runtime.cancel(); } + }, [&](const sd_image_t &image) { + const uint64_t size = uint64_t(image.width) * image.height * image.channel; + if (!image.data || image.width != uint32_t(width) || image.height != uint32_t(height) || (image.channel != 3 && image.channel != 4) || size > INT32_MAX) + throw std::runtime_error("The image engine returned invalid pixels."); + output = env->NewByteArray(static_cast(size)); + if (!output) throw std::runtime_error("Not enough memory to save the image."); + env->SetByteArrayRegion(output, 0, size, reinterpret_cast(image.data)); + }, previewInterval, [&](int step, const sd_image_t &image) { + const uint64_t size = uint64_t(image.width) * image.height * image.channel; + if (!image.data || (image.channel != 3 && image.channel != 4) || size > INT32_MAX) return; + AttachedEnv thread(vm); + if (!thread.env) return; + jbyteArray bytes = thread.env->NewByteArray(static_cast(size)); + if (bytes) { + thread.env->SetByteArrayRegion(bytes, 0, size, reinterpret_cast(image.data)); + thread.env->CallVoidMethod(owner, preview, bytes, image.width, image.height, image.channel, step); + thread.env->DeleteLocalRef(bytes); + } + if (thread.env->ExceptionCheck()) thread.env->ExceptionClear(); + }); + } catch (const std::exception &error) { + if (!env->ExceptionCheck()) { + auto klass = env->FindClass("java/lang/IllegalStateException"); + if (klass) { env->ThrowNew(klass, error.what()); env->DeleteLocalRef(klass); } + } + } + if (owner) env->DeleteGlobalRef(owner); + return output; +} diff --git a/android/app/src/main/java/ai/offgridmobile/MainApplication.kt b/android/app/src/main/java/ai/offgridmobile/MainApplication.kt index aeb5929fc..e5d3cae7a 100644 --- a/android/app/src/main/java/ai/offgridmobile/MainApplication.kt +++ b/android/app/src/main/java/ai/offgridmobile/MainApplication.kt @@ -30,6 +30,7 @@ class MainApplication : Application(), ReactApplication { // Packages that cannot be autolinked yet can be added manually here, for example: add(DownloadManagerPackage()) add(LocalDreamPackage()) + add(ai.offgridmobile.video.VideoGenerationPackage()) add(PDFExtractorPackage()) add(LiteRTPackage()) add(DeviceMemoryPackage()) diff --git a/android/app/src/main/java/ai/offgridmobile/downloads/SyncDownloadsModule.kt b/android/app/src/main/java/ai/offgridmobile/downloads/SyncDownloadsModule.kt index abdcff1e1..55a035086 100644 --- a/android/app/src/main/java/ai/offgridmobile/downloads/SyncDownloadsModule.kt +++ b/android/app/src/main/java/ai/offgridmobile/downloads/SyncDownloadsModule.kt @@ -1,6 +1,7 @@ package ai.offgridmobile.downloads import android.Manifest +import android.content.ContentValues import android.content.Intent import android.content.pm.PackageManager import android.net.Uri @@ -43,6 +44,38 @@ class SyncDownloadsModule( override fun getName(): String = "SyncDownloadsModule" + /** Export an app-owned file through scoped storage without broad media access. */ + @ReactMethod + fun saveFileToDownloads(sourcePath: String, displayName: String, mimeType: String, promise: Promise) { + Thread({ + var created: Uri? = null + try { + require(Build.VERSION.SDK_INT >= 29) { "Scoped Downloads requires Android 10." } + val source = File(sourcePath).canonicalFile + val roots = listOf(context.filesDir.canonicalFile, context.cacheDir.canonicalFile) + require(roots.any { source.path.startsWith(it.path + File.separator) } && source.isFile) { "The file is outside app storage." } + require(displayName == File(displayName).name && displayName.isNotBlank() && displayName.length <= 200) { "Invalid export filename." } + require(mimeType in listOf("video/mp4", "image/png", "image/jpeg", "image/webp")) { "Unsupported media type." } + val values = ContentValues().apply { + put(MediaStore.MediaColumns.DISPLAY_NAME, displayName) + put(MediaStore.MediaColumns.MIME_TYPE, mimeType) + put(MediaStore.MediaColumns.RELATIVE_PATH, Environment.DIRECTORY_DOWNLOADS) + put(MediaStore.MediaColumns.IS_PENDING, 1) + } + val uri = checkNotNull(context.contentResolver.insert(MediaStore.Downloads.EXTERNAL_CONTENT_URI, values)) + created = uri + checkNotNull(context.contentResolver.openOutputStream(uri)).use { output -> source.inputStream().use { it.copyTo(output) } } + values.clear() + values.put(MediaStore.MediaColumns.IS_PENDING, 0) + context.contentResolver.update(uri, values, null, null) + promise.resolve(uri.toString()) + } catch (error: Throwable) { + created?.let { runCatching { context.contentResolver.delete(it, null, null) } } + promise.reject("MEDIA_EXPORT_FAILED", error.message, error) + } + }, "offgrid-media-export").start() + } + @ReactMethod fun hasPermission(promise: Promise) { promise.resolve(granted() || allFilesAccess()) diff --git a/android/app/src/main/java/ai/offgridmobile/video/VideoEncoder.kt b/android/app/src/main/java/ai/offgridmobile/video/VideoEncoder.kt new file mode 100644 index 000000000..db8f3b2c4 --- /dev/null +++ b/android/app/src/main/java/ai/offgridmobile/video/VideoEncoder.kt @@ -0,0 +1,90 @@ +package ai.offgridmobile.video + +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaFormat +import android.media.MediaMuxer +import java.io.Closeable + +/** Platform MP4 adapter. Diffusion and job state remain in the shared runtime. */ +class VideoEncoder(path: String, private val width: Int, private val height: Int, private val fps: Int) : Closeable { + private val codec = MediaCodec.createEncoderByType("video/avc") + private val muxer = MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + private var track = -1 + private var started = false + private var frame = 0 + private val info = MediaCodec.BufferInfo() + init { + try { + val format = MediaFormat.createVideoFormat("video/avc", width, height).apply { + setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible) + setInteger(MediaFormat.KEY_BIT_RATE, width * height * fps * 2) + setInteger(MediaFormat.KEY_FRAME_RATE, fps) + setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) + } + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + codec.start() + } catch (error: Throwable) { codec.release(); muxer.release(); throw error } + } + fun append(rgb: ByteArray, channels: Int, cancelled: () -> Boolean) { + require(channels >= 3 && rgb.size == width * height * channels) + var index: Int + val deadline = System.nanoTime() + 30_000_000_000L + do { + check(!cancelled()) { "Video generation stopped." } + index = codec.dequeueInputBuffer(10_000) + drain(false, cancelled) + check(System.nanoTime() < deadline) { "Video encoder stalled." } + } while (index < 0) + val image = checkNotNull(codec.getInputImage(index)) { "Video encoder has no YUV input." } + for (y in 0 until height) for (x in 0 until width) { + val p = (y * width + x) * channels + val r = rgb[p].toInt() and 255; val g = rgb[p + 1].toInt() and 255; val b = rgb[p + 2].toInt() and 255 + fun put(plane: Int, px: Int, py: Int, value: Int) { + val target = image.planes[plane] + target.buffer.put(py * target.rowStride + px * target.pixelStride, value.coerceIn(0, 255).toByte()) + } + put(0, x, y, ((66 * r + 129 * g + 25 * b + 128) shr 8) + 16) + if (x % 2 == 0 && y % 2 == 0) { + put(1, x / 2, y / 2, ((-38 * r - 74 * g + 112 * b + 128) shr 8) + 128) + put(2, x / 2, y / 2, ((112 * r - 94 * g - 18 * b + 128) shr 8) + 128) + } + } + image.close() + codec.queueInputBuffer(index, 0, width * height * 3 / 2, frame++ * 1_000_000L / fps, 0) + drain(false, cancelled) + } + fun finish(cancelled: () -> Boolean) { + val deadline = System.nanoTime() + 30_000_000_000L + var index: Int + do { + check(!cancelled() && System.nanoTime() < deadline) { "Video encoding stopped." } + index = codec.dequeueInputBuffer(10_000); drain(false, cancelled) + } while (index < 0) + codec.queueInputBuffer(index, 0, 0, frame * 1_000_000L / fps, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + drain(true, cancelled) + } + private fun drain(finish: Boolean, cancelled: () -> Boolean) { + val deadline = System.nanoTime() + 30_000_000_000L + while (true) { + check(!cancelled() && System.nanoTime() < deadline) { "Video encoding stopped." } + val index = codec.dequeueOutputBuffer(info, if (finish) 10_000 else 0) + if (index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + check(!started); track = muxer.addTrack(codec.outputFormat); muxer.start(); started = true + } else if (index >= 0) { + val buffer = checkNotNull(codec.getOutputBuffer(index)) + if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) info.size = 0 + if (info.size > 0) { check(started); buffer.position(info.offset); buffer.limit(info.offset + info.size); muxer.writeSampleData(track, buffer, info) } + val done = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + codec.releaseOutputBuffer(index, false) + if (done) return + } else if (!finish) return + } + } + override fun close() { + try { codec.stop() } finally { + codec.release() + try { if (started) muxer.stop() } finally { muxer.release() } + } + } +} diff --git a/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationModule.kt b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationModule.kt new file mode 100644 index 000000000..09d50aa52 --- /dev/null +++ b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationModule.kt @@ -0,0 +1,350 @@ +package ai.offgridmobile.video + +import ai.offgridmobile.BuildConfig +import android.content.Intent +import android.view.Window +import android.view.WindowManager +import androidx.core.content.ContextCompat +import com.facebook.react.bridge.* +import com.facebook.react.common.LifecycleState +import com.facebook.react.modules.core.DeviceEventManagerModule +import java.io.File +import java.util.concurrent.Executors +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +class VideoGenerationModule(private val context: ReactApplicationContext) : ReactContextBaseJavaModule(context), LifecycleEventListener { + companion object { + init { System.loadLibrary("offgrid_video") } + // JNI owns one process-wide runtime, even when React recreates its bridge. + private val busy = AtomicBoolean(false) + @Volatile private var activeVideo: VideoGenerationModule? = null + @Volatile private var videoStatus: Map? = null + } + private val executor = Executors.newSingleThreadExecutor() + private val cancelled = AtomicBoolean(false) + private val videoScreenActive = AtomicBoolean(false) + private var awakeWindow: Window? = null + private var addedScreenFlag = false + private var encoder: VideoEncoder? = null + private var previewFile: File? = null + private var videoSteps = 0 + init { context.addLifecycleEventListener(this) } + private fun releaseScreenFlag() { + if (addedScreenFlag) awakeWindow?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + awakeWindow = null + addedScreenFlag = false + } + private fun updateScreenFlag() { + val window = if (videoScreenActive.get() && context.lifecycleState == LifecycleState.RESUMED) context.currentActivity?.window else null + if (window === awakeWindow) return + releaseScreenFlag() + if (window != null) { + awakeWindow = window + addedScreenFlag = window.attributes.flags and WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON == 0 + if (addedScreenFlag) window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + } + } + private fun keepVideoScreenAwake(active: Boolean) { + videoScreenActive.set(active) + UiThreadUtil.runOnUiThread { updateScreenFlag() } + } + override fun onHostResume() { updateScreenFlag() } + override fun onHostPause() { releaseScreenFlag() } + override fun onHostDestroy() { releaseScreenFlag() } + override fun getName() = "VideoGenerationModule" + // Start images run in sd.cpp; the in-process ESRGAN upscaler sharpens film shots. + override fun getConstants(): Map = mapOf("supportsInitImage" to true, "supportsUpscale" to true) + private external fun nativeGenerate(weight: String, vae: String, encoder: String, prompt: String, negative: String, width: Int, height: Int, frames: Int, fps: Int, steps: Int, guidance: Double, seed: Double, llm: String, embeddings: String, audioVae: String, flowShift: Double, diagnosticBackend: String, initImage: ByteArray?) + private external fun nativeUpscalerOpen(modelPath: String, threads: Int): Long + private external fun nativeUpscalerClose(handle: Long) + private external fun nativeUpscale(handle: Long, rgb: ByteArray, width: Int, height: Int, factor: Int): ByteArray? + + /** A photo as RGB at the clip size, centre-cropped to fill. */ + private fun initImageRgb(path: String?, width: Int, height: Int): ByteArray? { + if (path.isNullOrEmpty()) return null + val source = android.graphics.BitmapFactory.decodeFile(path.removePrefix("file://")) ?: return null + val scale = maxOf(width.toFloat() / source.width, height.toFloat() / source.height) + val crop = android.graphics.Bitmap.createBitmap(width, height, android.graphics.Bitmap.Config.ARGB_8888) + android.graphics.Canvas(crop).apply { + translate((width - source.width * scale) / 2f, (height - source.height * scale) / 2f) + scale(scale, scale) + drawBitmap(source, 0f, 0f, android.graphics.Paint(android.graphics.Paint.FILTER_BITMAP_FLAG)) + } + source.recycle() + return toRgb(crop).also { crop.recycle() } + } + + private fun toRgb(bitmap: android.graphics.Bitmap): ByteArray { + val pixels = IntArray(bitmap.width * bitmap.height) + bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height) + val rgb = ByteArray(pixels.size * 3) + for (i in pixels.indices) { + rgb[i * 3] = (pixels[i] shr 16).toByte(); rgb[i * 3 + 1] = (pixels[i] shr 8).toByte(); rgb[i * 3 + 2] = pixels[i].toByte() + } + return rgb + } + + /** A film owns one foreground service for its whole run; a clip inside it starts none of its own. */ + private fun filmServiceRunning(): Boolean = runCatching { + Class.forName("ai.offgridmobile.film.FilmRenderService").getDeclaredField("running").apply { isAccessible = true }.getBoolean(null) + }.getOrDefault(false) + + /** Upscale every frame of a clip with sd.cpp's ESRGAN, and write it at the clip's own rate. */ + @ReactMethod fun upscaleVideo(input: ReadableMap, promise: Promise) { + cancelled.set(false) + executor.execute { + val retriever = android.media.MediaMetadataRetriever() + var handle = 0L + try { + val inputPath = checkNotNull(input.getString("inputPath")); val outputPath = checkNotNull(input.getString("outputPath")) + val factor = input.getInt("scale") + retriever.setDataSource(inputPath) + val durationUs = (retriever.extractMetadata(android.media.MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLong() ?: 0L) * 1000 + val frameCount = retriever.extractMetadata(android.media.MediaMetadataRetriever.METADATA_KEY_VIDEO_FRAME_COUNT)?.toIntOrNull() ?: 0 + check(durationUs > 0 && frameCount > 0) { "The clip has no frames." } + val fps = maxOf(1, Math.round(frameCount * 1_000_000.0 / durationUs).toInt()) + handle = nativeUpscalerOpen(checkNotNull(input.getString("modelPath")), Runtime.getRuntime().availableProcessors().coerceIn(2, 6)) + check(handle != 0L) { "The upscaler model did not load." } + var writer: VideoEncoder? = null + try { + for (index in 0 until frameCount) { + check(!cancelled.get()) { "Upscaling stopped." } + val frame = retriever.getFrameAtTime(index * 1_000_000L / fps, android.media.MediaMetadataRetriever.OPTION_CLOSEST) ?: continue + val rgb = toRgb(frame); val w = frame.width; val h = frame.height + frame.recycle() + val up = nativeUpscale(handle, rgb, w, h, factor) ?: error("The upscaler failed on a frame.") + val target = writer ?: VideoEncoder(outputPath, w * factor, h * factor, fps).also { writer = it } + target.append(up, 3) { cancelled.get() } + } + writer?.finish { cancelled.get() } + } finally { writer?.close() } + promise.resolve(Arguments.createMap().apply { putString("path", outputPath) }) + } catch (error: Throwable) { + promise.reject("VIDEO_UPSCALE_FAILED", error.message ?: "Could not upscale the clip.", error) + } finally { + if (handle != 0L) nativeUpscalerClose(handle) + runCatching { retriever.release() } + } + } + } + private external fun nativeCancel() + private external fun nativePrepare() + private external fun nativeSetRuntimeDirectory(path: String) + private fun prepareHexagonRuntime() { + val directory = File(context.filesDir, "video-hexagon").apply { mkdirs() } + val names = context.assets.list("video-hexagon") ?: emptyArray() + check(names.isNotEmpty()) { "The video NPU runtime is missing from this build." } + for (name in names) { + check(name.matches(Regex("liboffgrid-video-htp-v[0-9]+\\.so"))) + val temporary = File(directory, "$name.tmp") + context.assets.open("video-hexagon/$name").use { input -> + temporary.outputStream().use { output -> input.copyTo(output) } + } + check(temporary.renameTo(File(directory, name))) { "Could not install the video NPU runtime." } + } + nativeSetRuntimeDirectory(directory.absolutePath) + } + @ReactMethod fun addListener(name: String) {} + @ReactMethod fun removeListeners(count: Int) {} + private fun stop() { cancelled.set(true); nativeCancel() } + @ReactMethod fun cancel(promise: Promise) { (activeVideo ?: this).stop(); promise.resolve(null) } + @ReactMethod fun getVideoStatus(outputPath: String, promise: Promise) { + val status = videoStatus?.takeIf { it["path"] == outputPath } + keepVideoScreenAwake(status?.get("phase") == "running") + promise.resolve(status?.let { Arguments.makeNativeMap(it) }) + } + // Called synchronously by JNI while its worker owns the runtime. + fun conditioning(backend: String) { emit("conditioning", 0, 0, backend) } + fun progress(step: Int, total: Int) { + // Loading and VAE counters share this callback; preserve the sampling snapshot. + if (total != videoSteps || step !in 0..videoSteps) return + emit("generating", step, total) + } + fun decoding(completed: Int, total: Int) { emit("decoding", completed, total) } + fun preview(rgb: ByteArray, width: Int, height: Int) { + if (cancelled.get()) return + val destination = previewFile ?: return + runCatching { + saveRgbPng(rgb, width, height, destination) + emit("encoding", 0, 0, preview = Arguments.createMap().apply { + putString("path", destination.path); putInt("width", width); putInt("height", height) + }) + }.onFailure { destination.delete() } + } + private fun saveRgbPng(rgb: ByteArray, width: Int, height: Int, destination: File, channels: Int = 3) { + require(width > 0 && height > 0 && channels in 3..4 && rgb.size.toLong() == width.toLong() * height * channels) + val pixels = IntArray(width * height) { i -> + val offset = i * channels + android.graphics.Color.argb(if (channels == 4) rgb[offset + 3].toInt() and 255 else 255, + rgb[offset].toInt() and 255, rgb[offset + 1].toInt() and 255, rgb[offset + 2].toInt() and 255) + } + val bitmap = android.graphics.Bitmap.createBitmap(pixels, width, height, android.graphics.Bitmap.Config.ARGB_8888) + try { destination.outputStream().use { check(bitmap.compress(android.graphics.Bitmap.CompressFormat.PNG, 100, it)) } } + finally { bitmap.recycle() } + } + fun frame(rgb: ByteArray, width: Int, height: Int, channels: Int) { + check(!cancelled.get()) { "Video generation stopped." } + emit("encoding", 0, 0) + checkNotNull(encoder).append(rgb, channels) { cancelled.get() } + } + private fun emit(stage: String, step: Int, total: Int, backend: String? = null, preview: WritableMap? = null) { + videoStatus = videoStatus?.plus(mapOf( + "stage" to stage, "step" to step, "total" to total, "backend" to backend, + ))?.let { status -> if (preview != null) status + ("preview" to preview.toHashMap()) else status } + if (!context.hasActiveReactInstance()) return + context.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java) + .emit("VideoGenerationProgress", Arguments.createMap().apply { putString("stage", stage); putInt("step", step); putInt("total", total); if (backend != null) putString("backend", backend); if (preview != null) putMap("preview", preview) }) + } + @ReactMethod fun generate(input: ReadableMap, promise: Promise) { + if (!busy.compareAndSet(false, true)) { promise.reject("VIDEO_BUSY", "Video generation is already running."); return } + activeVideo = this + videoStatus = mapOf("path" to input.getString("outputPath"), "phase" to "running", "stage" to "preparing", "step" to 0, "total" to 0) + cancelled.set(false); nativePrepare() + executor.execute { + var output: File? = null + var terminal: Map = mapOf("phase" to "failed", "code" to "VIDEO_FAILED", "error" to "Video generation failed.") + try { + keepVideoScreenAwake(true) + prepareHexagonRuntime() + val destination = File(checkNotNull(input.getString("outputPath"))) + output = destination + previewFile = File(destination.path + ".preview.png") + VideoGenerationService.admission = CompletableFuture() + VideoGenerationService.cancel = { stop() } + if (filmServiceRunning()) VideoGenerationService.admission.complete(Unit) + else ContextCompat.startForegroundService(context, Intent(context, VideoGenerationService::class.java)) + VideoGenerationService.admission.get(5, TimeUnit.SECONDS) + check(!cancelled.get()) { "Video generation stopped." } + videoSteps = input.getInt("steps") + emit("preparing", 0, videoSteps) + VideoEncoder(destination.path, input.getInt("width"), input.getInt("height"), input.getInt("fps")).use { writer -> + encoder = writer + nativeGenerate(checkNotNull(input.getString("weight")), checkNotNull(input.getString("vae")), if (input.hasKey("encoder")) input.getString("encoder") ?: "" else "", + checkNotNull(input.getString("prompt")), input.getString("negativePrompt") ?: "", input.getInt("width"), input.getInt("height"), + input.getInt("frames"), input.getInt("fps"), input.getInt("steps"), input.getDouble("guidance"), input.getDouble("seed"), + if (input.hasKey("llm")) input.getString("llm") ?: "" else "", + if (input.hasKey("embeddings")) input.getString("embeddings") ?: "" else "", + if (input.hasKey("audioVae")) input.getString("audioVae") ?: "" else "", + input.getDouble("flowShift"), when { + BuildConfig.DEBUG && input.hasKey("diagnosticBackend") -> input.getString("diagnosticBackend") ?: "auto" + // Fastest settings' choice: "cpu" runs on the CPU, "gpu" skips the NPU. + input.hasKey("backend") -> input.getString("backend") ?: "auto" + else -> "auto" + }, + initImageRgb(if (input.hasKey("initImage")) input.getString("initImage") else null, input.getInt("width"), input.getInt("height"))) + writer.finish { cancelled.get() } + } + check(destination.length() > 0) { "Video encoder produced no file." } + terminal = mapOf("phase" to "succeeded") + if (context.hasActiveReactInstance()) + promise.resolve(Arguments.createMap().apply { putString("path", destination.path) }) + } catch (error: Throwable) { + val code = if (cancelled.get()) "VIDEO_CANCELLED" else "VIDEO_FAILED" + terminal = mapOf("phase" to "failed", "code" to code, "error" to (error.message ?: "Video generation failed.")) + discard(output) + if (context.hasActiveReactInstance()) promise.reject(code, error.message, error) + } finally { + keepVideoScreenAwake(false) + encoder = null; previewFile = null; VideoGenerationService.cancel = null + context.stopService(Intent(context, VideoGenerationService::class.java)) + videoStatus = videoStatus?.plus(terminal) + activeVideo = null + busy.set(false) + } + } + } + private external fun nativeLoadImage(path: String, weight: String, vae: String, llm: String, threads: Int, cpuOnly: Boolean, family: String, sampler: String, scheduler: String) + private external fun nativeUnloadImage() + private external fun nativeImagePath(): String + private external fun nativeGenerateImage(prompt: String, negative: String, width: Int, height: Int, steps: Int, guidance: Double, seed: Double, previewInterval: Int): ByteArray + private var imagePreviewFile: File? = null + private var imageSteps = 0 + fun imagePreview(pixels: ByteArray, width: Int, height: Int, channels: Int, step: Int) { + if (cancelled.get()) return + val file = imagePreviewFile ?: return + runCatching { + saveRgbPng(pixels, width, height, file, channels) + if (context.hasActiveReactInstance()) context.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java) + .emit("SDImageProgress", Arguments.createMap().apply { + putInt("step", step); putInt("totalSteps", imageSteps) + putDouble("progress", if (imageSteps > 0) step.toDouble() / imageSteps else 0.0) + putString("previewPath", file.path) + }) + } + } + fun imageProgress(step: Int, total: Int) { + if (!context.hasActiveReactInstance()) return + context.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java).emit("SDImageProgress", Arguments.createMap().apply { + putInt("step", step); putInt("totalSteps", total); putDouble("progress", if (total > 0) step.toDouble() / total else 0.0) + }) + } + @ReactMethod fun getLoadedImagePath(promise: Promise) { promise.resolve(nativeImagePath().ifEmpty { null }) } + @ReactMethod fun loadImageModel(input: ReadableMap, promise: Promise) { + if (!busy.compareAndSet(false, true)) { promise.reject("IMAGE_BUSY", "Image or video generation is running."); return } + cancelled.set(false); nativePrepare() + executor.execute { + try { + prepareHexagonRuntime() + nativeLoadImage(checkNotNull(input.getString("modelPath")), checkNotNull(input.getString("weight")), + if (input.hasKey("vae")) input.getString("vae") ?: "" else "", + if (input.hasKey("llm")) input.getString("llm") ?: "" else "", + input.getInt("threads"), input.getBoolean("cpuOnly"), + if (input.hasKey("family")) input.getString("family") ?: "" else "", + if (input.hasKey("sampler")) input.getString("sampler") ?: "" else "", + if (input.hasKey("scheduler")) input.getString("scheduler") ?: "" else "") + promise.resolve(true) + } catch (error: Throwable) { promise.reject("IMAGE_LOAD_FAILED", error.message, error) } + finally { busy.set(false) } + } + } + @ReactMethod fun unloadImageModel(promise: Promise) { + if (!busy.compareAndSet(false, true)) { promise.reject("IMAGE_BUSY", "Image or video generation is running."); return } + executor.execute { + try { nativeUnloadImage(); promise.resolve(true) } + catch (error: Throwable) { promise.reject("IMAGE_UNLOAD_FAILED", error.message, error) } + finally { busy.set(false) } + } + } + @ReactMethod fun generateImage(input: ReadableMap, promise: Promise) { + if (!busy.compareAndSet(false, true)) { promise.reject("IMAGE_BUSY", "Image or video generation is running."); return } + cancelled.set(false); nativePrepare() + executor.execute { + var output: File? = null + try { + val width = input.getInt("width"); val height = input.getInt("height") + require(width in 64..2048 && height in 64..2048 && width % 16 == 0 && height % 16 == 0) + output = File(checkNotNull(input.getString("outputPath"))) + VideoGenerationService.admission = CompletableFuture() + VideoGenerationService.cancel = { stop() } + ContextCompat.startForegroundService(context, Intent(context, VideoGenerationService::class.java).putExtra("modality", "image")) + VideoGenerationService.admission.get(5, TimeUnit.SECONDS) + imagePreviewFile = File(output.path + ".preview.png") + imageSteps = input.getInt("steps") + val bytes = nativeGenerateImage(checkNotNull(input.getString("prompt")), input.getString("negativePrompt") ?: "", width, height, input.getInt("steps"), input.getDouble("guidanceScale"), input.getDouble("seed"), if (input.hasKey("previewInterval")) input.getInt("previewInterval").coerceAtLeast(0) else 0) + check(!cancelled.get()) { "Image generation stopped." } + saveRgbPng(bytes, width, height, output, bytes.size / (width * height)) + check(!cancelled.get()) { "Image generation stopped." } + promise.resolve(Arguments.createMap().apply { + putString("imagePath", output.path); putInt("width", width); putInt("height", height); putDouble("seed", input.getDouble("seed")); putString("id", input.getString("id")) + }) + } catch (error: Throwable) { discard(output); promise.reject("IMAGE_FAILED", error.message, error) } + finally { discard(imagePreviewFile); imagePreviewFile = null; VideoGenerationService.cancel = null; context.stopService(Intent(context, VideoGenerationService::class.java)); busy.set(false) } + } + } + /** Remove a partial output; a file that cannot be removed is logged, not ignored. */ + private fun discard(file: File?) { + if (file != null && file.exists() && !file.delete()) + android.util.Log.w("OffGridVideo", "Could not delete ${file.path}") + } + + override fun invalidate() { + keepVideoScreenAwake(false) + context.removeLifecycleEventListener(this) + // A bridge reload must not cancel a video owned by the foreground service. + // The replacement bridge reads its retained status and completion. + if (activeVideo == null) stop() + executor.shutdown(); super.invalidate() + } +} diff --git a/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationPackage.kt b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationPackage.kt new file mode 100644 index 000000000..9c0b2a1d3 --- /dev/null +++ b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationPackage.kt @@ -0,0 +1,9 @@ +package ai.offgridmobile.video +import com.facebook.react.ReactPackage +import com.facebook.react.bridge.NativeModule +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.uimanager.ViewManager +class VideoGenerationPackage : ReactPackage { + override fun createNativeModules(context: ReactApplicationContext): List = listOf(VideoGenerationModule(context)) + override fun createViewManagers(context: ReactApplicationContext): List> = emptyList() +} diff --git a/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationService.kt b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationService.kt new file mode 100644 index 000000000..c6301c118 --- /dev/null +++ b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationService.kt @@ -0,0 +1,36 @@ +package ai.offgridmobile.video + +import android.app.* +import android.content.Intent +import android.content.pm.ServiceInfo +import android.os.Build +import android.os.IBinder +import java.util.concurrent.CompletableFuture + +class VideoGenerationService : Service() { + companion object { + const val CHANNEL = "offgrid-video-generation" + var admission = CompletableFuture() + var cancel: (() -> Unit)? = null + } + override fun onBind(intent: Intent?): IBinder? = null + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + if (intent?.action == "cancel") { cancel?.invoke(); return START_NOT_STICKY } + try { + val manager = getSystemService(NotificationManager::class.java) + if (Build.VERSION.SDK_INT >= 26) manager.createNotificationChannel(NotificationChannel(CHANNEL, "Media generation", NotificationManager.IMPORTANCE_LOW)) + val stop = PendingIntent.getService(this, 0, Intent(this, VideoGenerationService::class.java).setAction("cancel"), PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT) + val builder = if (Build.VERSION.SDK_INT >= 26) Notification.Builder(this, CHANNEL) else Notification.Builder(this) + val notification = builder.setContentTitle(if (intent?.getStringExtra("modality") == "image") "Generating image" else "Generating video").setContentText("Off Grid is using this device.") + .setSmallIcon(android.R.drawable.ic_menu_slideshow).setOngoing(true) + .addAction(Notification.Action.Builder(null, "Stop", stop).build()).build() + if (Build.VERSION.SDK_INT >= 35) startForeground(7413, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROCESSING) + else if (Build.VERSION.SDK_INT >= 34) startForeground(7413, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE) + else startForeground(7413, notification) + admission.complete(Unit) + } catch (error: Throwable) { admission.completeExceptionally(error); cancel?.invoke(); stopSelf() } + return START_NOT_STICKY + } + override fun onTimeout(startId: Int, fgsType: Int) { cancel?.invoke(); stopSelf() } + override fun onDestroy() { cancel?.invoke(); super.onDestroy() } +} diff --git a/android/gradle.properties b/android/gradle.properties index fb614558a..4a02a3165 100644 --- a/android/gradle.properties +++ b/android/gradle.properties @@ -25,7 +25,14 @@ android.useAndroidX=true # Use this property to specify which architecture you want to build. # You can also override it from the CLI using # ./gradlew -PreactNativeArchitectures=x86_64 -reactNativeArchitectures=armeabi-v7a,arm64-v8a,x86,x86_64 +# +# 64-bit only. The 32-bit targets were the React Native template's default, never a choice: Play has +# required 64-bit since 2019, and a 32-bit process has about 3 GB of usable address space, so it +# could never load the models this app is for. Building them cost time and broke the build outright +# - the vendored Vulkan backend streams a vk::Buffer, which is a pointer on 64-bit but a uint64_t +# with no operator<< on armeabi-v7a, so every Android build failed on an ABI nothing we ship runs. +# x86_64 stays for the emulator. +reactNativeArchitectures=arm64-v8a,x86_64 # Use this property to enable support to the new architecture. # This will allow you to use TurboModules and the Fabric render in diff --git a/catalog/models.json b/catalog/models.json index cf8fc9000..e14a9efce 100644 --- a/catalog/models.json +++ b/catalog/models.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "updatedAt": "2026-10-23", + "updatedAt": "2026-10-27", "sections": { "models": [ { @@ -26,6 +26,646 @@ "role": "primary" } ] + }, + { + "id": "leejet/FLUX.2-klein-4B-GGUF", + "quantLabel": "Full quality", + "sourceModelId": "black-forest-labs/FLUX.2-klein-4B", + "name": "FLUX.2 klein 4B", + "kind": "image", + "tags": [ + "2026", + "Fast", + "Image editing" + ], + "org": "Black Forest Labs", + "description": "The small FLUX from Black Forest Labs: 4 steps, sharp detail and text, edits with a reference image. Apache-2.0. The Qwen3 text encoder and VAE are included in the download.", + "minRamGb": 11, + "quant": "Q8_0", + "releaseDate": "2026-01-15", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "flux-2-klein-4b-Q8_0.gguf", + "url": "https://huggingface.co/leejet/FLUX.2-klein-4B-GGUF/resolve/3b1f5a9dc3abb32238b053aeb3d823c30afdacbd/flux-2-klein-4b-Q8_0.gguf", + "role": "primary", + "sizeBytes": 4300629440, + "sha256": "0bba6951258ec8f92d51114a8fa13e66828297bfff58a738f52729b3ef66fa28" + }, + { + "name": "Qwen3-4B-Q4_K_M.gguf", + "url": "https://huggingface.co/unsloth/Qwen3-4B-GGUF/resolve/22c9fc8a8c7700b76a1789366280a6a5a1ad1120/Qwen3-4B-Q4_K_M.gguf", + "role": "aux", + "sizeBytes": 2497281312, + "sha256": "f6f851777709861056efcdad3af01da38b31223a3ba26e61a4f8bf3a2195813a" + }, + { + "name": "flux2-vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/Flux-2-Klein-4B/resolve/533c4cbc3d3b89af1743d9466c434e89810391c0/split_files/vae/flux2-vae.safetensors", + "role": "aux", + "sizeBytes": 336211292, + "sha256": "868fe7b343cc8f3a19dbcfcafbc3d5f888802be3f89bd81b65b3621a066ce8f3" + } + ] + }, + { + "id": "leejet/FLUX.2-klein-4B-GGUF-Q4", + "quantOf": "leejet/FLUX.2-klein-4B-GGUF", + "quantLabel": "Light", + "sourceModelId": "black-forest-labs/FLUX.2-klein-4B", + "name": "FLUX.2 klein 4B (Light)", + "kind": "image", + "tags": [ + "2026", + "Light", + "Fast", + "Image editing" + ], + "org": "Black Forest Labs", + "description": "The small FLUX from Black Forest Labs: 4 steps, sharp detail and text, edits with a reference image. Apache-2.0. The Qwen3 text encoder and VAE are included in the download.", + "minRamGb": 7, + "quant": "Q4_0", + "releaseDate": "2026-01-15", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "flux-2-klein-4b-Q4_0.gguf", + "url": "https://huggingface.co/leejet/FLUX.2-klein-4B-GGUF/resolve/3b1f5a9dc3abb32238b053aeb3d823c30afdacbd/flux-2-klein-4b-Q4_0.gguf", + "role": "primary", + "sizeBytes": 2460378560, + "sha256": "d1023499ef3f2f82ff7c50e6778495195c1b6cc34835741778868428111f9ff4" + }, + { + "name": "Qwen3-4B-Q4_K_M.gguf", + "url": "https://huggingface.co/unsloth/Qwen3-4B-GGUF/resolve/22c9fc8a8c7700b76a1789366280a6a5a1ad1120/Qwen3-4B-Q4_K_M.gguf", + "role": "aux", + "sizeBytes": 2497281312, + "sha256": "f6f851777709861056efcdad3af01da38b31223a3ba26e61a4f8bf3a2195813a" + }, + { + "name": "flux2-vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/Flux-2-Klein-4B/resolve/533c4cbc3d3b89af1743d9466c434e89810391c0/split_files/vae/flux2-vae.safetensors", + "role": "aux", + "sizeBytes": 336211292, + "sha256": "868fe7b343cc8f3a19dbcfcafbc3d5f888802be3f89bd81b65b3621a066ce8f3" + } + ] + }, + { + "id": "leejet/Z-Image-Turbo-GGUF", + "quantLabel": "Full quality", + "sourceModelId": "Tongyi-MAI/Z-Image-Turbo", + "name": "Z-Image Turbo (2026)", + "kind": "image", + "tags": [ + "Recommended", + "2026", + "Top quality" + ], + "org": "Alibaba Tongyi", + "description": "1024px in 8 steps, strong bilingual text. Apache-2.0. The Qwen3 text encoder and VAE are included in the download.", + "minRamGb": 15, + "quant": "Q4_K", + "imageModes": [ + "txt2img" + ], + "files": [ + { + "name": "z_image_turbo-Q4_K.gguf", + "url": "https://huggingface.co/leejet/Z-Image-Turbo-GGUF/resolve/c61c0e422dc8b541b7548cf33a4ef8302b0f8085/z_image_turbo-Q4_K.gguf", + "role": "primary", + "sizeBytes": 3864250304, + "sha256": "14b375ab4f226bc5378f68f37e899ef3c2242b8541e61e2bc1aff40976086fbd" + }, + { + "name": "Qwen3-4B-Instruct-2507-Q4_K_M.gguf", + "url": "https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/a06e946bb6b655725eafa393f4a9745d460374c9/Qwen3-4B-Instruct-2507-Q4_K_M.gguf", + "role": "aux", + "sizeBytes": 2497281120, + "sha256": "3605803b982cb64aead44f6c1b2ae36e3acdb41d8e46c8a94c6533bc4c67e597" + }, + { + "name": "ae.safetensors", + "url": "https://huggingface.co/second-state/FLUX.1-schnell-GGUF/resolve/8c45a2ba25e2d02bd34230989fb54983f39e44ec/ae.safetensors", + "role": "aux", + "sizeBytes": 335304388, + "sha256": "afc8e28272cd15db3919bacdb6918ce9c1ed22e96cb12c4d5ed0fba823529e38" + } + ] + }, + { + "id": "leejet/Z-Image-Turbo-GGUF-Q3", + "quantOf": "leejet/Z-Image-Turbo-GGUF", + "quantLabel": "Light", + "sourceModelId": "Tongyi-MAI/Z-Image-Turbo", + "name": "Z-Image Turbo (Light)", + "kind": "image", + "tags": [ + "2026", + "Light" + ], + "org": "Alibaba Tongyi", + "description": "1024px in 8 steps, strong bilingual text. Apache-2.0. The Qwen3 text encoder and VAE are included in the download.", + "minRamGb": 11, + "quant": "Q3_K", + "imageModes": [ + "txt2img" + ], + "files": [ + { + "name": "z_image_turbo-Q3_K.gguf", + "url": "https://huggingface.co/leejet/Z-Image-Turbo-GGUF/resolve/c61c0e422dc8b541b7548cf33a4ef8302b0f8085/z_image_turbo-Q3_K.gguf", + "role": "primary", + "sizeBytes": 3143559104, + "sha256": "4b44bdaa7814f20d7cf144e3939bd93aa32f50660204dd0c2aea5c5376232980" + }, + { + "name": "Qwen3-4B-Instruct-2507-Q4_K_M.gguf", + "url": "https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/a06e946bb6b655725eafa393f4a9745d460374c9/Qwen3-4B-Instruct-2507-Q4_K_M.gguf", + "role": "aux", + "sizeBytes": 2497281120, + "sha256": "3605803b982cb64aead44f6c1b2ae36e3acdb41d8e46c8a94c6533bc4c67e597" + }, + { + "name": "ae.safetensors", + "url": "https://huggingface.co/second-state/FLUX.1-schnell-GGUF/resolve/8c45a2ba25e2d02bd34230989fb54983f39e44ec/ae.safetensors", + "role": "aux", + "sizeBytes": 335304388, + "sha256": "afc8e28272cd15db3919bacdb6918ce9c1ed22e96cb12c4d5ed0fba823529e38" + } + ] + }, + { + "id": "mzwing/SDXL-Lightning-GGUF", + "supersededBy": "leejet/FLUX.2-klein-4B-GGUF", + "name": "SDXL Lightning (4-step)", + "kind": "image", + "tags": [ + "Recommended", + "Fast" + ], + "org": "ByteDance", + "description": "Near-SDXL quality at 1024px in 4 steps (~7× faster). ~4GB model. Best balance — recommended.", + "minRamGb": 8, + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "sdxl_lightning_4step.q8_0.gguf", + "url": "https://huggingface.co/mzwing/SDXL-Lightning-GGUF/resolve/main/sdxl_lightning_4step.q8_0.gguf", + "role": "primary", + "sizeBytes": 4098988672 + } + ] + }, + { + "id": "OlegSkutte/sdxl-turbo-GGUF", + "supersededBy": "leejet/FLUX.2-klein-4B-GGUF", + "name": "SDXL Turbo (fast drafts)", + "kind": "image", + "tags": [ + "Fastest", + "Drafts" + ], + "org": "Stability AI", + "description": "Distilled SDXL — 1-4 steps, ~10s drafts at 512px. Fastest option; lower fidelity.", + "minRamGb": 8, + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "sd_xl_turbo_1.0.q8_0.gguf", + "url": "https://huggingface.co/OlegSkutte/sdxl-turbo-GGUF/resolve/main/sd_xl_turbo_1.0.q8_0.gguf", + "role": "primary", + "sizeBytes": 4098988672 + } + ] + }, + { + "id": "offgrid-ai/realvisxl-v5.0-GGUF", + "quantLabel": "Full quality", + "name": "RealVisXL v5.0 (photoreal)", + "kind": "image", + "tags": [ + "High quality", + "Photoreal" + ], + "org": "RealVis", + "description": "Top photorealism SDXL — Off Grid GGUF build of SG161222/RealVisXL_V5.0.", + "minRamGb": 8, + "quant": "Q8_0", + "releaseDate": "2024-08-05", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "realvisxl-v5.0-Q8_0.gguf", + "url": "https://huggingface.co/off-grid-ai/realvisxl-v5.0-GGUF/resolve/main/realvisxl-v5.0-Q8_0.gguf", + "role": "primary", + "sizeBytes": 4180188864 + } + ] + }, + { + "id": "offgrid-ai/realvisxl-v5.0-GGUF-Q4", + "quantOf": "offgrid-ai/realvisxl-v5.0-GGUF", + "quantLabel": "Light", + "name": "RealVisXL v5.0 (Light)", + "kind": "image", + "tags": [ + "Photoreal", + "Light" + ], + "org": "RealVis", + "description": "Top photorealism SDXL. Q4 quant: ~35% less memory, small quality trade-off. Runs on a 16GB Mac. Off Grid GGUF build of SG161222/RealVisXL_V5.0.", + "minRamGb": 8, + "quant": "Q4_K", + "releaseDate": "2024-08-05", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "realvisxl-v5.0-Q4_K.gguf", + "url": "https://huggingface.co/off-grid-ai/realvisxl-v5.0-GGUF/resolve/main/realvisxl-v5.0-Q4_K.gguf", + "role": "primary", + "sizeBytes": 2797755008 + } + ] + }, + { + "id": "offgrid-ai/realvisxl-v5.0-lightning-GGUF", + "quantLabel": "Full quality", + "name": "RealVisXL v5.0 Lightning (photoreal)", + "kind": "image", + "tags": [ + "Photoreal" + ], + "org": "RealVis", + "description": "Photoreal SDXL, few-step (fast) — Off Grid GGUF build of SG161222/RealVisXL_V5.0_Lightning.", + "minRamGb": 8, + "quant": "Q8_0", + "releaseDate": "2024-09-02", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "realvisxl-v5.0-lightning-Q8_0.gguf", + "url": "https://huggingface.co/off-grid-ai/realvisxl-v5.0-lightning-GGUF/resolve/main/realvisxl-v5.0-lightning-Q8_0.gguf", + "role": "primary", + "sizeBytes": 4180188864 + } + ] + }, + { + "id": "offgrid-ai/realvisxl-v5.0-lightning-GGUF-Q4", + "quantOf": "offgrid-ai/realvisxl-v5.0-lightning-GGUF", + "quantLabel": "Light", + "name": "RealVisXL v5.0 Lightning (Light)", + "kind": "image", + "tags": [ + "Fast", + "Photoreal", + "Light" + ], + "org": "RealVis", + "description": "Photoreal SDXL, few-step (fast). Q4 quant: ~35% less memory, small quality trade-off. Runs on a 16GB Mac. Off Grid GGUF build of SG161222/RealVisXL_V5.0_Lightning.", + "minRamGb": 8, + "quant": "Q4_K", + "releaseDate": "2024-09-02", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "realvisxl-v5.0-lightning-Q4_K.gguf", + "url": "https://huggingface.co/off-grid-ai/realvisxl-v5.0-lightning-GGUF/resolve/main/realvisxl-v5.0-lightning-Q4_K.gguf", + "role": "primary", + "sizeBytes": 2797755008 + } + ] + }, + { + "id": "offgrid-ai/dreamshaper-xl-v2-turbo-GGUF", + "quantLabel": "Full quality", + "name": "DreamShaper XL v2 Turbo (versatile)", + "kind": "image", + "tags": [ + "Versatile" + ], + "org": "Lykon", + "description": "The all-rounder — photoreal, art, fantasy, 3D. Off Grid GGUF build of Lykon/dreamshaper-xl-v2-turbo. Full Q8 quant (best quality); best on 24GB+ RAM.", + "minRamGb": 8, + "quant": "Q8_0", + "releaseDate": "2024-02-07", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "dreamshaper-xl-v2-turbo-Q8_0.gguf", + "url": "https://huggingface.co/off-grid-ai/dreamshaper-xl-v2-turbo-GGUF/resolve/main/dreamshaper-xl-v2-turbo-Q8_0.gguf", + "role": "primary", + "sizeBytes": 4180813664 + } + ] + }, + { + "id": "offgrid-ai/dreamshaper-xl-v2-turbo-GGUF-Q4", + "quantOf": "offgrid-ai/dreamshaper-xl-v2-turbo-GGUF", + "quantLabel": "Light", + "name": "DreamShaper XL v2 Turbo (Light)", + "kind": "image", + "tags": [ + "Versatile", + "Fast", + "Light" + ], + "org": "Lykon", + "description": "The all-rounder — photoreal, art, fantasy, 3D. Q4 quant: ~35% less memory than the full model, small quality trade-off. Runs on a 16GB Mac. Off Grid GGUF build of Lykon/dreamshaper-xl-v2-turbo.", + "minRamGb": 8, + "quant": "Q4_K", + "releaseDate": "2024-02-07", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "dreamshaper-xl-v2-turbo-Q4_K.gguf", + "url": "https://huggingface.co/off-grid-ai/dreamshaper-xl-v2-turbo-GGUF/resolve/main/dreamshaper-xl-v2-turbo-Q4_K.gguf", + "role": "primary", + "sizeBytes": 2798084896 + } + ] + }, + { + "id": "offgrid-ai/juggernaut-xl-v9-GGUF", + "quantLabel": "Full quality", + "name": "Juggernaut XL v9 (photoreal)", + "kind": "image", + "tags": [ + "High quality", + "Photoreal" + ], + "org": "RunDiffusion", + "description": "Versatile photoreal SDXL — cinematic, portraits, landscapes. Off Grid GGUF build of RunDiffusion/Juggernaut-XL-v9.", + "minRamGb": 8, + "quant": "Q8_0", + "releaseDate": "2024-02-18", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "juggernaut-xl-v9-Q8_0.gguf", + "url": "https://huggingface.co/off-grid-ai/juggernaut-xl-v9-GGUF/resolve/main/juggernaut-xl-v9-Q8_0.gguf", + "role": "primary", + "sizeBytes": 4347454816 + } + ] + }, + { + "id": "offgrid-ai/juggernaut-xl-v9-GGUF-Q4", + "quantOf": "offgrid-ai/juggernaut-xl-v9-GGUF", + "quantLabel": "Light", + "name": "Juggernaut XL v9 (Light)", + "kind": "image", + "tags": [ + "Photoreal", + "Light" + ], + "org": "RunDiffusion", + "description": "Versatile photoreal SDXL. Q4 quant: ~35% less memory, small quality trade-off. Runs on a 16GB Mac. Off Grid GGUF build of RunDiffusion/Juggernaut-XL-v9.", + "minRamGb": 8, + "quant": "Q4_K", + "releaseDate": "2024-02-18", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "juggernaut-xl-v9-Q4_K.gguf", + "url": "https://huggingface.co/off-grid-ai/juggernaut-xl-v9-GGUF/resolve/main/juggernaut-xl-v9-Q4_K.gguf", + "role": "primary", + "sizeBytes": 2965023776 + } + ] + }, + { + "id": "offgrid-ai/animagine-xl-4.0-GGUF", + "quantLabel": "Full quality", + "name": "Animagine XL 4.0 (anime)", + "kind": "image", + "tags": [ + "High quality", + "Anime" + ], + "org": "Cagliostro", + "description": "Leading anime SDXL — strong character knowledge. Off Grid GGUF build of cagliostrolab/animagine-xl-4.0.", + "minRamGb": 8, + "quant": "Q8_0", + "releaseDate": "2025-01-10", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "animagine-xl-4.0-Q8_0.gguf", + "url": "https://huggingface.co/off-grid-ai/animagine-xl-4.0-GGUF/resolve/main/animagine-xl-4.0-Q8_0.gguf", + "role": "primary", + "sizeBytes": 4180186816 + } + ] + }, + { + "id": "offgrid-ai/animagine-xl-4.0-GGUF-Q4", + "quantOf": "offgrid-ai/animagine-xl-4.0-GGUF", + "quantLabel": "Light", + "name": "Animagine XL 4.0 (Light)", + "kind": "image", + "tags": [ + "Anime", + "Light" + ], + "org": "Cagliostro", + "description": "Leading anime SDXL — strong character knowledge. Q4 quant: ~35% less memory, small quality trade-off. Runs on a 16GB Mac. Off Grid GGUF build of cagliostrolab/animagine-xl-4.0.", + "minRamGb": 8, + "quant": "Q4_K", + "releaseDate": "2025-01-10", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "animagine-xl-4.0-Q4_K.gguf", + "url": "https://huggingface.co/off-grid-ai/animagine-xl-4.0-GGUF/resolve/main/animagine-xl-4.0-Q4_K.gguf", + "role": "primary", + "sizeBytes": 2797752960 + } + ] + }, + { + "id": "offgrid-ai/illustrious-xl-v2.0-GGUF", + "quantLabel": "Full quality", + "name": "Illustrious XL v2.0 (anime)", + "kind": "image", + "tags": [ + "High quality", + "Anime" + ], + "org": "OnomaAI", + "description": "Top anime / illustration SDXL base. Off Grid GGUF build of OnomaAIResearch/Illustrious-XL-v2.0.", + "minRamGb": 8, + "quant": "Q8_0", + "releaseDate": "2025-04-18", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "illustrious-xl-v2.0-Q8_0.gguf", + "url": "https://huggingface.co/off-grid-ai/illustrious-xl-v2.0-GGUF/resolve/main/illustrious-xl-v2.0-Q8_0.gguf", + "role": "primary", + "sizeBytes": 4180186816 + } + ] + }, + { + "id": "offgrid-ai/illustrious-xl-v2.0-GGUF-Q4", + "quantOf": "offgrid-ai/illustrious-xl-v2.0-GGUF", + "quantLabel": "Light", + "name": "Illustrious XL v2.0 (Light)", + "kind": "image", + "tags": [ + "Anime", + "Light" + ], + "org": "OnomaAI", + "description": "Top anime / illustration SDXL base. Q4 quant: ~35% less memory, small quality trade-off. Runs on a 16GB Mac. Off Grid GGUF build of OnomaAIResearch/Illustrious-XL-v2.0.", + "minRamGb": 8, + "quant": "Q4_K", + "releaseDate": "2025-04-18", + "imageModes": [ + "txt2img", + "img2img" + ], + "files": [ + { + "name": "illustrious-xl-v2.0-Q4_K.gguf", + "url": "https://huggingface.co/off-grid-ai/illustrious-xl-v2.0-GGUF/resolve/main/illustrious-xl-v2.0-Q4_K.gguf", + "role": "primary", + "sizeBytes": 2797752960 + } + ] + }, + { + "id": "off-grid-ai/Wan2.1-T2V-1.3B-GGUF-Q4", + "quantOf": "off-grid-ai/Wan2.1-T2V-1.3B-GGUF", + "quantLabel": "Light", + "sourceModelId": "Wan-AI/Wan2.1-T2V-1.3B", + "name": "Wan 2.1 (1.3B), Light", + "kind": "video", + "org": "Off Grid AI", + "description": "Create short, silent videos from a description. Converted to GGUF by Off Grid AI; the text encoder and VAE are included in the download.", + "minRamGb": 16, + "quant": "Q4_K", + "videoModes": [ + "text-to-video" + ], + "tags": [ + "Text to video", + "Short clips", + "Light" + ], + "files": [ + { + "name": "Wan2.1-T2V-1.3B-calibrated-Q4_K_S.gguf", + "url": "https://huggingface.co/off-grid-ai/Wan2.1-T2V-1.3B-GGUF/resolve/main/Wan2.1-T2V-1.3B-calibrated-Q4_K_S.gguf", + "sizeBytes": 816061024, + "sha256": "82788a4a3ce6e206edd3e8e5fe512b2a75344a3d272d80f07cd4e48b60b7aaf7", + "role": "primary" + }, + { + "name": "wan_2.1_vae.safetensors", + "url": "https://huggingface.co/off-grid-ai/Video-Companions-GGUF/resolve/main/wan_2.1_vae.safetensors", + "sizeBytes": 253815318, + "sha256": "2fc39d31359a4b0a64f55876d8ff7fa8d780956ae2cb13463b0223e15148976b", + "role": "aux" + }, + { + "name": "umt5-xxl-encoder-Q4_K_M.gguf", + "url": "https://huggingface.co/off-grid-ai/Video-Companions-GGUF/resolve/main/umt5-xxl-encoder-Q4_K_M.gguf", + "sizeBytes": 3655145312, + "sha256": "17cf97a5bbbc60a646d6105b832b6f657ce904a8a1ad970e4b59df0c67584a40", + "role": "aux" + } + ] + }, + { + "id": "off-grid-ai/Wan2.2-TI2V-5B-GGUF-Q4", + "quantOf": "off-grid-ai/Wan2.2-TI2V-5B-GGUF", + "quantLabel": "Light", + "sourceModelId": "Wan-AI/Wan2.2-TI2V-5B", + "name": "Wan 2.2 (5B), Light", + "kind": "video", + "org": "Off Grid AI", + "description": "Create short, silent videos with Wan 2.2, from a description or a photo. Converted to GGUF by Off Grid AI; the text encoder and VAE are included in the download.", + "minRamGb": 24, + "quant": "Q4_K", + "videoModes": [ + "text-to-video" + ], + "tags": [ + "Text to video", + "Short clips", + "Light" + ], + "files": [ + { + "name": "Wan2.2-TI2V-5B-Q4_K_S.gguf", + "url": "https://huggingface.co/off-grid-ai/Wan2.2-TI2V-5B-GGUF/resolve/main/Wan2.2-TI2V-5B-Q4_K_S.gguf", + "sizeBytes": 2861621472, + "sha256": "2221abe02e0f9024181aafebd92944cd999518955a0ae7706650fbc0941a14c2", + "role": "primary" + }, + { + "name": "wan2.2_vae.safetensors", + "url": "https://huggingface.co/off-grid-ai/Wan2.2-TI2V-5B-GGUF/resolve/main/wan2.2_vae.safetensors", + "sizeBytes": 1409400960, + "sha256": "e40321bd36b9709991dae2530eb4ac303dd168276980d3e9bc4b6e2b75fed156", + "role": "aux" + }, + { + "name": "umt5-xxl-encoder-Q4_K_M.gguf", + "url": "https://huggingface.co/off-grid-ai/Video-Companions-GGUF/resolve/main/umt5-xxl-encoder-Q4_K_M.gguf", + "sizeBytes": 3655145312, + "sha256": "17cf97a5bbbc60a646d6105b832b6f657ce904a8a1ad970e4b59df0c67584a40", + "role": "aux" + } + ] } ], "text": [ @@ -40,7 +680,16 @@ "isNew": true } ], - "transcription": [], + "transcription": [ + { + "id": "large-v3-turbo-q5_0", + "name": "Large v3 Turbo (Light)", + "size": 574, + "lang": "multi", + "url": "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q5_0.bin", + "description": "Large v3 Turbo at a third of the download, 99 languages" + } + ], "coreml": [], "litert": [] } diff --git a/docs/ANALYTICS.md b/docs/ANALYTICS.md new file mode 100644 index 000000000..cba381e03 --- /dev/null +++ b/docs/ANALYTICS.md @@ -0,0 +1,128 @@ +# Opt-in analytics + +Off Grid AI Mobile (OGAM) and Off Grid AI Desktop (OGAD) send anonymous usage data and crash reports to PostHog Cloud EU, only after the user opts in. This file is the single list of what we send. OGAM and OGAD keep an identical copy; change both together. + +## Rules + +- Off by default. The SDK is not created until the user opts in. Before opt-in the app makes no request to PostHog. +- Opt-out shuts the SDK down, drops anything queued and resets the anonymous ID. +- One random anonymous ID per install, made by the SDK. We never call `identify`, never send an email, license key, account ID or device hardware ID. +- PostHog project: EU Cloud, "discard client IP" on, GeoIP off, no autocapture, no session replay, no feature flags, no remote config, no surveys. +- Debug builds (`__DEV__`, unpackaged Electron) never start the SDK. +- Consent is per device. It never syncs between devices. +- Data retention: 12 months. + +## Never send + +- Prompts, replies, transcripts, reasoning, tool arguments or tool results. +- File names, document names, paths, URLs, domains, hostnames or IP addresses (including remote server and LAN addresses). +- Search terms. +- Names of imported model files, private Hugging Face repos or custom MCP connectors. Send `custom` instead. +- Error messages or stack frames before scrubbing (see Crashes). + +## Properties on every event + +| Property | Values | +|---|---| +| `platform` | `ios`, `android`, `macos`, `windows`, `linux` | +| `app_version` | e.g. `0.0.115` | +| `channel` | `stable`, `nightly` | +| `pro` | `true`, `false` | +| `device_class` | `phone`, `tablet`, `desktop` | +| `ram_gb` | total RAM, rounded to whole GB | +| `os_version` | e.g. `18.1`, `15` | +| `locale` | language only, e.g. `en` | + +## Shared value rules + +- `model`: catalog ID, or public Hugging Face `owner/repo`, or `custom`. Plus `quant` (e.g. `Q4_K_M`) and `size_bucket` (`<1GB`, `1-2GB`, `2-4GB`, `4-8GB`, `>8GB`). +- `engine`: `llama.cpp`, `mlx`, `litert`, `executorch`, `coreml`, `onnx`, `whisper`, `sd.cpp`, `remote`. +- `backend`: `cpu`, `gpu`, `npu`, `remote`. +- `error_code`: a fixed code from our own error classes, never a raw message. +- Durations in milliseconds, rounded. Rates rounded to one decimal. + +## Events + +### App and screens + +| Event | Properties | +|---|---| +| `app_opened` | `cold_start_ms` | +| `app_backgrounded` | `session_ms` | +| `$screen` (OGAM, SDK `screen()`) / `screen_viewed` (OGAD) | `screen`, `prev_screen`, `prev_ms` | + +### Consent + +| Event | Properties | +|---|---| +| `analytics_opted_in` | `source`: `setup`, `settings`, `failure_prompt`, `first_chat_prompt` | + +### Models + +| Event | Properties | +|---|---| +| `model_download_started` | `model`, `quant`, `size_bucket`, `network` (`wifi`, `cellular`, `ethernet`, `unknown`), `free_storage_gb` | +| `model_download_finished` | same + `duration_ms`, `mb_per_sec` | +| `model_download_failed` | same + `error_code` | +| `model_loaded` | `model`, `quant`, `engine`, `backend`, `soc`, `free_ram_gb`, `load_ms` | +| `model_load_failed` | same + `error_code` | + +A video model pack reports downloads once for the whole pack: `model` is its catalog ID and `quant` is empty. A video model loads inside each generation: OGAD sends `model_loaded` / `model_load_failed` for each clip's engine start; OGAM sends none. + +### Generation + +| Event | Properties | +|---|---| +| `generation_finished` | `modality` (`text`, `vision`, `image`, `video`, `stt`, `tts`, `embedding`), `model`, `engine`, `backend`, `outcome` (`done`, `stopped`, `failed`), `ttft_ms`, `tokens_per_sec`, `context_tokens_bucket` (`<2k`, `2-8k`, `8-32k`, `>32k`), `battery_pct` (rounded to 10), `charging`, `error_code`, `source` (`app`, `api`, `extension`; OGAD only). Video sends `duration_ms` (the whole generation) instead of `ttft_ms`, `tokens_per_sec` and `context_tokens_bucket`, plus `has_init_image` (a start image or a continued shot), `story` (part of a film) and `quality` (`draft`, `final`) for films. | + +### Actions + +| Event | Properties | +|---|---| +| `chat_created` | none | +| `message_sent` | `has_image`, `has_document`, `tools_enabled` | +| `reply_regenerated` | none | +| `project_created` | none | +| `document_added` | `file_type` (an extension the app opens, else `other`; `none` without one), `size_bucket` | +| `feature_used` | `feature`: `voice_input`, `tts`, `image_generation`, `video_generation`, `vision`, `web_search`, `tools`, `mcp_tool`, `remote_server`, `browser_action`, `clipboard`, `meetings`, `sync` | +| `features_snapshot` | once a day: one `true`/`false` property per feature that is switched on. `video_generation` is true when a local video model is active or a remote video model is picked. | +| `problem_reported` | none | + +### Flows (funnels) + +All multi-step journeys use three events with a `flow` property, so every funnel is built the same way in PostHog. + +| Event | Properties | +|---|---| +| `flow_started` | `flow` | +| `flow_step` | `flow`, `step` | +| `flow_finished` | `flow`, `outcome` (`done`, `failed`, `cancelled`), `duration_ms`, `failed_step`, `error_code` | + +| `flow` | Steps | +|---|---| +| `onboarding` | `setup_shown`, `download_started`, `download_done`, `model_loaded`, `first_message` | +| `model_transfer` | `opened`, `device_picked`, `model_picked`, `started` | +| `sync_pairing` | `opened`, `code_shown`, `code_entered`, `paired` | +| `remote_server` | `opened`, `address_entered`, `tested`, `connected` | +| `voice_setup` | `opened`, `model_picked`, `downloaded` | +| `image_setup` | `opened`, `model_picked`, `downloaded` | +| `video_setup` | `opened`, `model_picked`, `downloaded` | +| `video_story` | `storyboard_shown`, `draft_started`, `draft_finished`, `final_started` (`detail`: `now`, `tonight`), `final_finished`, `exported`. A film made on another device reports on the device that rendered it. Never the sentence, shot text or narration. | +| `connector_add` | `opened`, `connector_picked` (`catalog_id` or `custom`), `authorised` | +| `pro_upgrade` | `gate_shown` (`feature`), `paywall_viewed`, `trial_started`, `purchased` | + +### Crashes + +| Event | Properties | +|---|---| +| `$exception` | SDK exception capture. Scrub message and frames before send: replace paths with the file name only, drop any quoted text, keep type, file name, line, function. A file name is kept only for the app's own code (e.g. `index.js`); a person's file, the home folder and the account name become `` / ``. | +| `unclean_exit` | Sent on the next launch when the app died in the foreground during a model load or a generation. `stage` (`load`, `generate`; `generate` includes a video generation), `model`, `engine`, `backend`, `free_ram_gb`, `elapsed_ms`. Not sent after a background kill, an app update or a reboot. | +| `llama_server_crashed` (OGAD) | `model`, `engine`, `backend`, `exit_code`, `classified` | + +## Report a problem + +- OGAM: shows the debug log, then opens the share sheet addressed to support. iOS attaches the log file; Android puts the log's latest part in the message text. +- OGAD: shows the debug log with Copy, then opens a mail draft to the support address; the user pastes the log into it (a `mailto:` link cannot attach a file). +- The debug log never holds chat content: no log line may include prompts, replies, transcripts or tool output. +- The user sees the log before it is sent. +- It works whether or not analytics is on. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6a3022a74..417213e10 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -922,7 +922,7 @@ export const SPACING = { xs: 4, sm: 8, md: 12, lg: 16, xl: 24, xxl: 32 }; - Generate images with seed control for reproducibility - Save to device gallery or share directly -**Privacy:** Zero network activity after model download. Prompts never leave device. +**Privacy:** Prompts and images never leave the device. Opt-in analytics (see ANALYTICS.md) sends anonymous usage data only if the person turns it on. ### 3. Document Analysis with Vision @@ -1124,8 +1124,12 @@ All data stored in app's private storage, inaccessible to other apps (Android sa - Hugging Face CDN (model file downloads) - xororz HuggingFace repos (image model listings) +**Opt-in analytics (off by default):** +- PostHog Cloud EU (`eu.i.posthog.com`), only after the person turns on Usage data or Crash reports +- Anonymous events and scrubbed crash reports; never chats, files, prompts or replies (see `docs/ANALYTICS.md`) + **After model download:** -- Zero network activity +- No other network activity unless you turn on a feature that uses it (analytics, a remote server, web search, Sync) - Enable airplane mode and use indefinitely - All inference happens on-device diff --git a/docs/VIDEO_GENERATION.md b/docs/VIDEO_GENERATION.md new file mode 100644 index 000000000..32fef9bfe --- /dev/null +++ b/docs/VIDEO_GENERATION.md @@ -0,0 +1,87 @@ +# Video generation + +## Current engine + +Local generation uses stable-diffusion.cpp at the revision in +`scripts/video/revision`. Models uses the shared Hugging Face search, catalog, +and architecture-specific pack resolver. The catalog currently includes Wan 2.1, +Wan 2.2, LTX-2.3, LTX-2.5, and HunyuanVideo 1.5. Each pack must include every +required weight, encoder, VAE, and connector file. Catalog availability is not +proof that a model fits or has completed generation on a particular phone. + +Video request validation and settings keys live in `@offgrid/models`. Mobile's +video generation service owns local and OGAD jobs, progress, cancellation, +recovery, and completed chat/gallery records. The native bridges share the C++ +runtime lifecycle. Pro owns sync and model transfer through existing extension +points. + +## Build setup + +CocoaPods builds the pinned iOS framework during `pod install`. Gradle prepares the Android runtime before CMake configuration. Both paths reuse their build cache. For explicit setup, run from the mobile repository after installing JavaScript dependencies: + +```sh +bash scripts/prepare-video-runtime.sh +OFFGRID_IOS_DEVICE_ONLY=1 bash scripts/build-video-ios.sh +(cd ios && pod install) +``` + +The command above builds the physical arm64 iPhone framework only. Pass +`OFFGRID_IOS_DEVICE_ONLY=1` to the physical Xcode build as well. Building requires +an Xcode SDK with the iOS 26 background task APIs. Deployment remains iOS 17. + +For Android, install CMake, Make, Python 3, and the project's Android SDK/NDK: + +```sh +bash scripts/prepare-video-runtime.sh --android +(cd android && ./gradlew assembleDebug) +``` + +Android preparation builds a pinned host shader compiler. The NDK compiler is +not sufficient for this engine's Vulkan shaders. Generated sources, libraries, +and compiler tools stay in `.video-build` and are not committed. The small +Vulkan patch uses the engine's dynamic dispatcher so linking remains compatible +with the app's Android API 24 minimum. + +## Device limits and recovery + +Model download does not guarantee that a device has enough memory to generate. +The residency manager checks the pack and clip size before loading. Default +clips are 320 × 192, 17 frames, at 8 FPS. Speed and peak memory still need manual +checks on physical iOS and Android devices. + +On iOS 26, continued background GPU work requires OS support and admission. +If admission is not available, generation stops when the app enters the +background. Android uses a foreground service with a Stop action; an OS timeout +also cancels the job. Partial output is removed on failure or cancellation. + +After restart, interrupted jobs appear with Retry. Local Retry restarts the +clip. Remote Retry uses the same OGAD job ID, so a lost connection does not start +a second job. Settings and model selection can change while a saved request +retains its resolved generation parameters. + +Only OGAD remote video servers are supported. REST and MCP share OGAD's video +job owner. Model transfer sends and checks every file in the selected pack +before registration. Cancel and retry use the shared package transaction and +rollback. Local download resume preserves complete files; an incomplete file can +restart after process death. + +## Integration audit (2026-09-29) + +- Storage reads actual sizes for all installed video pack files, including old + records whose catalog size is missing. Download completion stores actual sizes. +- Auto Configure includes video only when the native runtime, complete catalog + pack, RAM minimum, and existing memory budget allow it. Models below these + limits remain manually selectable; automatic setup does not use Run Anyway. +- iOS excludes downloaded/transferred video model folders from device backups, + as it does for text and image models. SD image packs use `image_models` and its + existing exclusion. Android disables app backup in its manifest. Generated + videos and user content are not excluded by this model-cache policy. +- Remote video requires OGAD. Local and remote labels use the shared catalog, + while model IDs and per-model settings keys remain unchanged. +- Native progress changes and background admission details enter the app debug + log. A heartbeat alone is not proof that a sampling step advanced. + +Source checks cover these integrations. Device checks for the latest backup +exclusion and Auto Configure changes remain pending. Complete video quality and +all-model coverage are not established by successful builds or partial sampling. +No automated tests were added or run for this audit. diff --git a/docs/brand_tone_voice.md b/docs/brand_tone_voice.md index 04d6faf96..3c30d24a4 100644 --- a/docs/brand_tone_voice.md +++ b/docs/brand_tone_voice.md @@ -24,7 +24,7 @@ That's the energy in every word Off Grid writes. ### 1. Generous, not transactional -Off Grid asks nothing from you. No account. No API key. No data. The voice reflects this -- it gives without expecting anything back. It doesn't ask you to share, subscribe, follow, or review. It just hands you the thing. +Off Grid asks nothing from you. No account. No API key. Your chats and files never leave your device; anonymous usage data and crash reports are sent only if you turn them on. The voice reflects this -- it gives without expecting anything back. It doesn't ask you to share, subscribe, follow, or review. It just hands you the thing. This means copy never implies a trade. No "get access to," no "unlock," no implied paywall. The frame is always: here's what you have. @@ -41,7 +41,7 @@ Specificity is how you earn trust without asking for it. ``` "Phi-3 Mini runs at ~28 tok/s on a Snapdragon 8 Gen 3. Llama 3.1 8B runs at ~12 tok/s on the same chip." -"On-device. Offline. Zero data leaves your phone. Not a marketing claim -- airplane mode works." +"On-device. Offline. Your chats never leave your phone. Not a marketing claim -- airplane mode works." ``` ``` @@ -111,7 +111,7 @@ Every piece of Off Grid content follows the same arc: **Recognition -> Return -> Freedom** 1. **Recognition**: Name what's been happening. "Every query you've sent to a cloud AI has been logged, stored, and used." -2. **Return**: Show what's being given back. "With Off Grid, the model runs on your phone. Nothing leaves it." +2. **Return**: Show what's being given back. "With Off Grid, the model runs on your phone. Your chats never leave it." 3. **Freedom**: Hand them the capability without condition. "Go offline. It still works. It's yours." This isn't about doubt and resolution like a pricing tool. It's about something that was taken being handed back. The tone is quiet, clear, and generous -- not triumphant or righteous. @@ -129,7 +129,7 @@ Short sentences. State what it does, then prove it. The reader should feel like ``` "Chat. Generate images. Use tools. See. Listen. -All on your phone. All offline. Zero data leaves your device. +All on your phone. All offline. Your chats and files never leave your device. No account. No API key. No subscription. Download and go." ``` @@ -368,7 +368,7 @@ Load failed: "Load failed -- the file may be corrupted. Delete it and re-downloa Primary: **"Private AI. No cloud. No compromise."** Alternatives: -- "Run AI on your phone. Nothing leaves it." +- "Run AI on your phone. Your chats never leave it." - "The Swiss Army Knife of On-Device AI." - "AI that stays on your device." - "No cloud. No account. No compromise." @@ -382,6 +382,7 @@ Before publishing any Off Grid content: - [ ] Does it feel like something being given, not sold? - [ ] Are privacy claims stated as mechanisms ("the model runs in RAM"), not promises ("we value your privacy")? +- [ ] Do privacy claims stay true with opt-in analytics on? Say "your chats and files never leave your device", never "nothing leaves" or "no telemetry". - [ ] Does every performance claim include a number and a device? - [ ] Are device requirements stated before steps? - [ ] Does it follow the recognition -> return -> freedom arc? diff --git a/docs/design/DESIGN_PHILOSOPHY_SYSTEM.md b/docs/design/DESIGN_PHILOSOPHY_SYSTEM.md index acfe8235b..c71a4b656 100644 --- a/docs/design/DESIGN_PHILOSOPHY_SYSTEM.md +++ b/docs/design/DESIGN_PHILOSOPHY_SYSTEM.md @@ -2,7 +2,32 @@ ## Core Philosophy -OffgridMobile follows a **brutalist, minimal design system** inspired by terminal aesthetics and focused on functionality over decoration. The interface emphasizes clarity, efficiency, and respect for the user's attention. +OffgridMobile follows a **brutalist, minimal design system** inspired by terminal aesthetics. Visual quality and function are both essential. The interface must be clear, consistent, easy to use, and responsive. + +### Product priority: UI and UX are P0 + +Styling, UI/UX, consistency, ease of use, finesse, animation, and interactions are **P0 requirements**. Do not defer known problems in these areas as optional polish. A feature must work well and feel complete in use. + +- Reuse existing components, theme tokens, and interaction patterns across screens. +- Use restrained depth and theme-aware shadows to separate cards and show hierarchy. Minimal does not mean flat or unfinished. +- Follow the recent chat and remote-server card improvements. Mobile commits `93e90268` (chat card depth), `f87ae730` (remote-server card depth), and `4871a9ad` (enhanced prompt card styling) provide implementation references. +- Make taps, selection, expansion, collapse, loading, cancellation, and recovery clear and predictable. Show the actual state of the operation. +- Use smooth, purposeful animation and immediate interaction feedback. Respect reduced-motion settings and device performance. +- Keep common actions easy to find. Keep rows compact and easy to scan. Hide technical file details until the user needs them. +- Check the result in the real app on each affected platform. A source change alone does not establish visual or interaction quality. + +These rules apply to new features and changes to existing screens. Fix concrete feedback within the affected area; this is not a request for unrelated redesigns. + +### Mobile design decisions + +This document is the source of truth for Off Grid AI Mobile layout and interaction decisions. The shared brand philosophy supplies the common visual identity; use this document for the mobile details. + +- Use less visible text. Keep the words a person needs to choose or act. Put longer explanations in details, confirmation, or recovery states. +- Use a clear icon for a familiar action when it saves space. Give every icon action an accessible label and a touch target of at least 44px. Keep visible text when the icon could be misunderstood. +- Keep model rows lean and compact. Use the shared `ModelCard` for text, image, video, embedding, and speech models. Highlight the active model in the same list; do not make a separate active-model list. Put the main action in the row; do not add a large button below every card. +- Use the screen area well. Align headings, inputs, cards, and actions to the same content edge. Avoid repeated copy and empty space that makes a list harder to scan. +- Use standard `SPACING` tokens for gaps and padding. Keep the same spacing between equivalent parts across screens. +- Give cards useful depth with restrained, theme-aware shadows. Use the same depth pattern across related screens; keep borders crisp and avoid heavy elevation. --- @@ -273,12 +298,12 @@ header: { ## Anti-Patterns (What to Avoid) -❌ **Colorful gradients or heavy shadows** — Keep it flat; use theme shadows sparingly +❌ **Colorful gradients or heavy shadows** — Use restrained, theme-aware depth that matches existing cards ❌ **Multiple accent colors** — Emerald only ❌ **Rounded pill shapes** — Use minimal `8px` radius ❌ **Decorative animations** — Only functional animations (loading, state transitions) ❌ **Mixed font families** — Menlo only -❌ **Heavy borders or 3D effects** — Flat, sharp, minimal +❌ **Heavy borders or 3D effects** — Keep edges crisp and elevation restrained ❌ **Large empty spaces without purpose** — Dense when appropriate ❌ **Color-coded information** — Use hierarchy, not color ❌ **Cluttered layouts** — Remove before adding @@ -357,7 +382,7 @@ When building or refactoring a component: - Monospace font throughout - Uppercase label for section ("DOWNLOADED MODELS") - Metadata as small, muted text -- Subtle border, no shadow +- Subtle border and theme-aware shadow, consistent with the shared card pattern - Compact mode: horizontal layout with icon action buttons (download, select, delete) on right - Author shown as small pill-style tag, not a full line @@ -370,7 +395,7 @@ When building or refactoring a component: - Mixed fonts for metadata **After:** -- Flat background (`COLORS.surface`) +- Theme surface background with the established card depth and shadow - No avatars (text-first) - Compact padding (`SPACING.md`) - Sharp edges @@ -391,7 +416,7 @@ This design system was intentionally created to: 6. **Age gracefully** — minimalism doesn't go out of style As the app evolves: -- **Stay true to brutalism** — add functionality, not decoration +- **Keep the minimal aesthetic** — preserve visual quality, useful depth, and clear interactions - **Maintain consistency** — use tokens religiously - **Question additions** — does this serve the user or just look nice? - **Optimize for density** — information should be easy to scan @@ -408,4 +433,4 @@ As the app evolves: --- -**Remember:** Silence, clarity, and function over form. Let the AI's capabilities speak, not the interface. +**Remember:** Clarity, visual quality, and ease of use are part of the feature. Minimal design must still feel complete. diff --git a/docs/plans/offline-recordings.md b/docs/plans/offline-recordings.md index 4b283c0f2..289235b1a 100644 --- a/docs/plans/offline-recordings.md +++ b/docs/plans/offline-recordings.md @@ -8,7 +8,7 @@ Tier: Pro-gated A Pro surface in the mobile app to record audio on-device, transcribe it locally with Whisper, and generate an LLM title, summary, and action items. Recording works -in the foreground and in the background (lock screen). Nothing leaves the phone. +in the foreground and in the background (lock screen). Recordings and transcripts never leave the phone. This is mobile's native analog of the desktop meeting recorder. The desktop recorder relies on macOS ScreenCaptureKit + system-audio loopback, which iOS and Android do not diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-about-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-about-ios.png new file mode 100644 index 000000000..7506b6d2e Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-about-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-auto-setup-android.png b/docs/pr-screenshots/feature/opt-in-analytics/after-auto-setup-android.png new file mode 100644 index 000000000..c85138497 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-auto-setup-android.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-auto-setup-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-auto-setup-ios.png new file mode 100644 index 000000000..47e11b443 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-auto-setup-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-first-chat-prompt-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-first-chat-prompt-ios.png new file mode 100644 index 000000000..7e01b34fa Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-first-chat-prompt-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-home-desktop-card-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-home-desktop-card-ios.png new file mode 100644 index 000000000..89b7bc10b Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-home-desktop-card-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-pro-detail-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-pro-detail-ios.png new file mode 100644 index 000000000..d639eccce Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-pro-detail-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-android.png b/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-android.png new file mode 100644 index 000000000..7c91e2476 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-android.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-ios.png new file mode 100644 index 000000000..3436bfcef Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-share-android.png b/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-share-android.png new file mode 100644 index 000000000..c9081c937 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-share-android.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-share-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-share-ios.png new file mode 100644 index 000000000..d2001880c Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-report-problem-share-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-android.png b/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-android.png new file mode 100644 index 000000000..1bb15b4d4 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-android.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-ios.png new file mode 100644 index 000000000..2603bf80b Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-on-android.png b/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-on-android.png new file mode 100644 index 000000000..d1f7938f9 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-on-android.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-on-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-on-ios.png new file mode 100644 index 000000000..7a563a7b2 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-settings-analytics-on-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-share-sheet-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-share-sheet-ios.png new file mode 100644 index 000000000..8325252b4 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-share-sheet-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/after-what-we-share-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/after-what-we-share-ios.png new file mode 100644 index 000000000..7bcad2e87 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/after-what-we-share-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/before-about-android.png b/docs/pr-screenshots/feature/opt-in-analytics/before-about-android.png new file mode 100644 index 000000000..fec73356d Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/before-about-android.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/before-about-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/before-about-ios.png new file mode 100644 index 000000000..6c9a09f35 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/before-about-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/before-auto-setup-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/before-auto-setup-ios.png new file mode 100644 index 000000000..12d501691 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/before-auto-setup-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/before-home-desktop-card-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/before-home-desktop-card-ios.png new file mode 100644 index 000000000..d3011119b Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/before-home-desktop-card-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/before-pro-detail-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/before-pro-detail-ios.png new file mode 100644 index 000000000..fcf9707ef Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/before-pro-detail-ios.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/before-pro-voice-models-android.png b/docs/pr-screenshots/feature/opt-in-analytics/before-pro-voice-models-android.png new file mode 100644 index 000000000..bee94abb9 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/before-pro-voice-models-android.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/before-settings-privacy-android.png b/docs/pr-screenshots/feature/opt-in-analytics/before-settings-privacy-android.png new file mode 100644 index 000000000..c43327009 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/before-settings-privacy-android.png differ diff --git a/docs/pr-screenshots/feature/opt-in-analytics/before-settings-privacy-ios.png b/docs/pr-screenshots/feature/opt-in-analytics/before-settings-privacy-ios.png new file mode 100644 index 000000000..d83b73ac0 Binary files /dev/null and b/docs/pr-screenshots/feature/opt-in-analytics/before-settings-privacy-ios.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/android/image-sdcpp-realvisxl-light-cpu-26min-result.png b/docs/pr-screenshots/release-analytics-video/acceptance/android/image-sdcpp-realvisxl-light-cpu-26min-result.png new file mode 100644 index 000000000..85593e2e0 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/android/image-sdcpp-realvisxl-light-cpu-26min-result.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-coreml-01-working-then-app-killed.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-coreml-01-working-then-app-killed.png new file mode 100644 index 000000000..f27fba96d Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-coreml-01-working-then-app-killed.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-coreml-02-sd21-45s-generated.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-coreml-02-sd21-45s-generated.png new file mode 100644 index 000000000..555d71556 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-coreml-02-sd21-45s-generated.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-coreml-03-sd21-full-size.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-coreml-03-sd21-full-size.png new file mode 100644 index 000000000..fdc9ce87f Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-coreml-03-sd21-full-size.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-memory-guard-01-text-load-refused-not-killed.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-memory-guard-01-text-load-refused-not-killed.png new file mode 100644 index 000000000..6e0cf6677 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-memory-guard-01-text-load-refused-not-killed.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-01-not-enough-memory.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-01-not-enough-memory.png new file mode 100644 index 000000000..4f1f461d9 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-01-not-enough-memory.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-02-auto-mode-loads-text-model.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-02-auto-mode-loads-text-model.png new file mode 100644 index 000000000..691237850 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-02-auto-mode-loads-text-model.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-03-auto-mode-text-answer-no-image.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-03-auto-mode-text-answer-no-image.png new file mode 100644 index 000000000..aeac12ef4 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-03-auto-mode-text-answer-no-image.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-04-realvisxl-31s-generated.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-04-realvisxl-31s-generated.png new file mode 100644 index 000000000..458506281 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/image-sdcpp-04-realvisxl-31s-generated.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/remote-01-saved-desktop-address-stale.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/remote-01-saved-desktop-address-stale.png new file mode 100644 index 000000000..08b1fb39e Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/remote-01-saved-desktop-address-stale.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/remote-02-mac-reachable-401-needs-key-scan-finds-none.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/remote-02-mac-reachable-401-needs-key-scan-finds-none.png new file mode 100644 index 000000000..bfbd4a1a5 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/remote-02-mac-reachable-401-needs-key-scan-finds-none.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/remote-03-after-pairing-still-401.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/remote-03-after-pairing-still-401.png new file mode 100644 index 000000000..be504c9a0 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/remote-03-after-pairing-still-401.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/search-ondevice-01-snow-top-hits.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/search-ondevice-01-snow-top-hits.png new file mode 100644 index 000000000..7fc649d83 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/search-ondevice-01-snow-top-hits.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/video-ondevice-01-chip-disabled-no-action.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/video-ondevice-01-chip-disabled-no-action.png new file mode 100644 index 000000000..e1107e14d Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/video-ondevice-01-chip-disabled-no-action.png differ diff --git a/docs/pr-screenshots/release-analytics-video/acceptance/ios/video-ondevice-02-no-video-model-sheet.png b/docs/pr-screenshots/release-analytics-video/acceptance/ios/video-ondevice-02-no-video-model-sheet.png new file mode 100644 index 000000000..1a696d910 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/acceptance/ios/video-ondevice-02-no-video-model-sheet.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/animate/01-animate-muted.png b/docs/pr-screenshots/release-analytics-video/android/animate/01-animate-muted.png new file mode 100644 index 000000000..1b82c317d Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/animate/01-animate-muted.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/animate/02-animate-sheet-resume.png b/docs/pr-screenshots/release-analytics-video/android/animate/02-animate-sheet-resume.png new file mode 100644 index 000000000..cfbea1c96 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/animate/02-animate-sheet-resume.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/clip-card-and-chips/01-clip-card-with-chips-continue-muted.png b/docs/pr-screenshots/release-analytics-video/android/clip-card-and-chips/01-clip-card-with-chips-continue-muted.png new file mode 100644 index 000000000..1c8bd6314 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/clip-card-and-chips/01-clip-card-with-chips-continue-muted.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/home/01-home.png b/docs/pr-screenshots/release-analytics-video/android/home/01-home.png new file mode 100644 index 000000000..7b6346218 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/home/01-home.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/image-mnn/01-not-enough-memory.png b/docs/pr-screenshots/release-analytics-video/android/image-mnn/01-not-enough-memory.png new file mode 100644 index 000000000..d44daa5c5 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/image-mnn/01-not-enough-memory.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/image-mnn/02-result-after-run-anyway.png b/docs/pr-screenshots/release-analytics-video/android/image-mnn/02-result-after-run-anyway.png new file mode 100644 index 000000000..f5bd5479f Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/image-mnn/02-result-after-run-anyway.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/01-image-tab-recommendations.png b/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/01-image-tab-recommendations.png new file mode 100644 index 000000000..7132f5756 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/01-image-tab-recommendations.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/02-download-progress.png b/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/02-download-progress.png new file mode 100644 index 000000000..f9b715f2f Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/02-download-progress.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/03-not-enough-memory.png b/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/03-not-enough-memory.png new file mode 100644 index 000000000..22dceca9f Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/03-not-enough-memory.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/04-no-image-htp-vulkan.png b/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/04-no-image-htp-vulkan.png new file mode 100644 index 000000000..ea2825b1b Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/image-sdcpp/04-no-image-htp-vulkan.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/library/01-your-videos.png b/docs/pr-screenshots/release-analytics-video/android/library/01-your-videos.png new file mode 100644 index 000000000..2a897fc77 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/library/01-your-videos.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/making-it/01-could-not-finish.png b/docs/pr-screenshots/release-analytics-video/android/making-it/01-could-not-finish.png new file mode 100644 index 000000000..13d6dfa99 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/making-it/01-could-not-finish.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/models-picks/01-fast-balanced-best-for-this-phone.png b/docs/pr-screenshots/release-analytics-video/android/models-picks/01-fast-balanced-best-for-this-phone.png new file mode 100644 index 000000000..d765c9d3b Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/models-picks/01-fast-balanced-best-for-this-phone.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/off-grid-ai-mark/01-video-models-off-grid-builds-marked.png b/docs/pr-screenshots/release-analytics-video/android/off-grid-ai-mark/01-video-models-off-grid-builds-marked.png new file mode 100644 index 000000000..4ff3b2883 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/off-grid-ai-mark/01-video-models-off-grid-builds-marked.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/on-phone-video/01-memory-guard-wan22-light.png b/docs/pr-screenshots/release-analytics-video/android/on-phone-video/01-memory-guard-wan22-light.png new file mode 100644 index 000000000..d1f4afcb2 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/on-phone-video/01-memory-guard-wan22-light.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/setup/01-lean-plan.png b/docs/pr-screenshots/release-analytics-video/android/setup/01-lean-plan.png new file mode 100644 index 000000000..6254f7782 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/setup/01-lean-plan.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/setup/02-balanced-plan.png b/docs/pr-screenshots/release-analytics-video/android/setup/02-balanced-plan.png new file mode 100644 index 000000000..a3abab2f8 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/setup/02-balanced-plan.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/setup/03-extreme-plan.png b/docs/pr-screenshots/release-analytics-video/android/setup/03-extreme-plan.png new file mode 100644 index 000000000..9f672ba09 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/setup/03-extreme-plan.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/start-image-loop/01-follows-the-mac-download.png b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/01-follows-the-mac-download.png new file mode 100644 index 000000000..9eca4f48f Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/01-follows-the-mac-download.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/start-image-loop/02-light-offer-34gb-mac.png b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/02-light-offer-34gb-mac.png new file mode 100644 index 000000000..be983c5e5 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/02-light-offer-34gb-mac.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/start-image-loop/03-full-quality-offer-64gb-mac.png b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/03-full-quality-offer-64gb-mac.png new file mode 100644 index 000000000..54442480c Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/03-full-quality-offer-64gb-mac.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/start-image-loop/04-too-small-16gb-mac.png b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/04-too-small-16gb-mac.png new file mode 100644 index 000000000..740db5f25 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/04-too-small-16gb-mac.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/start-image-loop/05-older-desktop-light-offer.png b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/05-older-desktop-light-offer.png new file mode 100644 index 000000000..022d90ae0 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/05-older-desktop-light-offer.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/start-image-loop/06-installed-on-mac-use-it.png b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/06-installed-on-mac-use-it.png new file mode 100644 index 000000000..e86f3897f Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/06-installed-on-mac-use-it.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/start-image-loop/07-storyboard-start-from-a-photo.png b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/07-storyboard-start-from-a-photo.png new file mode 100644 index 000000000..69489c8a1 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/start-image-loop/07-storyboard-start-from-a-photo.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/storyboard/01-storyboard-shots.png b/docs/pr-screenshots/release-analytics-video/android/storyboard/01-storyboard-shots.png new file mode 100644 index 000000000..c72dff0cb Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/storyboard/01-storyboard-shots.png differ diff --git a/docs/pr-screenshots/release-analytics-video/android/viewer/01-viewer-continue-muted.png b/docs/pr-screenshots/release-analytics-video/android/viewer/01-viewer-continue-muted.png new file mode 100644 index 000000000..3d45305fe Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/android/viewer/01-viewer-continue-muted.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/animate/01-animate-muted.png b/docs/pr-screenshots/release-analytics-video/ios/animate/01-animate-muted.png new file mode 100644 index 000000000..a44345c63 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/animate/01-animate-muted.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/animate/02-animate-sheet.png b/docs/pr-screenshots/release-analytics-video/ios/animate/02-animate-sheet.png new file mode 100644 index 000000000..ce49c01b9 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/animate/02-animate-sheet.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/clip-card-and-chips/01-clip-with-chips.png b/docs/pr-screenshots/release-analytics-video/ios/clip-card-and-chips/01-clip-with-chips.png new file mode 100644 index 000000000..44eda3d52 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/clip-card-and-chips/01-clip-with-chips.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/ghost-rows/01-desktop-tools-one-row-ogad-mac-debug.png b/docs/pr-screenshots/release-analytics-video/ios/ghost-rows/01-desktop-tools-one-row-ogad-mac-debug.png new file mode 100644 index 000000000..cba34838a Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/ghost-rows/01-desktop-tools-one-row-ogad-mac-debug.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/ghost-rows/02-sync-screen-same-one-computer.png b/docs/pr-screenshots/release-analytics-video/ios/ghost-rows/02-sync-screen-same-one-computer.png new file mode 100644 index 000000000..4e5add827 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/ghost-rows/02-sync-screen-same-one-computer.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/image-coreml/01-generating.png b/docs/pr-screenshots/release-analytics-video/ios/image-coreml/01-generating.png new file mode 100644 index 000000000..0a32710ea Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/image-coreml/01-generating.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/image-coreml/02-result-in-chat.png b/docs/pr-screenshots/release-analytics-video/ios/image-coreml/02-result-in-chat.png new file mode 100644 index 000000000..8ef2ba2c4 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/image-coreml/02-result-in-chat.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/01-image-tab-recommendations.png b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/01-image-tab-recommendations.png new file mode 100644 index 000000000..95bac7e5b Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/01-image-tab-recommendations.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/02-download-progress.png b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/02-download-progress.png new file mode 100644 index 000000000..004584ee3 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/02-download-progress.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/03-loading-model.png b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/03-loading-model.png new file mode 100644 index 000000000..a10ee1248 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/03-loading-model.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/04-generating-steps.png b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/04-generating-steps.png new file mode 100644 index 000000000..da361a185 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/04-generating-steps.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/05-result-in-chat.png b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/05-result-in-chat.png new file mode 100644 index 000000000..6af870364 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/05-result-in-chat.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/06-home-image-model-sheet.png b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/06-home-image-model-sheet.png new file mode 100644 index 000000000..b8e9051d4 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/image-sdcpp/06-home-image-model-sheet.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/library/01-your-videos.png b/docs/pr-screenshots/release-analytics-video/ios/library/01-your-videos.png new file mode 100644 index 000000000..00f788595 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/library/01-your-videos.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/models-picks/01-text-picks-for-this-phone.png b/docs/pr-screenshots/release-analytics-video/ios/models-picks/01-text-picks-for-this-phone.png new file mode 100644 index 000000000..a6b8baec0 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/models-picks/01-text-picks-for-this-phone.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/off-grid-ai-mark/01-models-video-mark.png b/docs/pr-screenshots/release-analytics-video/ios/off-grid-ai-mark/01-models-video-mark.png new file mode 100644 index 000000000..d11722e7d Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/off-grid-ai-mark/01-models-video-mark.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/remote-servers/01-remote-servers.png b/docs/pr-screenshots/release-analytics-video/ios/remote-servers/01-remote-servers.png new file mode 100644 index 000000000..742119502 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/remote-servers/01-remote-servers.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/setup/01-lean-plan.png b/docs/pr-screenshots/release-analytics-video/ios/setup/01-lean-plan.png new file mode 100644 index 000000000..2a1677545 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/setup/01-lean-plan.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/setup/02-balanced-plan.png b/docs/pr-screenshots/release-analytics-video/ios/setup/02-balanced-plan.png new file mode 100644 index 000000000..b04da050a Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/setup/02-balanced-plan.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/setup/03-extreme-plan.png b/docs/pr-screenshots/release-analytics-video/ios/setup/03-extreme-plan.png new file mode 100644 index 000000000..4829608e8 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/setup/03-extreme-plan.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/start-image-loop/01-download-light.png b/docs/pr-screenshots/release-analytics-video/ios/start-image-loop/01-download-light.png new file mode 100644 index 000000000..8e89d62d8 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/start-image-loop/01-download-light.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/storyboard/01-storyboard.png b/docs/pr-screenshots/release-analytics-video/ios/storyboard/01-storyboard.png new file mode 100644 index 000000000..9f5246ab5 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/storyboard/01-storyboard.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/storyboard/02-start-from-a-photo-sheet.png b/docs/pr-screenshots/release-analytics-video/ios/storyboard/02-start-from-a-photo-sheet.png new file mode 100644 index 000000000..2fa4638c4 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/storyboard/02-start-from-a-photo-sheet.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/viewer/01-viewer.png b/docs/pr-screenshots/release-analytics-video/ios/viewer/01-viewer.png new file mode 100644 index 000000000..d1595f53c Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/viewer/01-viewer.png differ diff --git a/docs/pr-screenshots/release-analytics-video/ios/viewer/02-viewer-continue-sheet.png b/docs/pr-screenshots/release-analytics-video/ios/viewer/02-viewer-continue-sheet.png new file mode 100644 index 000000000..bc6e064a3 Binary files /dev/null and b/docs/pr-screenshots/release-analytics-video/ios/viewer/02-viewer-continue-sheet.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-analytics-crash-on-after-restart-ios.png b/docs/pr-screenshots/release/analytics-video/after-analytics-crash-on-after-restart-ios.png new file mode 100644 index 000000000..3151e6e3f Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-analytics-crash-on-after-restart-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-analytics-off-after-restart-ios.png b/docs/pr-screenshots/release/analytics-video/after-analytics-off-after-restart-ios.png new file mode 100644 index 000000000..0257182f1 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-analytics-off-after-restart-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-analytics-what-we-share-ios.png b/docs/pr-screenshots/release/analytics-video/after-analytics-what-we-share-ios.png new file mode 100644 index 000000000..aeca99435 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-analytics-what-we-share-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-auto-setup-android.png b/docs/pr-screenshots/release/analytics-video/after-auto-setup-android.png new file mode 100644 index 000000000..10f61bd16 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-auto-setup-android.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-auto-setup-ios.png b/docs/pr-screenshots/release/analytics-video/after-auto-setup-ios.png new file mode 100644 index 000000000..ffe16ea99 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-auto-setup-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-character-consent-off-ios.png b/docs/pr-screenshots/release/analytics-video/after-character-consent-off-ios.png new file mode 100644 index 000000000..55b10a2b8 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-character-consent-off-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-character-consent-ticked-ios.png b/docs/pr-screenshots/release/analytics-video/after-character-consent-ticked-ios.png new file mode 100644 index 000000000..6085f6ec7 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-character-consent-ticked-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-character-delete-confirm-ios.png b/docs/pr-screenshots/release/analytics-video/after-character-delete-confirm-ios.png new file mode 100644 index 000000000..74f541797 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-character-delete-confirm-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-chat-quick-settings-android.png b/docs/pr-screenshots/release/analytics-video/after-chat-quick-settings-android.png new file mode 100644 index 000000000..0197d6b7b Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-chat-quick-settings-android.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-chat-quick-settings-ios.png b/docs/pr-screenshots/release/analytics-video/after-chat-quick-settings-ios.png new file mode 100644 index 000000000..a5a9cd01b Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-chat-quick-settings-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-chat-video-generating-ios.png b/docs/pr-screenshots/release/analytics-video/after-chat-video-generating-ios.png new file mode 100644 index 000000000..38b9d7a58 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-chat-video-generating-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-chat-video-ios.png b/docs/pr-screenshots/release/analytics-video/after-chat-video-ios.png new file mode 100644 index 000000000..82217f290 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-chat-video-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-chat-video-stopped-ios.png b/docs/pr-screenshots/release/analytics-video/after-chat-video-stopped-ios.png new file mode 100644 index 000000000..2d2585d28 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-chat-video-stopped-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-desktop-model-loading-stop-ios.png b/docs/pr-screenshots/release/analytics-video/after-desktop-model-loading-stop-ios.png new file mode 100644 index 000000000..0b2749c4a Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-desktop-model-loading-stop-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-desktop-stop-next-call-works-ios.png b/docs/pr-screenshots/release/analytics-video/after-desktop-stop-next-call-works-ios.png new file mode 100644 index 000000000..10a3fbdc2 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-desktop-stop-next-call-works-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-film-make-it-on-phone-ios.png b/docs/pr-screenshots/release/analytics-video/after-film-make-it-on-phone-ios.png new file mode 100644 index 000000000..01bac815f Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-film-make-it-on-phone-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-film-plays-ios.png b/docs/pr-screenshots/release/analytics-video/after-film-plays-ios.png new file mode 100644 index 000000000..5eebea664 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-film-plays-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-film-waits-for-stills-ios.png b/docs/pr-screenshots/release/analytics-video/after-film-waits-for-stills-ios.png new file mode 100644 index 000000000..c46cb9610 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-film-waits-for-stills-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-gallery-film-tiles-ios.png b/docs/pr-screenshots/release/analytics-video/after-gallery-film-tiles-ios.png new file mode 100644 index 000000000..d535c8ecf Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-gallery-film-tiles-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-gallery-videos-ios.png b/docs/pr-screenshots/release/analytics-video/after-gallery-videos-ios.png new file mode 100644 index 000000000..02d6666e6 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-gallery-videos-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-image-old-model-loads-ios.png b/docs/pr-screenshots/release/analytics-video/after-image-old-model-loads-ios.png new file mode 100644 index 000000000..c7512429e Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-image-old-model-loads-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-making-screen-ios.png b/docs/pr-screenshots/release/analytics-video/after-making-screen-ios.png new file mode 100644 index 000000000..77b23454a Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-making-screen-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-models-embedding-ios.png b/docs/pr-screenshots/release/analytics-video/after-models-embedding-ios.png new file mode 100644 index 000000000..7d0e88d4a Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-models-embedding-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-models-embedding-switch-ios.png b/docs/pr-screenshots/release/analytics-video/after-models-embedding-switch-ios.png new file mode 100644 index 000000000..357edf51a Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-models-embedding-switch-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-models-tabs-ios.png b/docs/pr-screenshots/release/analytics-video/after-models-tabs-ios.png new file mode 100644 index 000000000..979479275 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-models-tabs-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-models-video-downloading-ios.png b/docs/pr-screenshots/release/analytics-video/after-models-video-downloading-ios.png new file mode 100644 index 000000000..09ef26593 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-models-video-downloading-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-models-video-tab-ios.png b/docs/pr-screenshots/release/analytics-video/after-models-video-tab-ios.png new file mode 100644 index 000000000..439f2fc23 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-models-video-tab-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-models-video-wan21-only-ios.png b/docs/pr-screenshots/release/analytics-video/after-models-video-wan21-only-ios.png new file mode 100644 index 000000000..b94f381de Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-models-video-wan21-only-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-quit-reopen-chat-kept-ios.png b/docs/pr-screenshots/release/analytics-video/after-quit-reopen-chat-kept-ios.png new file mode 100644 index 000000000..1151c4a5c Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-quit-reopen-chat-kept-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-settings-analytics-off-android.png b/docs/pr-screenshots/release/analytics-video/after-settings-analytics-off-android.png new file mode 100644 index 000000000..89212c5f6 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-settings-analytics-off-android.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-settings-analytics-off-ios.png b/docs/pr-screenshots/release/analytics-video/after-settings-analytics-off-ios.png new file mode 100644 index 000000000..72eb6d8ce Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-settings-analytics-off-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-settings-video-generation-ios.png b/docs/pr-screenshots/release/analytics-video/after-settings-video-generation-ios.png new file mode 100644 index 000000000..0c8b05a7e Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-settings-video-generation-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-story-cards-ios.png b/docs/pr-screenshots/release/analytics-video/after-story-cards-ios.png new file mode 100644 index 000000000..252ff4fa0 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-story-cards-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-video-budget-clip-plays-ios.png b/docs/pr-screenshots/release/analytics-video/after-video-budget-clip-plays-ios.png new file mode 100644 index 000000000..aad1d4e9f Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-video-budget-clip-plays-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-video-clip-plays-ios.png b/docs/pr-screenshots/release/analytics-video/after-video-clip-plays-ios.png new file mode 100644 index 000000000..88a2dee6c Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-video-clip-plays-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-video-delete-confirm-ios.png b/docs/pr-screenshots/release/analytics-video/after-video-delete-confirm-ios.png new file mode 100644 index 000000000..bec321c49 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-video-delete-confirm-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-video-keep-app-open-stopped-ios.png b/docs/pr-screenshots/release/analytics-video/after-video-keep-app-open-stopped-ios.png new file mode 100644 index 000000000..f8eed85ec Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-video-keep-app-open-stopped-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-video-models-offgrid-mark-android.png b/docs/pr-screenshots/release/analytics-video/after-video-models-offgrid-mark-android.png new file mode 100644 index 000000000..f9f74a331 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-video-models-offgrid-mark-android.png differ diff --git a/docs/pr-screenshots/release/analytics-video/after-video-needs-pro-ios.png b/docs/pr-screenshots/release/analytics-video/after-video-needs-pro-ios.png new file mode 100644 index 000000000..76e67afab Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/after-video-needs-pro-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/before-character-consent-tick-ios.png b/docs/pr-screenshots/release/analytics-video/before-character-consent-tick-ios.png new file mode 100644 index 000000000..38c55cece Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/before-character-consent-tick-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/before-character-delete-confirm-ios.png b/docs/pr-screenshots/release/analytics-video/before-character-delete-confirm-ios.png new file mode 100644 index 000000000..e8fe430bd Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/before-character-delete-confirm-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/before-chat-quick-settings-ios.png b/docs/pr-screenshots/release/analytics-video/before-chat-quick-settings-ios.png new file mode 100644 index 000000000..53c69af89 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/before-chat-quick-settings-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/before-chat-video-not-enough-memory-ios.png b/docs/pr-screenshots/release/analytics-video/before-chat-video-not-enough-memory-ios.png new file mode 100644 index 000000000..4613815d6 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/before-chat-video-not-enough-memory-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/before-models-tabs-ios.png b/docs/pr-screenshots/release/analytics-video/before-models-tabs-ios.png new file mode 100644 index 000000000..0faed8741 Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/before-models-tabs-ios.png differ diff --git a/docs/pr-screenshots/release/analytics-video/before-video-models-offgrid-mark-android.png b/docs/pr-screenshots/release/analytics-video/before-video-models-offgrid-mark-android.png new file mode 100644 index 000000000..4f577608e Binary files /dev/null and b/docs/pr-screenshots/release/analytics-video/before-video-models-offgrid-mark-android.png differ diff --git a/fastlane/Fastfile b/fastlane/Fastfile index b100ad292..ad52935d2 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -17,7 +17,7 @@ require "json" require "fileutils" -# Off Grid ships no telemetry. Disable fastlane's anonymous usage analytics too. +# Off Grid sends analytics only when the person opts in. Disable fastlane's anonymous usage analytics. opt_out_usage REPO_ROOT = File.expand_path("..", __dir__) diff --git a/fastlane/RELEASING.md b/fastlane/RELEASING.md index c0c8605fd..ce25889cf 100644 --- a/fastlane/RELEASING.md +++ b/fastlane/RELEASING.md @@ -3,8 +3,9 @@ Hand-written guide. Note: `fastlane/README.md` is auto-generated by fastlane (it gets overwritten every time fastlane runs), so durable docs live here. -Off Grid ships no telemetry; fastlane's own anonymous analytics are disabled in the -Fastfile via `opt_out_usage`. +Off Grid sends anonymous usage data and crash reports only when the person opts in (see +`docs/ANALYTICS.md`); chats and files never leave the device. fastlane's own anonymous analytics +are disabled in the Fastfile via `opt_out_usage`. ## Setup diff --git a/fastlane/metadata/android/en-US/full_description.txt b/fastlane/metadata/android/en-US/full_description.txt index ce1eb5db4..b2d9f5ba9 100644 --- a/fastlane/metadata/android/en-US/full_description.txt +++ b/fastlane/metadata/android/en-US/full_description.txt @@ -1,4 +1,4 @@ -Off Grid is a fully on-device AI assistant. Large language models run directly on your phone via llama.cpp and GGUF — no cloud, no account, no telemetry. Airplane-mode tested. +Off Grid is a fully on-device AI assistant. Large language models run directly on your phone via llama.cpp and GGUF. No cloud, no account. Your chats and files never leave your device; anonymous usage data and crash reports only if you turn them on. Airplane-mode tested. Capabilities: diff --git a/fastlane/metadata/android/en-US/short_description.txt b/fastlane/metadata/android/en-US/short_description.txt index 439b82f7d..240b3c52f 100644 --- a/fastlane/metadata/android/en-US/short_description.txt +++ b/fastlane/metadata/android/en-US/short_description.txt @@ -1 +1 @@ -Run open-source LLMs on your device. Voice, vision, image gen — all offline. +On-device AI: chat, voice, vision, images. Your chats never leave your phone. diff --git a/ios/DeviceMemoryModule.swift b/ios/DeviceMemoryModule.swift index d636accf3..b6a7f4986 100644 --- a/ios/DeviceMemoryModule.swift +++ b/ios/DeviceMemoryModule.swift @@ -38,9 +38,28 @@ class DeviceMemoryModule: NSObject { footprintBytes = info.phys_footprint } + // System-wide pages the phone can hand out without killing anything: free + inactive + + // purgeable. os_proc_available_memory only reports this app's own limit, so it still says + // "plenty" while the whole phone is short of pages and the OS kills the app (vm-pageshortage). + var systemAvailableBytes: UInt64 = 0 + var vmStats = vm_statistics64_data_t() + var vmCount = mach_msg_type_number_t( + MemoryLayout.size / MemoryLayout.size) + let vmResult = withUnsafeMutablePointer(to: &vmStats) { ptr -> kern_return_t in + ptr.withMemoryRebound(to: integer_t.self, capacity: Int(vmCount)) { intPtr in + host_statistics64(mach_host_self(), HOST_VM_INFO64, intPtr, &vmCount) + } + } + if vmResult == KERN_SUCCESS { + let pages = UInt64(vmStats.free_count) + UInt64(vmStats.inactive_count) + UInt64(vmStats.purgeable_count) + systemAvailableBytes = pages * UInt64(vm_kernel_page_size) + } + resolve([ // The real budget: what this process may still allocate right now. "processAvailableBytes": NSNumber(value: Double(availableBytes)), + // What the whole phone can still hand out (0 when it could not be read). + "systemAvailableBytes": NSNumber(value: Double(systemAvailableBytes)), "footprintBytes": NSNumber(value: Double(footprintBytes)), // iOS has no system-wide low-memory flag here; the available number is authoritative. "lowMemory": false, diff --git a/ios/OffgridMobile/Info.plist b/ios/OffgridMobile/Info.plist index 350fa2054..592d2f8d3 100644 --- a/ios/OffgridMobile/Info.plist +++ b/ios/OffgridMobile/Info.plist @@ -108,6 +108,10 @@ UILaunchStoryboardName LaunchScreen + UIBackgroundModes + + audio + UIRequiredDeviceCapabilities arm64 @@ -124,5 +128,9 @@ UIViewControllerBasedStatusBarAppearance + BGTaskSchedulerPermittedIdentifiers + + $(PRODUCT_BUNDLE_IDENTIFIER).video.* + diff --git a/ios/OffgridMobile/OffgridMobile.entitlements b/ios/OffgridMobile/OffgridMobile.entitlements index 510563fbc..e480f62a3 100644 --- a/ios/OffgridMobile/OffgridMobile.entitlements +++ b/ios/OffgridMobile/OffgridMobile.entitlements @@ -6,5 +6,7 @@ com.apple.developer.kernel.extended-virtual-addressing + com.apple.developer.background-tasks.continued-processing.gpu + diff --git a/ios/OffgridMobile/PrivacyInfo.xcprivacy b/ios/OffgridMobile/PrivacyInfo.xcprivacy index 53631f09b..3376f7b12 100644 --- a/ios/OffgridMobile/PrivacyInfo.xcprivacy +++ b/ios/OffgridMobile/PrivacyInfo.xcprivacy @@ -41,7 +41,73 @@ NSPrivacyCollectedDataTypes - + + + NSPrivacyCollectedDataType + NSPrivacyCollectedDataTypeProductInteraction + NSPrivacyCollectedDataTypeLinked + + NSPrivacyCollectedDataTypeTracking + + NSPrivacyCollectedDataTypePurposes + + NSPrivacyCollectedDataTypePurposeAnalytics + NSPrivacyCollectedDataTypePurposeAppFunctionality + + + + NSPrivacyCollectedDataType + NSPrivacyCollectedDataTypeCrashData + NSPrivacyCollectedDataTypeLinked + + NSPrivacyCollectedDataTypeTracking + + NSPrivacyCollectedDataTypePurposes + + NSPrivacyCollectedDataTypePurposeAnalytics + NSPrivacyCollectedDataTypePurposeAppFunctionality + + + + NSPrivacyCollectedDataType + NSPrivacyCollectedDataTypePerformanceData + NSPrivacyCollectedDataTypeLinked + + NSPrivacyCollectedDataTypeTracking + + NSPrivacyCollectedDataTypePurposes + + NSPrivacyCollectedDataTypePurposeAnalytics + NSPrivacyCollectedDataTypePurposeAppFunctionality + + + + NSPrivacyCollectedDataType + NSPrivacyCollectedDataTypeOtherDiagnosticData + NSPrivacyCollectedDataTypeLinked + + NSPrivacyCollectedDataTypeTracking + + NSPrivacyCollectedDataTypePurposes + + NSPrivacyCollectedDataTypePurposeAnalytics + NSPrivacyCollectedDataTypePurposeAppFunctionality + + + + NSPrivacyCollectedDataType + NSPrivacyCollectedDataTypeDeviceID + NSPrivacyCollectedDataTypeLinked + + NSPrivacyCollectedDataTypeTracking + + NSPrivacyCollectedDataTypePurposes + + NSPrivacyCollectedDataTypePurposeAnalytics + NSPrivacyCollectedDataTypePurposeAppFunctionality + + + NSPrivacyTracking diff --git a/ios/Podfile b/ios/Podfile index fd0564901..a56c23f49 100644 --- a/ios/Podfile +++ b/ios/Podfile @@ -20,6 +20,10 @@ end target 'OffgridMobile' do config = use_native_modules! + # The script reuses a framework only when its pinned revision and headers match. + video_builder = File.expand_path('../scripts/build-video-ios.sh', __dir__) + raise 'Video runtime build failed' unless system('bash', video_builder) + pod 'OffgridVideo', :path => '../native/video' use_react_native!( :path => config[:reactNativePath], diff --git a/ios/Podfile.lock b/ios/Podfile.lock index f83dc1848..1de561501 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -42,6 +42,8 @@ PODS: - MMKV (2.4.0): - MMKVCore (~> 2.4.0) - MMKVCore (2.4.0) + - OffgridVideo (0.1.0): + - React-Core - op-sqlite (15.2.5): - boost - DoubleConversion @@ -2303,6 +2305,92 @@ PODS: - react-native-tcp-socket (6.4.1): - CocoaAsyncSocket - React-Core + - react-native-video (6.19.3): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - react-native-video/Video (= 6.19.3) + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - SocketRocket + - Yoga + - react-native-video/Fabric (6.19.3): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - SocketRocket + - Yoga + - react-native-video/Video (6.19.3): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - react-native-video/Fabric + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - SocketRocket + - Yoga - react-native-voice (3.2.4): - React-Core - react-native-zeroconf (0.14.0): @@ -3506,6 +3594,7 @@ DEPENDENCIES: - glog (from `../node_modules/react-native/third-party-podspecs/glog.podspec`) - hermes-engine (from `../node_modules/react-native/sdks/hermes-engine/hermes-engine.podspec`) - llama-rn (from `../node_modules/llama.rn`) + - OffgridVideo (from `../native/video`) - "op-sqlite (from `../node_modules/@op-engineering/op-sqlite`)" - RCT-Folly (from `../node_modules/react-native/third-party-podspecs/RCT-Folly.podspec`) - RCTDeprecation (from `../node_modules/react-native/ReactApple/Libraries/RCTFoundation/RCTDeprecation`) @@ -3554,6 +3643,7 @@ DEPENDENCIES: - react-native-safe-area-context (from `../node_modules/react-native-safe-area-context`) - "react-native-slider (from `../node_modules/@react-native-community/slider`)" - react-native-tcp-socket (from `../node_modules/react-native-tcp-socket`) + - react-native-video (from `../node_modules/react-native-video`) - "react-native-voice (from `../node_modules/@react-native-voice/voice`)" - react-native-zeroconf (from `../node_modules/react-native-zeroconf`) - React-NativeModulesApple (from `../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios`) @@ -3636,6 +3726,8 @@ EXTERNAL SOURCES: :tag: hermes-v0.14.0 llama-rn: :path: "../node_modules/llama.rn" + OffgridVideo: + :path: "../native/video" op-sqlite: :path: "../node_modules/@op-engineering/op-sqlite" RCT-Folly: @@ -3730,6 +3822,8 @@ EXTERNAL SOURCES: :path: "../node_modules/@react-native-community/slider" react-native-tcp-socket: :path: "../node_modules/react-native-tcp-socket" + react-native-video: + :path: "../node_modules/react-native-video" react-native-voice: :path: "../node_modules/@react-native-voice/voice" react-native-zeroconf: @@ -3849,6 +3943,7 @@ SPEC CHECKSUMS: llama-rn: dd60415f7ae47554b7498aed4f4e39b522bb19c3 MMKV: 86859fdfa2b0b21db1fd6e48788474a6416a2c77 MMKVCore: 3d16ce9f7d411e135020915fde98a056859a1efa + OffgridVideo: 3a0d4acebd54e2ab315c9ff737aa017ccc3d36b8 op-sqlite: ab292d8057b994f220b8c61f2683f5ff9286958e opencv-rne: 2305807573b6e29c8c87e3416ab096d09047a7a0 RCT-Folly: 846fda9475e61ec7bcbf8a3fe81edfcaeb090669 @@ -3897,6 +3992,7 @@ SPEC CHECKSUMS: react-native-safe-area-context: c00143b4823773bba23f2f19f85663ae89ceb460 react-native-slider: 34064ca1a6864d7b263e44dd76a2d794e8d26744 react-native-tcp-socket: 7c7e53a07f122ecf00fb3626684bc0ca82c4f044 + react-native-video: 61841250a61ed594812b0bd4ce3f03549d574efc react-native-voice: 908a0eba96c8c3d643e4f98b7232c6557d0a6f9c react-native-zeroconf: eb2e5584308f20f5fc3eb0cea2ceafbbd345b48b React-NativeModulesApple: a2c3d2cbec893956a5b3e4060322db2984fff75b @@ -3953,6 +4049,6 @@ SPEC CHECKSUMS: whisper-rn: 7566faf9b7d78e39ab9fc634cb90fdee81177793 Yoga: 5456bb010373068fc92221140921b09d126b116e -PODFILE CHECKSUM: d2e7563af7ca0531bcab79cceff20997d2fbe48c +PODFILE CHECKSUM: a51a87720248f5187c458d3af5dfc690def5388e COCOAPODS: 1.16.2 diff --git a/ios/StreamingHashModule.m b/ios/StreamingHashModule.m index b7a3a52e5..67b419c56 100644 --- a/ios/StreamingHashModule.m +++ b/ios/StreamingHashModule.m @@ -6,4 +6,8 @@ @interface RCT_EXTERN_MODULE(StreamingHashModule, NSObject) resolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) +RCT_EXTERN_METHOD(sha256:(NSString *)path + resolver:(RCTPromiseResolveBlock)resolve + rejecter:(RCTPromiseRejectBlock)reject) + @end diff --git a/ios/StreamingHashModule.swift b/ios/StreamingHashModule.swift index 8b97e2363..9d3f76c3d 100644 --- a/ios/StreamingHashModule.swift +++ b/ios/StreamingHashModule.swift @@ -8,12 +8,25 @@ enum StreamingFileHasher { at url: URL, chunkSize: Int = defaultChunkSize, didConsumeChunk: (() -> Void)? = nil + ) throws -> String { + try digestHex(at: url, using: SHA512(), chunkSize: chunkSize, didConsumeChunk: didConsumeChunk) + } + + static func sha256Hex(at url: URL) throws -> String { + try digestHex(at: url, using: SHA256()) + } + + private static func digestHex( + at url: URL, + using initialHasher: H, + chunkSize: Int = defaultChunkSize, + didConsumeChunk: (() -> Void)? = nil ) throws -> String { precondition(chunkSize > 0) let handle = try FileHandle(forReadingFrom: url) defer { try? handle.close() } - var hasher = SHA512() + var hasher = initialHasher while true { // FileHandle can back each Swift Data with an autoreleased NSData. This work runs in one // long-lived GCD block, so the queue's outer autorelease pool is not drained until the whole @@ -45,6 +58,24 @@ final class StreamingHashModule: NSObject { _ path: String, resolver resolve: @escaping RCTPromiseResolveBlock, rejecter reject: @escaping RCTPromiseRejectBlock + ) { + hash(path, digest: { try StreamingFileHasher.sha512Hex(at: $0) }, resolve: resolve, reject: reject) + } + + @objc + func sha256( + _ path: String, + resolver resolve: @escaping RCTPromiseResolveBlock, + rejecter reject: @escaping RCTPromiseRejectBlock + ) { + hash(path, digest: StreamingFileHasher.sha256Hex, resolve: resolve, reject: reject) + } + + private func hash( + _ path: String, + digest: @escaping (URL) throws -> String, + resolve: @escaping RCTPromiseResolveBlock, + reject: @escaping RCTPromiseRejectBlock ) { DispatchQueue.global(qos: .utility).async { do { @@ -54,7 +85,7 @@ final class StreamingHashModule: NSObject { defer { if scoped { url.stopAccessingSecurityScopedResource() } } - resolve(try StreamingFileHasher.sha512Hex(at: url)) + resolve(try digest(url)) } catch { reject("streaming_hash_failed", error.localizedDescription, error) } diff --git a/jest.config.js b/jest.config.js index 0ad98da3f..d7896688b 100644 --- a/jest.config.js +++ b/jest.config.js @@ -110,10 +110,23 @@ module.exports = { './pro': { statements: 75, branches: 75, functions: 75, lines: 75 }, // New standalone modules in this change set are held to 100% on every axis. Changed // legacy files have their NEW branches covered by the suites but aren't whole-file-100%. - './src/utils/imageModelIntegrity.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/services/imageModelIntegrity.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, './src/utils/imageGenAdvice.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, './src/services/modelLoadErrors.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, './src/components/ImageGenAdviceCard.tsx': { statements: 100, branches: 100, functions: 100, lines: 100 }, './src/components/VoiceRecordButton/derive.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + // Opt-in analytics (docs/ANALYTICS.md): every new module is held to 100% on every axis. + './src/services/analytics/analytics.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/services/analytics/client.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/services/analytics/consentPrompt.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/services/analytics/featureSnapshot.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/services/analytics/modelFacts.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/services/analytics/scrub.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/services/analytics/uncleanExit.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/services/analytics/values.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/services/problemReport.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/components/AnalyticsConsentPrompt.tsx': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/screens/AnalyticsInfoScreen.tsx': { statements: 100, branches: 100, functions: 100, lines: 100 }, + './src/screens/SettingsAnalyticsSection.tsx': { statements: 100, branches: 100, functions: 100, lines: 100 }, }, }; diff --git a/jest.setup.ts b/jest.setup.ts index f3cfe1244..d0cc27841 100644 --- a/jest.setup.ts +++ b/jest.setup.ts @@ -468,6 +468,14 @@ jest.mock('react-native-device-info', () => ({ getDeviceId: jest.fn(() => 'test-device-id'), getHardware: jest.fn(() => Promise.resolve('unknown')), getIpAddress: jest.fn(() => Promise.resolve('192.168.1.20')), + // Battery + form factor (analytics `battery_pct`, `charging`, `device_class`). + getPowerState: jest.fn(() => Promise.resolve({ batteryLevel: 0.64, batteryState: 'unplugged', lowPowerMode: false })), + isTablet: jest.fn(() => false), + // Synchronous build/device identity the PostHog SDK reads for its app properties. + getApplicationName: jest.fn(() => 'Off Grid'), + getBundleId: jest.fn(() => 'ai.offgridmobile'), + getManufacturerSync: jest.fn(() => 'Google'), + isEmulatorSync: jest.fn(() => false), })); // react-native-image-picker mock @@ -631,6 +639,12 @@ jest.mock('react-native-reanimated', () => { }, FadeIn: { duration: jest.fn().mockReturnThis(), delay: jest.fn().mockReturnThis() }, FadeOut: { duration: jest.fn().mockReturnThis(), delay: jest.fn().mockReturnThis() }, + FadeInDown: { + duration: jest.fn().mockReturnThis(), + delay: jest.fn().mockReturnThis(), + springify: jest.fn().mockReturnThis(), + damping: jest.fn().mockReturnThis(), + }, SlideInDown: { duration: jest.fn().mockReturnThis() }, SlideOutDown: { duration: jest.fn().mockReturnThis() }, LinearTransition: { duration: jest.fn().mockReturnThis() }, @@ -656,6 +670,8 @@ jest.mock('@op-engineering/op-sqlite', () => { execute: jest.fn(() => Promise.resolve(mockResults)), close: jest.fn(), delete: jest.fn(), + // op-sqlite's real DB answers where its file lives (Storage sizes the document index, 8bc6073e3). + getDbPath: jest.fn(() => '/mock/databases/rag.db'), }; return { open: jest.fn(() => mockDb), diff --git a/knip.json b/knip.json index 5df8a954d..88d9bc7a2 100644 --- a/knip.json +++ b/knip.json @@ -15,6 +15,7 @@ ], "ignoreDependencies": [ "@offgrid/pro", + "@offgrid/film-native", "buffer", "js-sha512", "tweetnacl", diff --git a/native/video/OffgridVideo.podspec b/native/video/OffgridVideo.podspec new file mode 100644 index 000000000..2b607031b --- /dev/null +++ b/native/video/OffgridVideo.podspec @@ -0,0 +1,15 @@ +Pod::Spec.new do |s| + s.name = 'OffgridVideo' + s.version = '0.1.0' + s.summary = 'Local Wan video generation and MP4 encoding for Off Grid AI' + s.homepage = 'https://github.com/off-grid-ai/OGAM' + s.license = { :type => 'AGPL-3.0', :file => '../../LICENSE' } + s.author = 'Off Grid AI' + s.source = { :git => 'https://github.com/off-grid-ai/OGAM.git' } + s.platform = :ios, '17.0' + s.source_files = '*.{h,mm}', 'common/*.hpp' + s.vendored_frameworks = 'OffgridVideoRuntime.xcframework' + s.frameworks = 'BackgroundTasks', 'AVFoundation', 'CoreVideo', 'Metal', 'Accelerate', 'UIKit' + s.dependency 'React-Core' + s.pod_target_xcconfig = { 'CLANG_CXX_LANGUAGE_STANDARD' => 'c++17' } +end diff --git a/native/video/RUNTIME-LICENSE b/native/video/RUNTIME-LICENSE new file mode 100644 index 000000000..56e1e5a63 --- /dev/null +++ b/native/video/RUNTIME-LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 leejet + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/native/video/VideoEncoder.h b/native/video/VideoEncoder.h new file mode 100644 index 000000000..aa823f535 --- /dev/null +++ b/native/video/VideoEncoder.h @@ -0,0 +1,5 @@ +#import +#import +#include +BOOL OGEncodeVideo(sd_image_t *frames, int count, int fps, NSString *path, + const std::atomic_bool &cancelled, NSError **error); diff --git a/native/video/VideoEncoder.mm b/native/video/VideoEncoder.mm new file mode 100644 index 000000000..09a96b031 --- /dev/null +++ b/native/video/VideoEncoder.mm @@ -0,0 +1,64 @@ +#import "VideoEncoder.h" +#import + +BOOL OGEncodeVideo(sd_image_t *frames, int count, int fps, NSString *path, + const std::atomic_bool &cancelled, NSError **error) { + if (!frames || count < 1 || fps < 1) return NO; + AVAssetWriter *writer = [[AVAssetWriter alloc] initWithURL:[NSURL fileURLWithPath:path] + fileType:AVFileTypeMPEG4 error:error]; + if (!writer) return NO; + NSDictionary *settings = @{ AVVideoCodecKey: AVVideoCodecTypeH264, + AVVideoWidthKey: @(frames[0].width), AVVideoHeightKey: @(frames[0].height) }; + AVAssetWriterInput *input = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo outputSettings:settings]; + input.expectsMediaDataInRealTime = NO; + NSDictionary *attributes = @{ (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (id)kCVPixelBufferWidthKey: @(frames[0].width), (id)kCVPixelBufferHeightKey: @(frames[0].height), + (id)kCVPixelBufferIOSurfacePropertiesKey: @{} }; + AVAssetWriterInputPixelBufferAdaptor *adaptor = [AVAssetWriterInputPixelBufferAdaptor + assetWriterInputPixelBufferAdaptorWithAssetWriterInput:input sourcePixelBufferAttributes:attributes]; + if (![writer canAddInput:input]) return NO; + [writer addInput:input]; + if (![writer startWriting]) { if (error) *error = writer.error; return NO; } + [writer startSessionAtSourceTime:kCMTimeZero]; + BOOL ok = YES; + for (int i = 0; i < count && ok; ++i) { + @autoreleasepool { + // Bound a stalled encoder and permit Stop while waiting for its next buffer. + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:30]; + while (!input.readyForMoreMediaData && writer.status == AVAssetWriterStatusWriting && + !cancelled.load() && deadline.timeIntervalSinceNow > 0) [NSThread sleepForTimeInterval:0.01]; + if (cancelled.load() || !input.readyForMoreMediaData || frames[i].channel < 3 || !frames[i].data || frames[i].width != frames[0].width || frames[i].height != frames[0].height) { ok = NO; break; } + CVPixelBufferRef buffer = nil; + if (CVPixelBufferPoolCreatePixelBuffer(NULL, adaptor.pixelBufferPool, &buffer) != kCVReturnSuccess) { ok = NO; break; } + CVPixelBufferLockBaseAddress(buffer, 0); + auto *pixels = (uint8_t *)CVPixelBufferGetBaseAddress(buffer); + size_t stride = CVPixelBufferGetBytesPerRow(buffer); + for (uint32_t y = 0; y < frames[i].height; ++y) { + for (uint32_t x = 0; x < frames[i].width; ++x) { + const uint8_t *rgb = frames[i].data + (y * frames[i].width + x) * frames[i].channel; + uint8_t *bgra = pixels + y * stride + x * 4; + bgra[0] = rgb[2]; bgra[1] = rgb[1]; bgra[2] = rgb[0]; bgra[3] = 255; + } + } + CVPixelBufferUnlockBaseAddress(buffer, 0); + ok = [adaptor appendPixelBuffer:buffer withPresentationTime:CMTimeMake(i, fps)]; + CVPixelBufferRelease(buffer); + } + } + if (ok) { + [input markAsFinished]; + dispatch_semaphore_t finished = dispatch_semaphore_create(0); + [writer finishWritingWithCompletionHandler:^{ dispatch_semaphore_signal(finished); }]; + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:30]; + while (dispatch_semaphore_wait(finished, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_MSEC * 100)) != 0) { + if (cancelled.load() || deadline.timeIntervalSinceNow <= 0) { [writer cancelWriting]; break; } + } + ok = writer.status == AVAssetWriterStatusCompleted && !cancelled.load(); + } + if (!ok) { + if (error) *error = writer.error; + [writer cancelWriting]; + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } + return ok; +} diff --git a/native/video/VideoGenerationModule.mm b/native/video/VideoGenerationModule.mm new file mode 100644 index 000000000..b412a4532 --- /dev/null +++ b/native/video/VideoGenerationModule.mm @@ -0,0 +1,448 @@ +#import +#import +#import +#import +#import +#import "VideoEncoder.h" +#include "common/VideoRuntime.hpp" +#include +#include + +static BOOL OGSaveRgbPng(const sd_image_t &image, NSString *output) { + if (!image.data || (image.channel != 3 && image.channel != 4)) return NO; + CFDataRef data = CFDataCreate(kCFAllocatorDefault, image.data, image.width * image.height * image.channel); + CGDataProviderRef provider = data ? CGDataProviderCreateWithCFData(data) : nullptr; + CGColorSpaceRef color = CGColorSpaceCreateDeviceRGB(); + CGImageRef bitmap = provider ? CGImageCreate(image.width, image.height, 8, 8 * image.channel, image.width * image.channel, color, image.channel == 4 ? kCGImageAlphaLast : kCGImageAlphaNone, provider, nullptr, false, kCGRenderingIntentDefault) : nullptr; + NSData *png = bitmap ? UIImagePNGRepresentation([UIImage imageWithCGImage:bitmap]) : nil; + if (bitmap) CGImageRelease(bitmap); + CGColorSpaceRelease(color); if (provider) CGDataProviderRelease(provider); if (data) CFRelease(data); + return png && [png writeToFile:output atomically:YES]; +} + +@interface VideoGenerationModule : RCTEventEmitter +@end +/// A start image as RGB at the clip size, centre-cropped to fill (Animate this photo, Continue). +static std::vector OGInitImageRGB(NSString *path, int width, int height) { + UIImage *image = [UIImage imageWithContentsOfFile:[path stringByReplacingOccurrencesOfString:@"file://" withString:@""]]; + if (!image.CGImage || width <= 0 || height <= 0) return {}; + std::vector rgba(static_cast(width) * height * 4); + CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate(rgba.data(), width, height, 8, width * 4, space, + kCGImageAlphaNoneSkipLast | kCGBitmapByteOrder32Big); + CGColorSpaceRelease(space); + if (!context) return {}; + const CGFloat scale = MAX(width / image.size.width, height / image.size.height); + const CGSize drawn = CGSizeMake(image.size.width * scale, image.size.height * scale); + UIGraphicsPushContext(context); + CGContextTranslateCTM(context, 0, height); + CGContextScaleCTM(context, 1, -1); + [image drawInRect:CGRectMake((width - drawn.width) / 2, (height - drawn.height) / 2, drawn.width, drawn.height)]; + UIGraphicsPopContext(); + CGContextRelease(context); + std::vector rgb(static_cast(width) * height * 3); + for (size_t pixel = 0, count = static_cast(width) * height; pixel < count; ++pixel) { + rgb[pixel * 3] = rgba[pixel * 4]; rgb[pixel * 3 + 1] = rgba[pixel * 4 + 1]; rgb[pixel * 3 + 2] = rgba[pixel * 4 + 2]; + } + return rgb; +} + +@implementation VideoGenerationModule { + offgrid::VideoRuntime _runtime; + dispatch_queue_t _worker; + BOOL _busy; + BOOL _listeners; + BOOL _holdsIdleTimer; + BOOL _previousIdleTimerDisabled; + NSString *_videoInterruptionReason; + NSString *_imageNativeError; + BGTask *_continued; + NSString *_taskIdentifier; + dispatch_block_t _continuedWork; +} +RCT_EXPORT_MODULE(VideoGenerationModule) ++ (BOOL)requiresMainQueueSetup { return YES; } +// Start images and sd.cpp's in-process ESRGAN upscaler (exported by this runtime build) run here. +- (NSDictionary *)constantsToExport { return @{@"supportsInitImage": @YES, @"supportsUpscale": @YES}; } +- (instancetype)init { + if ((self = [super init])) { + _worker = dispatch_queue_create("ai.offgrid.video", DISPATCH_QUEUE_SERIAL); + + [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(backgrounded) + name:UIApplicationDidEnterBackgroundNotification object:nil]; + } + return self; +} +- (void)dealloc { + [[NSNotificationCenter defaultCenter] removeObserver:self]; + if (_holdsIdleTimer) { + BOOL previous = _previousIdleTimerDisabled; + dispatch_async(dispatch_get_main_queue(), ^{ UIApplication.sharedApplication.idleTimerDisabled = previous; }); + } +} +- (void)keepScreenAwake { + if (_holdsIdleTimer) return; + _previousIdleTimerDisabled = UIApplication.sharedApplication.idleTimerDisabled; + _holdsIdleTimer = YES; + UIApplication.sharedApplication.idleTimerDisabled = YES; +} +- (void)restoreScreenIdleTimer { + if (!_holdsIdleTimer) return; + UIApplication.sharedApplication.idleTimerDisabled = _previousIdleTimerDisabled; + _holdsIdleTimer = NO; +} +- (NSArray *)supportedEvents { return @[@"VideoGenerationProgress", @"SDImageProgress"]; } +- (void)startObserving { _listeners = YES; } +- (void)stopObserving { _listeners = NO; } +- (void)logLifecycle:(NSString *)detail { + NSLog(@"[VideoLifecycle] %@", detail); + if (_listeners) [self sendEventWithName:@"VideoGenerationProgress" + body:@{@"lifecycle":detail, @"at":@([[NSDate date] timeIntervalSince1970] * 1000)}]; +} +- (void)beginImageDiagnostics { + @synchronized (self) { _imageNativeError = nil; } + sd_set_log_callback([](enum sd_log_level_t level, const char *text, void *data) { + if (!text) return; + std::fputs(text, stderr); + // Retain placement and failure evidence, without prompts or model contents. + if (std::strstr(text, "prompt")) return; + if (level < SD_LOG_WARN && !std::strstr(text, "auto-fit") && + !std::strstr(text, "backend") && !std::strstr(text, "MiB") && + !std::strstr(text, "sampling completed") && !std::strstr(text, "cancelling")) return; + VideoGenerationModule *owner = (__bridge VideoGenerationModule *)data; + NSString *detail = [NSString stringWithUTF8String:text]; + if (!detail) return; + if (detail.length > 2048) detail = [detail substringToIndex:2048]; + if (level == SD_LOG_ERROR) { + @synchronized (owner) { owner->_imageNativeError = detail; } + } + dispatch_async(dispatch_get_main_queue(), ^{ + if (owner->_listeners) [owner sendEventWithName:@"SDImageProgress" body:@{@"diagnostic":detail}]; + }); + }, (__bridge void *)self); +} +- (NSString *)finishImageDiagnostics:(NSString *)failure { + sd_set_log_callback(nullptr, nullptr); + @synchronized (self) { + return failure && _imageNativeError.length + ? [NSString stringWithFormat:@"%@ %@", failure, _imageNativeError] : failure; + } +} +- (void)interruptVideo:(NSString *)reason { + _videoInterruptionReason = reason; + _runtime.cancel(); + if (_listeners) [self sendEventWithName:@"VideoGenerationProgress" + body:@{@"interruption":reason}]; +} +- (void)backgrounded { + // Metal work must stop before the OS removes GPU access. Continued GPU tasks + // are admitted separately by the background coordinator on supported systems. + [self logLifecycle:[NSString stringWithFormat:@"background busy=%d admittedGPU=%d", _busy, _continued != nil]]; + if (_busy && !_continued) { + [self interruptVideo:@"Video generation stopped when the app went into the background. Keep Off Grid open and try again."]; + } +} +- (void)emitStage:(NSString *)stage step:(int)step total:(int)total { + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_videoInterruptionReason) return; + if (@available(iOS 26.0, *)) { + if (self->_continued && total > 0) { + BGContinuedProcessingTask *task = (BGContinuedProcessingTask *)self->_continued; + task.progress.totalUnitCount = total + 2; + task.progress.completedUnitCount = MIN(step + 1, total + 1); + [task updateTitle:@"Generating video" subtitle:stage]; + } + } + if (self->_listeners) [self sendEventWithName:@"VideoGenerationProgress" + body:@{@"stage":stage, @"step":@(step), @"total":@(total)}]; + }); +} +/// Upscale every frame of a clip with sd.cpp's ESRGAN, then write it at the clip's own rate. +RCT_REMAP_METHOD(upscaleVideo, upscaleVideo:(NSDictionary *)input resolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { + NSString *inputPath = input[@"inputPath"], *outputPath = input[@"outputPath"], *modelPath = input[@"modelPath"]; + const uint32_t factor = [input[@"scale"] unsignedIntValue] ?: 2; + self->_runtime.cancelled.store(false); + dispatch_async(self->_worker, ^{ + @autoreleasepool { + AVAsset *asset = [AVAsset assetWithURL:[NSURL fileURLWithPath:inputPath]]; + AVAssetTrack *track = [[asset tracksWithMediaType:AVMediaTypeVideo] firstObject]; + NSError *error = nil; + AVAssetReader *reader = track ? [AVAssetReader assetReaderWithAsset:asset error:&error] : nil; + if (!reader) { reject(@"VIDEO_UPSCALE_FAILED", @"The clip has no video.", error); return; } + AVAssetReaderTrackOutput *output = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:track + outputSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)}]; + [reader addOutput:output]; + upscaler_ctx_t *upscaler = new_upscaler_ctx(modelPath.UTF8String, false, 4, 128, nullptr, nullptr); + if (!upscaler) { reject(@"VIDEO_UPSCALE_FAILED", @"The upscaler model did not load.", nil); return; } + std::vector frames; + const int fps = MAX(1, (int)lroundf(track.nominalFrameRate)); + BOOL ok = [reader startReading]; + while (ok && !self->_runtime.cancelled.load()) { + CMSampleBufferRef sample = [output copyNextSampleBuffer]; + if (!sample) break; + CVPixelBufferRef pixels = CMSampleBufferGetImageBuffer(sample); + CVPixelBufferLockBaseAddress(pixels, kCVPixelBufferLock_ReadOnly); + const size_t w = CVPixelBufferGetWidth(pixels), h = CVPixelBufferGetHeight(pixels), stride = CVPixelBufferGetBytesPerRow(pixels); + const uint8_t *bgra = static_cast(CVPixelBufferGetBaseAddress(pixels)); + std::vector rgb(w * h * 3); + for (size_t y = 0; y < h; ++y) for (size_t x = 0; x < w; ++x) { + const uint8_t *p = bgra + y * stride + x * 4; + rgb[(y * w + x) * 3] = p[2]; rgb[(y * w + x) * 3 + 1] = p[1]; rgb[(y * w + x) * 3 + 2] = p[0]; + } + CVPixelBufferUnlockBaseAddress(pixels, kCVPixelBufferLock_ReadOnly); + CFRelease(sample); + sd_image_t *up = nullptr; int count = 0; + if (!upscale(upscaler, {static_cast(w), static_cast(h), 3, rgb.data()}, factor, &up, &count) || !up || count < 1) { ok = NO; break; } + frames.push_back(up[0]); + free(up); + } + free_upscaler_ctx(upscaler); + NSError *encodeError = nil; + if (ok && !frames.empty() && !self->_runtime.cancelled.load()) + ok = OGEncodeVideo(frames.data(), static_cast(frames.size()), fps, outputPath, self->_runtime.cancelled, &encodeError); + else ok = NO; + for (auto &frame : frames) free(frame.data); + if (ok) resolve(@{@"path": outputPath}); + else reject(@"VIDEO_UPSCALE_FAILED", encodeError.localizedDescription ?: @"Could not upscale the clip.", encodeError); + } + }); +} +RCT_REMAP_METHOD(cancel, cancelWithResolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { + dispatch_async(dispatch_get_main_queue(), ^{ + self->_runtime.cancel(); + if (self->_continuedWork) { + if (self->_taskIdentifier) [BGTaskScheduler.sharedScheduler cancelTaskRequestWithIdentifier:self->_taskIdentifier]; + dispatch_block_t pending = self->_continuedWork; self->_continuedWork = nil; + pending(); // The worker observes cancellation and settles the JS promise. + } + resolve(nil); + }); +} +RCT_REMAP_METHOD(generate, generate:(NSDictionary *)input resolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { + // The request's paths and prompt become std::string; a missing one ([nil UTF8String] is NULL) is + // undefined behaviour there, so it is refused here instead. + for (NSString *key in @[@"weight", @"vae", @"prompt", @"outputPath"]) { + if (![input[key] isKindOfClass:NSString.class]) { + reject(@"VIDEO_BAD_REQUEST", [NSString stringWithFormat:@"The video request has no %@.", key], nil); + return; + } + } + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_busy) { reject(@"VIDEO_BUSY", @"Video generation is already running.", nil); return; } + self->_busy = YES; self->_runtime.cancelled.store(false); + self->_videoInterruptionReason = nil; + [self keepScreenAwake]; + [self startContinuedWork:^{ + dispatch_async(self->_worker, ^{ + @autoreleasepool { + NSString *output = input[@"outputPath"]; + NSError *failure = nil; + // Keep targeted runtime diagnostics in the same persistent lifecycle log. + sd_set_log_callback([](enum sd_log_level_t, const char *text, void *data) { + if (!text) return; + std::fputs(text, stderr); + if (!std::strstr(text, "Wan VAE decode backend=")) return; + VideoGenerationModule *owner = (__bridge VideoGenerationModule *)data; + NSString *detail = [NSString stringWithUTF8String:text]; + dispatch_async(dispatch_get_main_queue(), ^{ [owner logLifecycle:detail]; }); + }, (__bridge void *)self); + try { + offgrid::VideoRequest request{ + [input[@"weight"] UTF8String], [input[@"vae"] UTF8String], [(input[@"encoder"] ?: @"") UTF8String], + [input[@"prompt"] UTF8String], [(input[@"negativePrompt"] ?: @"") UTF8String], + [input[@"width"] intValue], [input[@"height"] intValue], [input[@"frames"] intValue], + [input[@"fps"] intValue], [input[@"steps"] intValue], [input[@"guidance"] floatValue], [input[@"seed"] longLongValue] + }; + request.llm = [(input[@"llm"] ?: @"") UTF8String]; + request.embeddings = [(input[@"embeddings"] ?: @"") UTF8String]; + request.audioVae = [(input[@"audioVae"] ?: @"") UTF8String]; + request.flowShift = [input[@"flowShift"] floatValue]; + // Fastest settings' choice: the CPU when it was placed there, Metal otherwise. + request.cpuOnly = [input[@"backend"] isEqual:@"cpu"]; + if ([input[@"initImage"] isKindOfClass:NSString.class]) + request.initImage = OGInitImageRGB(input[@"initImage"], request.width, request.height); + [self emitStage:@"preparing" step:0 total:request.steps]; + self->_runtime.run(request, [&](int step, int total) { + [self emitStage:@"generating" step:step total:total]; + }, [&](sd_image_t *frames, int count, int fps) { + [self emitStage:@"encoding" step:0 total:count]; + NSError *encodingError = nil; + if (!OGEncodeVideo(frames, count, fps, output, self->_runtime.cancelled, &encodingError)) + throw std::runtime_error(encodingError ? encodingError.localizedDescription.UTF8String : "Video encoding stopped."); + }, [&](const char *) { [self emitStage:@"conditioning" step:0 total:0]; }, + [&](int completed, int total) { [self emitStage:@"decoding" step:completed total:total]; }, + [&](const sd_image_t &image) { + if (self->_runtime.cancelled.load()) return; + NSString *path = [output stringByAppendingString:@".preview.png"]; + @try { + if (!OGSaveRgbPng(image, path)) return; + } @catch (NSException *exception) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + return; + } + NSDictionary *preview = @{@"path":path, @"width":@(image.width), @"height":@(image.height)}; + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_listeners) [self sendEventWithName:@"VideoGenerationProgress" + body:@{@"stage":@"encoding", @"step":@0, @"total":@0, @"preview":preview}]; + }); + }); + } catch (const std::exception &error) { + failure = [NSError errorWithDomain:@"OffgridVideo" code:1 userInfo:@{NSLocalizedDescriptionKey:@(error.what())}]; + [[NSFileManager defaultManager] removeItemAtPath:output error:nil]; + } + sd_set_log_callback(nullptr, nullptr); + dispatch_async(dispatch_get_main_queue(), ^{ + self->_busy = NO; + [self restoreScreenIdleTimer]; + if (self->_continued) { + [self->_continued setTaskCompletedWithSuccess:failure == nil]; self->_continued = nil; + } + if (failure && self->_videoInterruptionReason) + reject(@"VIDEO_BACKGROUND_INTERRUPTED", self->_videoInterruptionReason, failure); + else if (failure) reject(self->_runtime.cancelled.load() ? @"VIDEO_CANCELLED" : @"VIDEO_FAILED", failure.localizedDescription, failure); + else resolve(@{@"path":output}); + }); + } + }); + }]; + }); +} +RCT_REMAP_METHOD(getLoadedImagePath, imagePathWithResolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { + auto path = _runtime.loadedImagePath(); resolve(path.empty() ? nil : @(path.c_str())); +} +RCT_REMAP_METHOD(loadImageModel, loadImage:(NSDictionary *)input resolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_busy) { reject(@"IMAGE_BUSY", @"Image or video generation is running.", nil); return; } + self->_busy = YES; self->_runtime.cancelled.store(false); + dispatch_async(self->_worker, ^{ + NSString *failure = nil; + [self beginImageDiagnostics]; + try { + offgrid::VideoRequest request{}; + request.weight = [input[@"weight"] UTF8String]; request.vae = [(input[@"vae"] ?: @"") UTF8String]; + request.llm = [(input[@"llm"] ?: @"") UTF8String]; + request.imageFamily = [(input[@"family"] ?: @"") UTF8String]; + request.imageSampler = [(input[@"sampler"] ?: @"") UTF8String]; + request.imageScheduler = [(input[@"scheduler"] ?: @"") UTF8String]; + request.threads = [input[@"threads"] intValue]; request.cpuOnly = [input[@"cpuOnly"] boolValue]; + self->_runtime.loadImage(request, [input[@"modelPath"] UTF8String]); + } catch (const std::exception &error) { failure = @(error.what()); } + failure = [self finishImageDiagnostics:failure]; + dispatch_async(dispatch_get_main_queue(), ^{ + self->_busy = NO; + if (failure) reject(@"IMAGE_LOAD_FAILED", failure, nil); else resolve(@YES); + }); + }); + }); +} +RCT_REMAP_METHOD(unloadImageModel, unloadImageWithResolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_busy) { reject(@"IMAGE_BUSY", @"Image or video generation is running.", nil); return; } + self->_busy = YES; + dispatch_async(self->_worker, ^{ + NSString *failure = nil; + try { self->_runtime.unloadImage(); } catch (const std::exception &error) { failure = @(error.what()); } + dispatch_async(dispatch_get_main_queue(), ^{ + self->_busy = NO; + if (failure) reject(@"IMAGE_UNLOAD_FAILED", failure, nil); else resolve(@YES); + }); + }); + }); +} +RCT_REMAP_METHOD(generateImage, generateImage:(NSDictionary *)input resolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_busy) { reject(@"IMAGE_BUSY", @"Image or video generation is running.", nil); return; } + self->_busy = YES; self->_runtime.cancelled.store(false); + [self keepScreenAwake]; + dispatch_async(self->_worker, ^{ + @autoreleasepool { + NSString *output = input[@"outputPath"]; + NSString *failure = nil; + [self beginImageDiagnostics]; + try { + offgrid::VideoRequest request{}; + request.prompt = [input[@"prompt"] UTF8String]; request.negative = [input[@"negativePrompt"] UTF8String]; + request.width = [input[@"width"] intValue]; request.height = [input[@"height"] intValue]; + request.steps = [input[@"steps"] intValue]; request.guidance = [input[@"guidanceScale"] floatValue]; request.seed = [input[@"seed"] longLongValue]; + if (request.width < 64 || request.width > 2048 || request.height < 64 || request.height > 2048 || request.width % 16 || request.height % 16) + throw std::runtime_error("Image dimensions are not supported."); + self->_runtime.image(request, [&](int step, int total) { + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_listeners) [self sendEventWithName:@"SDImageProgress" body:@{@"step":@(step), @"totalSteps":@(total), @"progress":@(total > 0 ? double(step) / total : 0)}]; + }); + }, [&](const sd_image_t &image) { + if (!image.data || (image.channel != 3 && image.channel != 4) || image.width != request.width || image.height != request.height) + throw std::runtime_error("The image engine returned invalid pixels."); + if (!OGSaveRgbPng(image, output)) throw std::runtime_error("Could not save the image."); + }, std::max(0, [input[@"previewInterval"] intValue]), [&](int step, const sd_image_t &image) { + NSString *path = [output stringByAppendingString:@".preview.png"]; + if (!OGSaveRgbPng(image, path)) return; + const int total = request.steps; + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_listeners) [self sendEventWithName:@"SDImageProgress" body:@{ + @"step":@(step), @"totalSteps":@(total), @"progress":@(total > 0 ? double(step) / total : 0), @"previewPath":path + }]; + }); + }); + if (self->_runtime.cancelled.load()) throw std::runtime_error("Image generation stopped."); + } catch (const std::exception &error) { + failure = @(error.what()); [[NSFileManager defaultManager] removeItemAtPath:output error:nil]; + } + failure = [self finishImageDiagnostics:failure]; + dispatch_async(dispatch_get_main_queue(), ^{ + self->_busy = NO; + [self restoreScreenIdleTimer]; + [[NSFileManager defaultManager] removeItemAtPath:[output stringByAppendingString:@".preview.png"] error:nil]; + if (failure) reject(@"IMAGE_FAILED", failure, nil); + else resolve(@{@"id":input[@"id"], @"imagePath":output, @"width":input[@"width"], @"height":input[@"height"], @"seed":input[@"seed"]}); + }); + } + }); + }); +} +- (void)startContinuedWork:(dispatch_block_t)work { + _continuedWork = [work copy]; + if (@available(iOS 26.0, *)) { + BGContinuedProcessingTaskRequestResources resources = BGTaskScheduler.supportedResources; + [self logLifecycle:[NSString stringWithFormat:@"supportedBackgroundResources=%lu gpu=%d", (unsigned long)resources, (resources & BGContinuedProcessingTaskRequestResourcesGPU) != 0]]; + if (resources & BGContinuedProcessingTaskRequestResourcesGPU) { + if (!_taskIdentifier) { + _taskIdentifier = [NSString stringWithFormat:@"%@.video.generation", NSBundle.mainBundle.bundleIdentifier]; + __weak VideoGenerationModule *weakSelf = self; + BOOL registered = [BGTaskScheduler.sharedScheduler registerForTaskWithIdentifier:_taskIdentifier usingQueue:dispatch_get_main_queue() launchHandler:^(BGTask *task) { + VideoGenerationModule *owner = weakSelf; + if (!owner || !owner->_busy) { [task setTaskCompletedWithSuccess:NO]; return; } + owner->_continued = task; + [owner logLifecycle:[NSString stringWithFormat:@"admitted continued GPU task %@", task.identifier]]; + __weak BGTask *expiringTask = task; + task.expirationHandler = ^{ + dispatch_async(dispatch_get_main_queue(), ^{ + VideoGenerationModule *active = weakSelf; + if (active && active->_continued == expiringTask && active->_busy) { + [active logLifecycle:[NSString stringWithFormat:@"continued GPU task expired %@", expiringTask.identifier]]; + [active interruptVideo:@"iOS stopped background video generation. Keep Off Grid open and try again."]; + } + }); + }; + dispatch_block_t admitted = owner->_continuedWork; owner->_continuedWork = nil; + if (admitted) admitted(); + }]; + [self logLifecycle:[NSString stringWithFormat:@"registration %@ success=%d", _taskIdentifier, registered]]; + if (!registered) { _taskIdentifier = nil; _continuedWork = nil; work(); return; } + } + BGContinuedProcessingTaskRequest *request = [[BGContinuedProcessingTaskRequest alloc] initWithIdentifier:_taskIdentifier title:@"Generating video" subtitle:@"Loading model"]; + request.requiredResources = BGContinuedProcessingTaskRequestResourcesGPU; + request.strategy = BGContinuedProcessingTaskRequestSubmissionStrategyFail; + NSError *submissionError = nil; + if ([BGTaskScheduler.sharedScheduler submitTaskRequest:request error:&submissionError]) { + [self logLifecycle:[NSString stringWithFormat:@"submitted continued GPU task %@", _taskIdentifier]]; + return; + } + [self logLifecycle:[NSString stringWithFormat:@"continued GPU task refused domain=%@ code=%ld detail=%@", submissionError.domain, (long)submissionError.code, submissionError.localizedDescription]]; + } + } + [self logLifecycle:[NSString stringWithFormat:@"running in foreground without admitted background GPU access"]]; + _continuedWork = nil; + work(); +} +@end diff --git a/native/video/common/VideoRuntime.hpp b/native/video/common/VideoRuntime.hpp new file mode 100644 index 000000000..5ead6408d --- /dev/null +++ b/native/video/common/VideoRuntime.hpp @@ -0,0 +1,349 @@ +#pragma once +#ifdef __APPLE__ +#include +#else +#include "stable-diffusion.h" +#include "ggml-backend.h" +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +namespace offgrid { +struct VideoRequest { + std::string weight, vae, encoder, prompt, negative; + int width, height, frames, fps, steps; + float guidance; + int64_t seed; + std::string llm, embeddings, audioVae; + float flowShift = 0; + bool cpuOnly = false; + bool skipNpu = false; + int threads = 4; + std::string imageFamily, imageSampler, imageScheduler; + // Optional start image (Animate this photo, Continue): RGB, width x height x 3 of this request. + std::vector initImage; +}; +// One instance per process. Both host bridges use the same native lifecycle. +class VideoRuntime { + std::mutex contextMutex; + std::mutex executionMutex; + sd_ctx_t *context = nullptr; + sd_ctx_t *imageContext = nullptr; + std::string imagePath; + // What the image context was loaded from and on, so a generation that fails on an accelerated + // backend can reload one step down and try once more. + VideoRequest imageLoad; + enum class ImagePlace { npu, gpu, cpu } imagePlace = ImagePlace::gpu; + // Whether the last loadContext placed the model on the CPU (by request, by rule or as its fallback). + bool lastLoadOnCpu = false; + sample_method_t imageSampler = EULER_SAMPLE_METHOD; + scheduler_t imageScheduler = SCHEDULER_COUNT; + sd_ctx_t *loadContext(const VideoRequest &request, std::string &preferred, bool image = false) { + sd_ctx_params_t config; + sd_ctx_params_init(&config); + lastLoadOnCpu = false; + if (request.imageFamily == "checkpoint") config.model_path = request.weight.c_str(); + else config.diffusion_model_path = request.weight.c_str(); + config.vae_path = request.vae.c_str(); + config.t5xxl_path = request.encoder.c_str(); + config.llm_path = request.llm.c_str(); + config.embeddings_connectors_path = request.embeddings.c_str(); + config.audio_vae_path = request.audioVae.c_str(); + config.enable_mmap = true; + config.diffusion_flash_attn = true; + config.auto_fit = true; + config.eager_load = false; +#ifdef __ANDROID__ + // Probe the NPU before choosing it. Devices without a usable Hexagon + // runtime retain Vulkan diffusion and the CPU conditioning fallback. + sd_list_devices(nullptr, 0); + const auto gpuDevice = ggml_backend_dev_by_name("Vulkan0"); + const std::string gpuDescription = gpuDevice ? ggml_backend_dev_description(gpuDevice) : ""; + const std::string vaeName = request.vae.substr(request.vae.find_last_of("/\\") + 1); + const bool wanVae = vaeName == "wan_2.1_vae.safetensors" || vaeName == "wan2.2_vae.safetensors"; + const bool adrenoWan = wanVae && gpuDescription.find("Adreno") != std::string::npos; + // The Hexagon backend returns a constant Wan DiT output: it ignores the + // latent, timestep and prompt, so sampling leaves the start noise intact. + // Adreno Vulkan crashes in the same DiT graph. Wan diffusion runs on CPU. + const bool wanCpu = adrenoWan || (wanVae && !gpuDevice); + // Adreno Vulkan also crashes in the SD/SDXL UNet graph (SIGSEGV in ggml_vk_build_graph, + // ggml-vulkan.cpp:12252; OnePlus SM8635, RealVisXL Lightning Q4, 2026-10-27), and a crash ends the + // process before any fallback can run. SD images on Adreno run on the CPU, as Wan diffusion does. + const bool imageCpu = image && gpuDescription.find("Adreno") != std::string::npos; + // An SD/SDXL image UNet does not fit the HTP0 cap, so the scheduler splits it across HTP0 and + // Vulkan0, and Hexagon cannot compute on a view Vulkan holds ("HTP0 cannot execute MUL: tensor + // (view) uses Vulkan0 buffer"; OnePlus SM8635, RealVisXL Lightning Q4, 2026-10-27). No sd.cpp + // image has generated on HTP0, so images skip the NPU, as Wan does. + const bool skipNpu = request.skipNpu || wanVae || image; + + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + auto device = ggml_backend_dev_get(i); + const char *name = ggml_backend_dev_name(device); + if (request.cpuOnly || skipNpu || std::string(name).rfind("HTP", 0) != 0) continue; + try { + auto probe = ggml_backend_dev_init(device, nullptr); + if (!probe) continue; + preferred = name; + ggml_backend_free(probe); + break; + } catch (const std::exception &) { + // An installed driver can still reject the DSP session. + } + } + // Wan decode still loses the Adreno device between spatial tiles after + // the matvec compiler workaround. Route this decoder to CPU before it can + // invalidate the GPU context. + const std::string vaeBackend = gpuDevice && !adrenoWan ? ",vae=Vulkan0" : ",vae=cpu"; + const std::string npuBackend = preferred + vaeBackend; + const std::string gpuBackend = "Vulkan0,te=cpu" + vaeBackend; + config.backend = preferred.empty() ? gpuBackend.c_str() : npuBackend.c_str(); + // Stream NPU weights from disk per graph segment. Keeping a second + // complete copy on Vulkan retains the text encoder during diffusion and + // can trigger Android's low-memory killer. The graph cap still controls + // staging even though explicit disk placement disables auto-fit. + if (!preferred.empty()) { + config.params_backend = "disk"; + config.auto_fit = false; + } + config.max_vram = "HTP0=1.5"; + config.disable_prefetch = true; +#endif + config.n_threads = request.threads; +#ifdef __ANDROID__ + if (request.cpuOnly || wanCpu || imageCpu) { +#else + if (request.cpuOnly) { +#endif + config.backend = "cpu"; config.params_backend = nullptr; config.auto_fit = true; + lastLoadOnCpu = true; +#ifdef __ANDROID__ + preferred.clear(); +#endif + } + sd_ctx_t *loaded = nullptr; +#ifdef __ANDROID__ + try { loaded = new_sd_ctx(&config); } + catch (const std::exception &) { + if (request.cpuOnly || cancelled.load()) throw; + } + if (!loaded && !preferred.empty() && !cancelled.load()) { + preferred.clear(); + config.params_backend = nullptr; + config.auto_fit = true; + config.backend = gpuBackend.c_str(); + try { loaded = new_sd_ctx(&config); } + catch (const std::exception &) { + if (cancelled.load()) throw; + } + } + if (!loaded && !cancelled.load()) { + config.backend = "cpu"; + lastLoadOnCpu = true; + loaded = new_sd_ctx(&config); + } +#else + loaded = new_sd_ctx(&config); +#endif + return loaded; + } +public: + ~VideoRuntime() { + if (context) free_sd_ctx(context); + if (imageContext) free_sd_ctx(imageContext); + } + std::atomic_bool cancelled{false}; + void cancel() { + cancelled.store(true); + std::lock_guard guard(contextMutex); + if (context) sd_cancel_generation(context, SD_CANCEL_ALL); + if (imageContext) sd_cancel_generation(imageContext, SD_CANCEL_ALL); + } + void loadImage(const VideoRequest &request, const std::string &path) { + std::unique_lock execution(executionMutex, std::try_to_lock); + if (!execution.owns_lock()) throw std::runtime_error("Image or video generation is running."); + // Packs (a diffusion transformer, its LLM text encoder and VAE) load the same way; FLUX 3 + // Klein needs a newer engine than this one (stable-diffusion.cpp 8cd731e). + static const std::set families{"checkpoint", "qwen-image-2.1", "z-image", "flux2-klein"}; + if (!request.imageFamily.empty() && !families.count(request.imageFamily)) + throw std::runtime_error("This image model family is not supported."); + const auto sampler = request.imageSampler.empty() ? EULER_SAMPLE_METHOD : str_to_sample_method(request.imageSampler.c_str()); + const auto scheduler = request.imageScheduler.empty() ? SCHEDULER_COUNT : str_to_scheduler(request.imageScheduler.c_str()); + if (sampler == SAMPLE_METHOD_COUNT || (!request.imageScheduler.empty() && scheduler == SCHEDULER_COUNT)) + throw std::runtime_error("The image sampler or scheduler is not supported."); + { + std::lock_guard guard(contextMutex); + if (imageContext) free_sd_ctx(imageContext); + imageContext = nullptr; imagePath.clear(); + } + std::string preferred; + auto loaded = loadContext(request, preferred, true); + if (!loaded) throw std::runtime_error("Could not load the image model pack."); + std::lock_guard guard(contextMutex); + if (cancelled.load()) { free_sd_ctx(loaded); throw std::runtime_error("Image loading stopped."); } + imageContext = loaded; imagePath = path; + imageLoad = request; + imagePlace = !preferred.empty() ? ImagePlace::npu : lastLoadOnCpu ? ImagePlace::cpu : ImagePlace::gpu; + imageSampler = sampler; + imageScheduler = scheduler == SCHEDULER_COUNT ? sd_get_default_scheduler(loaded, sampler) : scheduler; + } + void unloadImage() { + std::unique_lock execution(executionMutex, std::try_to_lock); + if (!execution.owns_lock()) throw std::runtime_error("Image or video generation is running."); + std::lock_guard guard(contextMutex); + if (imageContext) free_sd_ctx(imageContext); + imageContext = nullptr; imagePath.clear(); + } + std::string loadedImagePath() { + std::lock_guard guard(contextMutex); + return imageContext ? imagePath : ""; + } + void image(const VideoRequest &request, + const std::function &progress, + const std::function &save, + int previewInterval = 0, + const std::function &preview = {}) { + std::unique_lock execution(executionMutex, std::try_to_lock); + if (!execution.owns_lock()) throw std::runtime_error("Image or video generation is running."); + if (!imageContext) throw std::runtime_error("Image model is unloaded."); + sd_image_t *images = nullptr; int count = 0; + auto cleanup = [&] { + sd_set_progress_callback(nullptr, nullptr); + sd_set_preview_callback(nullptr, PREVIEW_NONE, 0, false, false, nullptr); + if (images) free_sd_images(images, count); + }; + try { + if (cancelled.load()) throw std::runtime_error("Image generation stopped."); + sd_set_progress_callback([](int step, int total, float, void *data) { + (*static_cast *>(data))(step, total); + }, const_cast(static_cast(&progress))); + if (preview && previewInterval > 0) { + // Project the existing latent tensor. Do not run an extra VAE decode. + sd_set_preview_callback([](int step, int count, sd_image_t* frames, bool, void* data) { + if (count < 1 || !frames || !frames[0].data) return; + try { (*static_cast*>(data))(step, frames[0]); } + catch (...) { /* Optional previews must not fail generation. */ } + }, PREVIEW_PROJ, previewInterval, true, false, + const_cast(static_cast(&preview))); + } + sd_img_gen_params_t params; sd_img_gen_params_init(¶ms); + params.prompt = request.prompt.c_str(); params.negative_prompt = request.negative.c_str(); + params.width = request.width; params.height = request.height; params.seed = request.seed; + params.batch_count = 1; + params.sample_params.sample_steps = request.steps; + params.sample_params.guidance.txt_cfg = request.guidance; + params.sample_params.sample_method = imageSampler; + params.sample_params.scheduler = imageScheduler; + params.vae_tiling_params.enabled = true; + bool made = generate_image(imageContext, ¶ms, &images, &count) && images && count == 1; + // A graph that fails on an accelerated backend gets one more try one step down (NPU -> GPU, + // GPU -> CPU), as loading falls back: the person gets an image, not an error. + if (!made && imagePlace != ImagePlace::cpu && !cancelled.load()) { + if (images) { free_sd_images(images, count); images = nullptr; count = 0; } + VideoRequest retry = imageLoad; + if (imagePlace == ImagePlace::npu) retry.skipNpu = true; else retry.cpuOnly = true; + std::string preferred; + sd_ctx_t *reloaded = loadContext(retry, preferred, true); + if (reloaded) { + std::lock_guard guard(contextMutex); + free_sd_ctx(imageContext); + imageContext = reloaded; + imageLoad = retry; + imagePlace = lastLoadOnCpu ? ImagePlace::cpu : ImagePlace::gpu; + made = generate_image(imageContext, ¶ms, &images, &count) && images && count == 1; + } + } + if (!made) throw std::runtime_error("The image engine produced no image."); + if (cancelled.load()) throw std::runtime_error("Image generation stopped."); + save(images[0]); + } catch (...) { cleanup(); throw; } + cleanup(); + } + void run(const VideoRequest &request, + const std::function &progress, + const std::function &encode, + const std::function &conditioning, + const std::function &decodeProgress, + const std::function &preview) { + std::unique_lock execution(executionMutex, std::try_to_lock); + if (!execution.owns_lock()) throw std::runtime_error("Video generation is already running."); + sd_image_t *frames = nullptr; + int count = 0, fps = request.fps; + struct DecodeObserver { + const std::function &progress; + const std::function &frame; + } observer{decodeProgress, preview}; + auto cleanup = [&] { + if (frames) free_sd_images(frames, count); + sd_set_video_decode_callback(nullptr, nullptr, nullptr); + sd_set_progress_callback(nullptr, nullptr); + std::lock_guard guard(contextMutex); + if (context) free_sd_ctx(context); + context = nullptr; + }; + try { + if (cancelled.load()) throw std::runtime_error("Video generation stopped."); + std::string preferred; + sd_ctx_t *loaded = loadContext(request, preferred); + { + std::lock_guard guard(contextMutex); + context = loaded; + } + if (!loaded) throw std::runtime_error("Could not load the video model pack."); + if (cancelled.load()) throw std::runtime_error("Video generation stopped."); + sd_set_progress_callback([](int step, int total, float, void *data) { + (*static_cast *>(data))(step, total); + }, const_cast(static_cast(&progress))); + sd_set_video_decode_callback([](int completed, int total, void *data) { + static_cast(data)->progress(completed, total); + }, [](const sd_image_t *image, void *data) { + try { static_cast(data)->frame(*image); } + catch (...) { /* The optional preview must not fail generation. */ } + }, &observer); + sd_vid_gen_params_t params; + sd_vid_gen_params_init(¶ms); + params.prompt = request.prompt.c_str(); params.negative_prompt = request.negative.c_str(); + params.width = request.width; params.height = request.height; + params.video_frames = request.frames; params.fps = request.fps; params.seed = request.seed; + params.sample_params.sample_steps = request.steps; + params.sample_params.guidance.txt_cfg = request.guidance; + params.sample_params.flow_shift = request.flowShift > 0 ? request.flowShift : std::numeric_limits::infinity(); + params.sample_params.sample_method = EULER_SAMPLE_METHOD; + params.sample_params.scheduler = sd_get_default_scheduler(loaded, EULER_SAMPLE_METHOD); + params.vae_tiling_params.enabled = true; + if (request.initImage.size() == static_cast(request.width) * request.height * 3) + params.init_image = {static_cast(request.width), static_cast(request.height), 3, + const_cast(request.initImage.data())}; +#ifdef __ANDROID__ + // Mobile Vulkan drivers can lose the device on a full temporal decode. + // Keep each decode graph small instead of waiting for allocation failure: + // a lost GPU cannot recover through the allocator's tiling retry. + params.vae_tiling_params.temporal_tiling = true; + params.vae_tiling_params.tile_size_w = 128; + params.vae_tiling_params.tile_size_h = 128; + params.vae_tiling_params.extra_tiling_args = "temporal_tile_frames=1"; + conditioning(preferred.empty() ? "cpu" : "npu"); +#else + // Wan's stateful decoder retains causal feature caches between temporal + // chunks. Bound the graph without dropping frames or changing resolution. + const std::string vaeName = request.vae.substr(request.vae.find_last_of("/\\") + 1); + if (vaeName == "wan_2.1_vae.safetensors" || vaeName == "wan2.2_vae.safetensors") { + params.vae_tiling_params.temporal_tiling = true; + params.vae_tiling_params.extra_tiling_args = "temporal_tile_frames=1"; + } + conditioning(""); +#endif + if (!generate_video(loaded, ¶ms, &frames, &count, nullptr, &fps) || !frames || count == 0) + throw std::runtime_error(cancelled.load() ? "Video generation stopped." : "The video engine produced no frames."); + if (cancelled.load()) throw std::runtime_error("Video generation stopped."); + encode(frames, count, fps); + } catch (...) { cleanup(); throw; } + cleanup(); + } +}; +} diff --git a/package-lock.json b/package-lock.json index c02622daf..013e50cd8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "llama.rn": "0.13.0-rc.7", "node-html-parser": "^7.1.0", "patch-package": "^8.0.1", + "posthog-react-native": "^4.81.10", "react": "19.2.0", "react-compiler-runtime": "^1.0.0", "react-native": "0.83.1", @@ -59,6 +60,8 @@ "react-native-tcp-socket": "^6.4.1", "react-native-url-polyfill": "^3.0.0", "react-native-vector-icons": "^10.3.0", + "react-native-video": "^6.19.3", + "react-native-video-player": "0.16.3", "react-native-vision-camera": "^4.7.3", "react-native-worklets": "^0.8.3", "react-native-zeroconf": "^0.14.0", @@ -143,6 +146,7 @@ "dependencies": { "@noble/hashes": "1.8.0", "@offgrid/automation": "*", + "@offgrid/models": "*", "bonjour-service": "^1.2.1", "js-sha512": "^0.9.0", "tweetnacl": "^1.0.3", @@ -5517,6 +5521,21 @@ "url": "https://opencollective.com/pkgr" } }, + "node_modules/@posthog/core": { + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.60.0.tgz", + "integrity": "sha512-d8jmcvJPmgjkcW0kVbIkDXcD3jJ+y/wS6oA4PeEGL914Rj8iM9HfB6VWvMWc6W57UJwpI9+L0hFJkizraVBA1Q==", + "license": "MIT", + "dependencies": { + "@posthog/types": "^1.416.0" + } + }, + "node_modules/@posthog/types": { + "version": "1.416.0", + "resolved": "https://registry.npmjs.org/@posthog/types/-/types-1.416.0.tgz", + "integrity": "sha512-0/9+02s/0uDyrH4G/qUg1yyl6gXUg8Uuxw/PlDad3KLzo9FFS9zS/kAQaEshx/saVGupj1aAy/4EUBxWisBogw==", + "license": "MIT" + }, "node_modules/@react-native-async-storage/async-storage": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@react-native-async-storage/async-storage/-/async-storage-2.2.0.tgz", @@ -14719,6 +14738,79 @@ "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", "license": "MIT" }, + "node_modules/posthog-react-native": { + "version": "4.81.10", + "resolved": "https://registry.npmjs.org/posthog-react-native/-/posthog-react-native-4.81.10.tgz", + "integrity": "sha512-vUX1uHDwOJB7EJ/j8N105A9lQ5xdiSjpaonWyqd6rTdDNjUsuqApKGDWQEfhCMhJfeVmdNM2sM65gnXkf3No3A==", + "license": "MIT", + "dependencies": { + "@posthog/core": "^1.59.4", + "@posthog/types": "^1.416.0" + }, + "bin": { + "posthog-react-native-xcode": "tooling/posthog-xcode.sh" + }, + "peerDependencies": { + "@posthog/react-native-plugin": ">= 2.9.3", + "@react-native-async-storage/async-storage": ">=1.0.0", + "@react-navigation/native": ">= 5.0.0", + "expo-application": ">= 4.0.0", + "expo-device": ">= 4.0.0", + "expo-file-system": ">= 13.0.0", + "expo-localization": ">= 11.0.0", + "expo-updates": ">= 0.25.0", + "posthog-react-native-session-replay": ">= 1.6.0", + "react-native-device-info": ">= 10.0.0", + "react-native-localize": ">= 3.0.0", + "react-native-navigation": ">= 6.0.0", + "react-native-safe-area-context": ">= 4.0.0", + "react-native-svg": ">= 15.0.0" + }, + "peerDependenciesMeta": { + "@posthog/react-native-plugin": { + "optional": true + }, + "@react-native-async-storage/async-storage": { + "optional": true + }, + "@react-navigation/native": { + "optional": true + }, + "expo-application": { + "optional": true + }, + "expo-device": { + "optional": true + }, + "expo-file-system": { + "optional": true + }, + "expo-localization": { + "optional": true + }, + "expo-updates": { + "optional": true + }, + "posthog-react-native-session-replay": { + "optional": true + }, + "react-native-device-info": { + "optional": true + }, + "react-native-localize": { + "optional": true + }, + "react-native-navigation": { + "optional": true + }, + "react-native-safe-area-context": { + "optional": true + }, + "react-native-svg": { + "optional": true + } + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -15571,6 +15663,31 @@ "node": ">=10" } }, + "node_modules/react-native-video": { + "version": "6.19.3", + "resolved": "https://registry.npmjs.org/react-native-video/-/react-native-video-6.19.3.tgz", + "integrity": "sha512-hI0YhEH6NdkfbFfMoZTZizEmLIm13rLlMf6KyWlGSe0+6If7/gvhapR2j/BMKBEwmlAIZjU4A6ax4Qn7yssjLQ==", + "license": "MIT", + "peerDependencies": { + "react": "*", + "react-native": "*" + } + }, + "node_modules/react-native-video-player": { + "version": "0.16.3", + "resolved": "https://registry.npmjs.org/react-native-video-player/-/react-native-video-player-0.16.3.tgz", + "integrity": "sha512-7FMPl2uO9HRkklAaAGqtuAx+8qphbluacWTxyOhV19AOLc0GC4hkD0FrQoj6ksmgrwFJEXIkBlWkkfu1GS1YyA==", + "license": "MIT", + "workspaces": [ + "example", + "docs" + ], + "peerDependencies": { + "react": "*", + "react-native": "*", + "react-native-video": "^6.8.0" + } + }, "node_modules/react-native-vision-camera": { "version": "4.7.3", "resolved": "https://registry.npmjs.org/react-native-vision-camera/-/react-native-vision-camera-4.7.3.tgz", @@ -17413,7 +17530,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/package.json b/package.json index 2c0a21430..2406abd99 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "llama.rn": "0.13.0-rc.7", "node-html-parser": "^7.1.0", "patch-package": "^8.0.1", + "posthog-react-native": "^4.81.10", "react": "19.2.0", "react-compiler-runtime": "^1.0.0", "react-native": "0.83.1", @@ -87,6 +88,8 @@ "react-native-tcp-socket": "^6.4.1", "react-native-url-polyfill": "^3.0.0", "react-native-vector-icons": "^10.3.0", + "react-native-video": "^6.19.3", + "react-native-video-player": "0.16.3", "react-native-vision-camera": "^4.7.3", "react-native-worklets": "^0.8.3", "react-native-zeroconf": "^0.14.0", diff --git a/patches/react-native-video-player+0.16.3.patch b/patches/react-native-video-player+0.16.3.patch new file mode 100644 index 000000000..f799f32bd --- /dev/null +++ b/patches/react-native-video-player+0.16.3.patch @@ -0,0 +1,448 @@ +diff --git a/node_modules/react-native-video-player/lib/commonjs/Video.js b/node_modules/react-native-video-player/lib/commonjs/Video.js +index 9ff315d..181c87a 100644 +--- a/node_modules/react-native-video-player/lib/commonjs/Video.js ++++ b/node_modules/react-native-video-player/lib/commonjs/Video.js +@@ -19,6 +19,8 @@ const RenderVideo = exports.RenderVideo = /*#__PURE__*/(0, _react.memo)(/*#__PUR + defaultMuted, + disableControlsAutoHide, + disableFullscreen, ++ disableMute, ++ additionalControl, + disableSeek, + fullScreenOnLongPress, + hideControlsOnStart = false, +@@ -123,7 +125,8 @@ const RenderVideo = exports.RenderVideo = /*#__PURE__*/(0, _react.memo)(/*#__PUR + const _onLoad = (0, _react.useCallback)(event => { + if (onLoad) onLoad(event); + setDuration(event.duration); +- }, [onLoad]); ++ if (!hideControlsOnStart) _showControls(); ++ }, [onLoad, hideControlsOnStart, _showControls]); + const _onPlayPress = (0, _react.useCallback)(() => { + if (onPlayPress) onPlayPress(); + setIsPlaying(prev => !prev); +@@ -179,6 +182,8 @@ const RenderVideo = exports.RenderVideo = /*#__PURE__*/(0, _react.memo)(/*#__PUR + customStyles: customStyles, + showDuration: showDuration, + disableFullscreen: disableFullscreen, ++ disableMute: disableMute, ++ additionalControl: additionalControl, + duration: props.duration || duration, + isPlaying: isPlaying, + isMuted: isMuted, +diff --git a/node_modules/react-native-video-player/lib/commonjs/controls/Controls.js b/node_modules/react-native-video-player/lib/commonjs/controls/Controls.js +index bb476fb..54ba537 100644 +--- a/node_modules/react-native-video-player/lib/commonjs/controls/Controls.js ++++ b/node_modules/react-native-video-player/lib/commonjs/controls/Controls.js +@@ -20,6 +20,8 @@ const Controls = exports.Controls = /*#__PURE__*/(0, _react.forwardRef)(({ + onToggleFullScreen, + duration, + disableFullscreen, ++ disableMute, ++ additionalControl, + onMutePress, + isMuted, + animationDuration, +@@ -63,6 +65,7 @@ const Controls = exports.Controls = /*#__PURE__*/(0, _react.forwardRef)(({ + children: [/*#__PURE__*/(0, _jsxRuntime.jsx)(_PlayButton.PlayButton, { + controlButtonCustomStyles: customStyles?.controlButton, + playControlCustomStyles: customStyles?.playControl, ++ controlIconCustomStyles: customStyles?.controlIcon, + isPlaying: isPlaying, + onPlayPress: onPlayPress + }), /*#__PURE__*/(0, _jsxRuntime.jsx)(_Seekbar.Seekbar, { +@@ -83,12 +86,12 @@ const Controls = exports.Controls = /*#__PURE__*/(0, _react.forwardRef)(({ + ref: durationRef, + duration: duration, + durationTextCustomStyles: customStyles?.durationText +- }), /*#__PURE__*/(0, _jsxRuntime.jsx)(_Mute.Mute, { ++ }), !disableMute && /*#__PURE__*/(0, _jsxRuntime.jsx)(_Mute.Mute, { + controlButtonCustomStyles: customStyles?.controlButton, + controlIconCustomStyles: customStyles?.controlIcon, + isMuted: isMuted, + onMutePress: onMutePress +- }), !disableFullscreen && /*#__PURE__*/(0, _jsxRuntime.jsx)(_Fullscreen.Fullscreen, { ++ }), additionalControl, !disableFullscreen && /*#__PURE__*/(0, _jsxRuntime.jsx)(_Fullscreen.Fullscreen, { + onToggleFullScreen: onToggleFullScreen, + controlButtonCustomStyles: customStyles?.controlButton, + controlIconCustomStyles: customStyles?.controlIcon +diff --git a/node_modules/react-native-video-player/lib/commonjs/controls/PlayButton.js b/node_modules/react-native-video-player/lib/commonjs/controls/PlayButton.js +index 2c49655..340f714 100644 +--- a/node_modules/react-native-video-player/lib/commonjs/controls/PlayButton.js ++++ b/node_modules/react-native-video-player/lib/commonjs/controls/PlayButton.js +@@ -9,14 +9,18 @@ var _react = require("react"); + var _jsxRuntime = require("react/jsx-runtime"); + const PlayButton = exports.PlayButton = /*#__PURE__*/(0, _react.memo)(({ + isPlaying, ++ controlIconCustomStyles, + onPlayPress, + playControlCustomStyles, + controlButtonCustomStyles + }) => { + return /*#__PURE__*/(0, _jsxRuntime.jsx)(_reactNative.TouchableOpacity, { ++ accessibilityRole: 'button', ++ accessibilityLabel: isPlaying ? 'Pause video' : 'Play video', + onPress: onPlayPress, + style: [styles.playControl, controlButtonCustomStyles, playControlCustomStyles], + children: /*#__PURE__*/(0, _jsxRuntime.jsx)(_reactNative.Image, { ++ style: controlIconCustomStyles, + source: isPlaying ? require('../img/pause.png') : require('../img/play.png') + }) + }); +diff --git a/node_modules/react-native-video-player/lib/commonjs/index.js b/node_modules/react-native-video-player/lib/commonjs/index.js +index 02eadd9..b1b30a2 100644 +--- a/node_modules/react-native-video-player/lib/commonjs/index.js ++++ b/node_modules/react-native-video-player/lib/commonjs/index.js +@@ -12,6 +12,7 @@ var _jsxRuntime = require("react/jsx-runtime"); + const VideoPlayer = /*#__PURE__*/(0, _react.forwardRef)((props, ref) => { + const { + autoplay = false, ++ preload = false, + customStyles = {}, + endThumbnail, + endWithThumbnail, +@@ -79,7 +80,7 @@ const VideoPlayer = /*#__PURE__*/(0, _react.forwardRef)((props, ref) => { + customStylesPlayButton: customStyles.playButton, + customStylesPlayArrow: customStyles.playArrow + }); +- if (!isStarted) return /*#__PURE__*/(0, _jsxRuntime.jsx)(_reactNative.View, { ++ if (!isStarted && !preload) return /*#__PURE__*/(0, _jsxRuntime.jsx)(_reactNative.View, { + style: [styles.preloadingPlaceholder, sizeStyles, style], + children: /*#__PURE__*/(0, _jsxRuntime.jsx)(_Thumbnail.StartButton, { + onStart: _onStart, +@@ -96,7 +97,7 @@ const VideoPlayer = /*#__PURE__*/(0, _react.forwardRef)((props, ref) => { + onEnd: _onEnd, + sizeStyle: sizeStyles + }); +- }, [hasEnded, endThumbnail, isStarted, thumbnail, style, sizeStyles, _onStart, customStyles, rest, autoplay, _onEnd]); ++ }, [preload, hasEnded, endThumbnail, isStarted, thumbnail, style, sizeStyles, _onStart, customStyles, rest, autoplay, _onEnd]); + return /*#__PURE__*/(0, _jsxRuntime.jsx)(_reactNative.View, { + onLayout: onLayout, + style: customStyles.wrapper, +diff --git a/node_modules/react-native-video-player/lib/module/Video.js b/node_modules/react-native-video-player/lib/module/Video.js +index d190ce5..733710b 100644 +--- a/node_modules/react-native-video-player/lib/module/Video.js ++++ b/node_modules/react-native-video-player/lib/module/Video.js +@@ -14,6 +14,8 @@ export const RenderVideo = /*#__PURE__*/memo(/*#__PURE__*/forwardRef((props, ref + defaultMuted, + disableControlsAutoHide, + disableFullscreen, ++ disableMute, ++ additionalControl, + disableSeek, + fullScreenOnLongPress, + hideControlsOnStart = false, +@@ -118,7 +120,8 @@ export const RenderVideo = /*#__PURE__*/memo(/*#__PURE__*/forwardRef((props, ref + const _onLoad = useCallback(event => { + if (onLoad) onLoad(event); + setDuration(event.duration); +- }, [onLoad]); ++ if (!hideControlsOnStart) _showControls(); ++ }, [onLoad, hideControlsOnStart, _showControls]); + const _onPlayPress = useCallback(() => { + if (onPlayPress) onPlayPress(); + setIsPlaying(prev => !prev); +@@ -174,6 +177,8 @@ export const RenderVideo = /*#__PURE__*/memo(/*#__PURE__*/forwardRef((props, ref + customStyles: customStyles, + showDuration: showDuration, + disableFullscreen: disableFullscreen, ++ disableMute: disableMute, ++ additionalControl: additionalControl, + duration: props.duration || duration, + isPlaying: isPlaying, + isMuted: isMuted, +diff --git a/node_modules/react-native-video-player/lib/module/controls/Controls.js b/node_modules/react-native-video-player/lib/module/controls/Controls.js +index 64b64f4..9ed6aa2 100644 +--- a/node_modules/react-native-video-player/lib/module/controls/Controls.js ++++ b/node_modules/react-native-video-player/lib/module/controls/Controls.js +@@ -16,6 +16,8 @@ export const Controls = /*#__PURE__*/forwardRef(({ + onToggleFullScreen, + duration, + disableFullscreen, ++ disableMute, ++ additionalControl, + onMutePress, + isMuted, + animationDuration, +@@ -59,6 +61,7 @@ export const Controls = /*#__PURE__*/forwardRef(({ + children: [/*#__PURE__*/_jsx(PlayButton, { + controlButtonCustomStyles: customStyles?.controlButton, + playControlCustomStyles: customStyles?.playControl, ++ controlIconCustomStyles: customStyles?.controlIcon, + isPlaying: isPlaying, + onPlayPress: onPlayPress + }), /*#__PURE__*/_jsx(Seekbar, { +@@ -79,12 +82,12 @@ export const Controls = /*#__PURE__*/forwardRef(({ + ref: durationRef, + duration: duration, + durationTextCustomStyles: customStyles?.durationText +- }), /*#__PURE__*/_jsx(Mute, { ++ }), !disableMute && /*#__PURE__*/_jsx(Mute, { + controlButtonCustomStyles: customStyles?.controlButton, + controlIconCustomStyles: customStyles?.controlIcon, + isMuted: isMuted, + onMutePress: onMutePress +- }), !disableFullscreen && /*#__PURE__*/_jsx(Fullscreen, { ++ }), additionalControl, !disableFullscreen && /*#__PURE__*/_jsx(Fullscreen, { + onToggleFullScreen: onToggleFullScreen, + controlButtonCustomStyles: customStyles?.controlButton, + controlIconCustomStyles: customStyles?.controlIcon +diff --git a/node_modules/react-native-video-player/lib/module/controls/PlayButton.js b/node_modules/react-native-video-player/lib/module/controls/PlayButton.js +index 1dea3ae..b177c91 100644 +--- a/node_modules/react-native-video-player/lib/module/controls/PlayButton.js ++++ b/node_modules/react-native-video-player/lib/module/controls/PlayButton.js +@@ -5,14 +5,18 @@ import { memo } from 'react'; + import { jsx as _jsx } from "react/jsx-runtime"; + export const PlayButton = /*#__PURE__*/memo(({ + isPlaying, ++ controlIconCustomStyles, + onPlayPress, + playControlCustomStyles, + controlButtonCustomStyles + }) => { + return /*#__PURE__*/_jsx(TouchableOpacity, { ++ accessibilityRole: 'button', ++ accessibilityLabel: isPlaying ? 'Pause video' : 'Play video', + onPress: onPlayPress, + style: [styles.playControl, controlButtonCustomStyles, playControlCustomStyles], + children: /*#__PURE__*/_jsx(Image, { ++ style: controlIconCustomStyles, + source: isPlaying ? require('../img/pause.png') : require('../img/play.png') + }) + }); +diff --git a/node_modules/react-native-video-player/lib/module/index.js b/node_modules/react-native-video-player/lib/module/index.js +index 5fc06ec..83e65b1 100644 +--- a/node_modules/react-native-video-player/lib/module/index.js ++++ b/node_modules/react-native-video-player/lib/module/index.js +@@ -8,6 +8,7 @@ import { jsx as _jsx } from "react/jsx-runtime"; + const VideoPlayer = /*#__PURE__*/forwardRef((props, ref) => { + const { + autoplay = false, ++ preload = false, + customStyles = {}, + endThumbnail, + endWithThumbnail, +@@ -75,7 +76,7 @@ const VideoPlayer = /*#__PURE__*/forwardRef((props, ref) => { + customStylesPlayButton: customStyles.playButton, + customStylesPlayArrow: customStyles.playArrow + }); +- if (!isStarted) return /*#__PURE__*/_jsx(View, { ++ if (!isStarted && !preload) return /*#__PURE__*/_jsx(View, { + style: [styles.preloadingPlaceholder, sizeStyles, style], + children: /*#__PURE__*/_jsx(StartButton, { + onStart: _onStart, +@@ -92,7 +93,7 @@ const VideoPlayer = /*#__PURE__*/forwardRef((props, ref) => { + onEnd: _onEnd, + sizeStyle: sizeStyles + }); +- }, [hasEnded, endThumbnail, isStarted, thumbnail, style, sizeStyles, _onStart, customStyles, rest, autoplay, _onEnd]); ++ }, [preload, hasEnded, endThumbnail, isStarted, thumbnail, style, sizeStyles, _onStart, customStyles, rest, autoplay, _onEnd]); + return /*#__PURE__*/_jsx(View, { + onLayout: onLayout, + style: customStyles.wrapper, +diff --git a/node_modules/react-native-video-player/lib/typescript/commonjs/src/index.d.ts b/node_modules/react-native-video-player/lib/typescript/commonjs/src/index.d.ts +index a5eb66e..8b79e74 100644 +--- a/node_modules/react-native-video-player/lib/typescript/commonjs/src/index.d.ts ++++ b/node_modules/react-native-video-player/lib/typescript/commonjs/src/index.d.ts +@@ -27,11 +27,14 @@ export interface CustomStyles { + export interface VideoPlayerProps extends ReactVideoProps { + animationDuration?: number; + autoplay?: boolean; ++ preload?: boolean; + controlsTimeout?: number; + customStyles?: CustomStyles; + defaultMuted?: boolean; + disableControlsAutoHide?: boolean; + disableFullscreen?: boolean; ++ disableMute?: boolean; ++ additionalControl?: React.ReactNode; + disableSeek?: boolean; + duration?: number; + endThumbnail?: ImageSourcePropType; +diff --git a/node_modules/react-native-video-player/lib/typescript/module/src/index.d.ts b/node_modules/react-native-video-player/lib/typescript/module/src/index.d.ts +index a5eb66e..8b79e74 100644 +--- a/node_modules/react-native-video-player/lib/typescript/module/src/index.d.ts ++++ b/node_modules/react-native-video-player/lib/typescript/module/src/index.d.ts +@@ -27,11 +27,14 @@ export interface CustomStyles { + export interface VideoPlayerProps extends ReactVideoProps { + animationDuration?: number; + autoplay?: boolean; ++ preload?: boolean; + controlsTimeout?: number; + customStyles?: CustomStyles; + defaultMuted?: boolean; + disableControlsAutoHide?: boolean; + disableFullscreen?: boolean; ++ disableMute?: boolean; ++ additionalControl?: React.ReactNode; + disableSeek?: boolean; + duration?: number; + endThumbnail?: ImageSourcePropType; +diff --git a/node_modules/react-native-video-player/src/Video.tsx b/node_modules/react-native-video-player/src/Video.tsx +index 191e724..1d0484b 100644 +--- a/node_modules/react-native-video-player/src/Video.tsx ++++ b/node_modules/react-native-video-player/src/Video.tsx +@@ -31,6 +31,8 @@ type RenderVideoProps = Pick< + | 'defaultMuted' + | 'disableControlsAutoHide' + | 'disableFullscreen' ++ | 'disableMute' ++ | 'additionalControl' + | 'disableSeek' + | 'duration' + | 'fullScreenOnLongPress' +@@ -63,6 +65,8 @@ export const RenderVideo = memo( + defaultMuted, + disableControlsAutoHide, + disableFullscreen, ++ disableMute, ++ additionalControl, + disableSeek, + fullScreenOnLongPress, + hideControlsOnStart = false, +@@ -190,8 +194,9 @@ export const RenderVideo = memo( + (event: OnLoadData) => { + if (onLoad) onLoad(event); + setDuration(event.duration); ++ if (!hideControlsOnStart) _showControls(); + }, +- [onLoad] ++ [onLoad, hideControlsOnStart, _showControls] + ); + + const _onPlayPress = useCallback(() => { +@@ -262,6 +267,8 @@ export const RenderVideo = memo( + customStyles={customStyles} + showDuration={showDuration} + disableFullscreen={disableFullscreen} ++ disableMute={disableMute} ++ additionalControl={additionalControl} + duration={props.duration || duration} + isPlaying={isPlaying} + isMuted={isMuted} +diff --git a/node_modules/react-native-video-player/src/controls/Controls.tsx b/node_modules/react-native-video-player/src/controls/Controls.tsx +index 7ac7e72..eca3620 100644 +--- a/node_modules/react-native-video-player/src/controls/Controls.tsx ++++ b/node_modules/react-native-video-player/src/controls/Controls.tsx +@@ -11,6 +11,8 @@ interface ControlsProps + extends Omit { + customStyles: VideoPlayerProps['customStyles']; + showDuration: VideoPlayerProps['showDuration']; ++ disableMute?: boolean; ++ additionalControl?: React.ReactNode; + disableFullscreen: VideoPlayerProps['disableFullscreen']; + animationDuration: number; + duration: number; +@@ -36,6 +38,8 @@ export const Controls = forwardRef( + onToggleFullScreen, + duration, + disableFullscreen, ++ disableMute, ++ additionalControl, + onMutePress, + isMuted, + animationDuration, +@@ -88,6 +92,7 @@ export const Controls = forwardRef( + +@@ -113,12 +118,13 @@ export const Controls = forwardRef( + durationTextCustomStyles={customStyles?.durationText} + /> + )} +- ++ />} ++ {additionalControl} + {!disableFullscreen && ( + void; + } +@@ -12,12 +13,15 @@ interface PlayButtonProps { + export const PlayButton = memo( + ({ + isPlaying, ++ controlIconCustomStyles, + onPlayPress, + playControlCustomStyles, + controlButtonCustomStyles, + }: PlayButtonProps) => { + return ( + + ( + (props, ref) => { + const { + autoplay = false, ++ preload = false, + customStyles = {}, + endThumbnail, + endWithThumbnail, +@@ -167,7 +171,7 @@ const VideoPlayer = forwardRef( + customStylesPlayArrow={customStyles.playArrow} + /> + ); +- if (!isStarted) ++ if (!isStarted && !preload) + return ( + + ( + /> + ); + }, [ ++ preload, + hasEnded, + endThumbnail, + isStarted, diff --git a/pro b/pro index 752337cb9..14d3415b6 160000 --- a/pro +++ b/pro @@ -1 +1 @@ -Subproject commit 752337cb94b18f20464a02342e57540f9cdbc3b8 +Subproject commit 14d3415b6f1227dffc47179f19e2208d1e922f9c diff --git a/react-native.config.js b/react-native.config.js new file mode 100644 index 000000000..74051d6df --- /dev/null +++ b/react-native.config.js @@ -0,0 +1,11 @@ +const fs = require('node:fs'); +const path = require('node:path'); + +// Pro's native modules (Video Studio film join) live in the private pro/ submodule. Autolink them +// only when pro/ is checked out, so the open-source build compiles and runs without them. +const filmRoot = path.resolve(__dirname, 'pro/native/film'); +const proExists = fs.existsSync(path.resolve(__dirname, 'pro/package.json')); + +module.exports = { + dependencies: proExists && fs.existsSync(filmRoot) ? { '@offgrid/film-native': { root: filmRoot } } : {}, +}; diff --git a/scripts/build-video-ios.sh b/scripts/build-video-ios.sh new file mode 100755 index 000000000..3670806ba --- /dev/null +++ b/scripts/build-video-ios.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +REVISION="$(cat "$ROOT/scripts/video/revision")" +CACHE="${OFFGRID_VIDEO_BUILD_DIR:-$ROOT/.video-build}" +SOURCE="${OFFGRID_SD_SOURCE:-$CACHE/source}" +OUTPUT="$ROOT/native/video/OffgridVideoRuntime.xcframework" +BUILD_KEY="${OFFGRID_IOS_DEVICE_ONLY:-0}-$(cat "$ROOT/scripts/video/revision" "$ROOT/scripts/video/exports.txt" "$ROOT/scripts/video/CMakeLists.txt" "$ROOT/scripts/video/conditioning-errors.patch" "$ROOT/scripts/video/video-decode-observer.patch" | shasum -a 256 | cut -d' ' -f1)" +if [ -f "$OUTPUT/ios-arm64/OffgridVideoRuntime.framework/Headers/stable-diffusion.h" ] && [ -f "$OUTPUT/revision" ] && [ "$(cat "$OUTPUT/revision")" = "$BUILD_KEY" ]; then exit 0; fi +mkdir -p "$CACHE" +if [ ! -d "$SOURCE/.git" ]; then + git clone --filter=blob:none --no-checkout https://github.com/leejet/stable-diffusion.cpp.git "$SOURCE" +fi +if [ "$(git -C "$SOURCE" rev-parse HEAD)" != "$REVISION" ]; then + git -C "$SOURCE" fetch origin "$REVISION" + git -C "$SOURCE" checkout --detach "$REVISION" +fi +git -C "$SOURCE" submodule update --init --depth 1 ggml +for PATCH in "$ROOT/scripts/video/conditioning-errors.patch" "$ROOT/scripts/video/video-decode-observer.patch"; do + if ! git -C "$SOURCE" apply --reverse --check "$PATCH" 2>/dev/null; then + git -C "$SOURCE" apply "$PATCH" + fi +done +SDKS=(iphoneos) +if [ "${OFFGRID_IOS_DEVICE_ONLY:-0}" != "1" ]; then SDKS+=(iphonesimulator); fi +for SDK in "${SDKS[@]}"; do + cmake -S "$ROOT/scripts/video" -B "$CACHE/$SDK" -G Xcode \ + -DSD_SOURCE="$SOURCE" -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_SYSROOT="$SDK" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_OSX_DEPLOYMENT_TARGET=17.0 + cmake --build "$CACHE/$SDK" --config Release --target stable-diffusion -- -quiet CODE_SIGNING_ALLOWED=NO + done +# Replace only this generated build artifact. +rm -rf "$OUTPUT" +FRAMEWORKS=() +for SDK in "${SDKS[@]}"; do FRAMEWORKS+=(-framework "$CACHE/$SDK/bin/Release/OffgridVideoRuntime.framework"); done +xcodebuild -create-xcframework "${FRAMEWORKS[@]}" -output "$OUTPUT" +printf '%s\n' "$BUILD_KEY" > "$OUTPUT/revision" +cp "$SOURCE/LICENSE" "$ROOT/native/video/RUNTIME-LICENSE" diff --git a/scripts/prepare-video-runtime.sh b/scripts/prepare-video-runtime.sh new file mode 100644 index 000000000..326708667 --- /dev/null +++ b/scripts/prepare-video-runtime.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SOURCE="$ROOT/.video-build/source" +REVISION="$(cat "$ROOT/scripts/video/revision")" +mkdir -p "$ROOT/.video-build" +if [ ! -d "$SOURCE/.git" ]; then + git clone --filter=blob:none --no-checkout https://github.com/leejet/stable-diffusion.cpp.git "$SOURCE" +fi +git -C "$SOURCE" fetch origin "$REVISION" +git -C "$SOURCE" checkout --detach "$REVISION" +git -C "$SOURCE" submodule update --init --depth 1 ggml +# Use the initialized Vulkan dispatcher for Vulkan 1.1 features. Direct symbols +# are absent from Android API 24's loader even when the device supports them. +PATCH="$ROOT/scripts/video/android-vulkan-dispatch.patch" +if ! git -C "$SOURCE/ggml" apply --reverse --check "$PATCH" 2>/dev/null; then + git -C "$SOURCE/ggml" apply "$PATCH" +fi +# This overlay extends the scheduler inserted by android-hardware-fallback.patch. +# Remove it before checking the base patch, then apply it again in order below. +# Do not reset the source tree: other platform patches must remain intact. +ROUTING_PATCH="$ROOT/scripts/video/android-attention-query-routing.patch" +if git -C "$SOURCE" apply --reverse --check "$ROUTING_PATCH" 2>/dev/null; then + git -C "$SOURCE" apply --reverse "$ROUTING_PATCH" +fi +for entry in "ggml:android-vulkan-device-fault.patch" "ggml:android-vulkan-pipeline-diagnostics.patch" "ggml:android-vulkan-matvec-fallback.patch" "ggml:android-vulkan-attention-fallback.patch" "ggml:android-hexagon.patch" "ggml:android-hexagon-buffer-validation.patch" "ggml:android-hexagon-precision.patch" ".:android-hardware-fallback.patch" ".:conditioning-errors.patch" ".:android-attention-buffer-ownership.patch" ".:android-attention-query-chunks.patch" ".:android-attention-query-routing.patch" ".:video-decode-observer.patch" ".:android-video-numerics.patch"; do + target="${entry%%:*}" + patch="$ROOT/scripts/video/${entry#*:}" + if ! git -C "$SOURCE/$target" apply --reverse --check "$patch" 2>/dev/null; then + git -C "$SOURCE/$target" apply "$patch" + fi +done +# Match the header versions used by the runtime's Vulkan build. These are build +# inputs only; the device still supplies its Vulkan driver. +prepare_headers() { + local project="$1" revision="$2" directory="$ROOT/.video-build/$1-source" + if [ ! -d "$directory/.git" ]; then git clone --filter=blob:none --no-checkout "https://github.com/KhronosGroup/$project.git" "$directory"; fi + git -C "$directory" fetch origin "$revision" + git -C "$directory" checkout --detach "$revision" + cmake -S "$directory" -B "$directory/build" -DCMAKE_INSTALL_PREFIX="$ROOT/.video-build/vulkan-header-prefix" + cmake --install "$directory/build" +} +prepare_headers SPIRV-Headers 01e0577914a75a2569c846778c2f93aa8e6feddd +prepare_headers Vulkan-Headers 19725e4d48082fe78e26622b15d3080ccd54112b + +if [ "${1:-}" = "--android" ]; then + bash "$ROOT/scripts/video/build-shaderc.sh" + bash "$ROOT/scripts/video/build-hexagon.sh" +fi + +touch "$ROOT/.video-build/runtime-ready" diff --git a/scripts/rag/__tests__/embeddingCatalog.test.cjs b/scripts/rag/__tests__/embeddingCatalog.test.cjs new file mode 100644 index 000000000..8df15a99d --- /dev/null +++ b/scripts/rag/__tests__/embeddingCatalog.test.cjs @@ -0,0 +1,67 @@ +/* eslint-env node, es2022 */ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const ts = require('typescript'); + +// HTTP is the external boundary. Repository and file responses use the Hub API contract. +function browser(fetchBoundary) { + const filename = path.resolve(__dirname, '../../../src/services/huggingFaceModelBrowser.ts'); + const source = ts.transpileModule(readFileSync(filename, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 }, + }).outputText; + const module = { exports: {} }; + const nativeRequire = name => name === 'react-native-fs' ? require('node:fs/promises') : require(name); + vm.runInThisContext(`(function(require,module,exports,fetch){${source}\n})`, { filename })(nativeRequire, module, module.exports, fetchBoundary); + return module.exports; +} +const revision = 'a'.repeat(40); +const response = data => new Response(JSON.stringify(data)); + +test('search combines embedding tasks and returns pinned single-file candidates only', async () => { + const { searchEmbeddingModels } = browser(async address => { + const url = new URL(address); + if (url.pathname === '/api/models') { + return response(url.searchParams.get('pipeline_tag') === 'feature-extraction' + ? [{ id: 'org/encoder' }, { id: 'org/decoder' }] + : [{ id: 'org/encoder' }]); + } + if (url.pathname === '/api/models/org/decoder') return response({ sha: revision, gguf: { architecture: 'llama' }, siblings: [] }); + if (url.pathname === '/api/models/org/encoder') return response({ + sha: revision, gguf: { architecture: 'bert' }, siblings: [ + { rfilename: 'nested/encoder Q8.gguf', lfs: { size: 40000000, sha256: 'b'.repeat(64) } }, + { rfilename: 'encoder-00001-of-00002.gguf', size: 30 }, + { rfilename: 'mmproj.gguf', size: 30 }, + { rfilename: 'encoder.onnx', size: 30 }, + { rfilename: 'unknown.gguf' }, + ], + }); + throw new Error(`Unexpected API resource: ${url.pathname}`); + }); + const found = await searchEmbeddingModels('encoder'); + assert.equal(found.length, 1); + assert.equal(found[0].id, `org/encoder@${revision}/nested/encoder Q8.gguf`); + assert.equal(found[0].downloadUrl, `https://huggingface.co/org/encoder/resolve/${revision}/nested/encoder%20Q8.gguf`); + assert.equal(found[0].size, 40000000); + assert.equal(found[0].sha256, 'b'.repeat(64)); +}); + +test('search reports a failed file listing instead of claiming no compatible models exist', async () => { + const { searchEmbeddingModels } = browser(async address => new URL(address).pathname === '/api/models' + ? response([{ id: 'org/encoder' }]) : new Response('', { status: 503 })); + await assert.rejects(searchEmbeddingModels('encoder'), /Could not read embedding model files/); +}); + +test('recommendations have pinned identities, checksums, and explicit download bytes', () => { + const { RECOMMENDED_EMBEDDING_MODELS, BUNDLED_EMBEDDING_MODEL } = browser(fetch); + assert.equal(BUNDLED_EMBEDDING_MODEL.size, 0); + assert.equal(BUNDLED_EMBEDDING_MODEL.downloadUrl, undefined); + for (const candidate of RECOMMENDED_EMBEDDING_MODELS) { + assert.match(candidate.id, /@[a-f0-9]{40}\/.+\.gguf$/); + assert.match(candidate.downloadUrl, /\/resolve\/[a-f0-9]{40}\/.+\.gguf$/); + assert.match(candidate.sha256, /^[a-f0-9]{64}$/); + assert.ok(candidate.size > 0); + } +}); diff --git a/scripts/video/CMakeLists.txt b/scripts/video/CMakeLists.txt new file mode 100644 index 000000000..8b38e0bf6 --- /dev/null +++ b/scripts/video/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.24) +project(OffgridVideoRuntime LANGUAGES C CXX OBJC OBJCXX) +set(SD_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(SD_BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) +set(SD_BUILD_SHARED_GGML_LIB OFF CACHE BOOL "" FORCE) +set(SD_METAL ON CACHE BOOL "" FORCE) +set(SD_WEBP OFF CACHE BOOL "" FORCE) +set(SD_WEBM OFF CACHE BOOL "" FORCE) +set(GGML_METAL_EMBED_LIBRARY ON CACHE BOOL "" FORCE) +set(GGML_NATIVE OFF CACHE BOOL "" FORCE) +set(GGML_OPENMP OFF CACHE BOOL "" FORCE) +set(CMAKE_C_VISIBILITY_PRESET hidden) +set(CMAKE_CXX_VISIBILITY_PRESET hidden) +add_subdirectory("${SD_SOURCE}" runtime) +target_sources(stable-diffusion PRIVATE "${SD_SOURCE}/include/stable-diffusion.h") +set_target_properties(stable-diffusion PROPERTIES + FRAMEWORK TRUE FRAMEWORK_VERSION A + OUTPUT_NAME OffgridVideoRuntime + MACOSX_FRAMEWORK_IDENTIFIER ai.offgrid.video.runtime + XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER ai.offgrid.video.runtime + PUBLIC_HEADER "${SD_SOURCE}/include/stable-diffusion.h" + XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED NO + INSTALL_NAME_DIR "@rpath") +target_link_options(stable-diffusion PRIVATE "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/exports.txt") diff --git a/scripts/video/android-attention-buffer-ownership.patch b/scripts/video/android-attention-buffer-ownership.patch new file mode 100644 index 000000000..00447423e --- /dev/null +++ b/scripts/video/android-attention-buffer-ownership.patch @@ -0,0 +1,24 @@ +--- a/src/core/ggml_extend.cpp ++++ b/src/core/ggml_extend.cpp +@@ -802,11 +802,21 @@ + + auto kq = ggml_mul_mat(ctx, k, q); // [N * n_head, L_q, L_k] + ggml_mul_mat_set_prec(kq, GGML_PREC_F32); ++#ifdef __ANDROID__ ++ // F32 attention matmul can fall back from HTP to Vulkan. In-place ++ // outputs would retain that device's buffer across scheduler splits. ++ kq = ggml_scale(ctx, kq, scale); ++ if (mask) { ++ kq = ggml_add(ctx, kq, mask); ++ } ++ kq = ggml_soft_max(ctx, kq); ++#else + kq = ggml_scale_inplace(ctx, kq, scale); + if (mask) { + kq = ggml_add_inplace(ctx, kq, mask); + } + kq = ggml_soft_max_inplace(ctx, kq); ++#endif + + kqv = ggml_mul_mat(ctx, v, kq); // [N * n_head, L_q, d_head] + diff --git a/scripts/video/android-attention-query-chunks.patch b/scripts/video/android-attention-query-chunks.patch new file mode 100644 index 000000000..33c4af2b5 --- /dev/null +++ b/scripts/video/android-attention-query-chunks.patch @@ -0,0 +1,78 @@ +--- a/src/core/ggml_extend.cpp ++++ b/src/core/ggml_extend.cpp +@@ -721,6 +721,46 @@ + mask_in = ggml_cast(ctx, mask_in, GGML_TYPE_F16); + } + ++#ifdef __ANDROID__ ++ // Adreno can lose the device on a single long video attention dispatch. ++ // Split only queries: each query still attends to every original key/value. ++ auto is_adreno = [](ggml_backend_dev_t device) { ++ const char* name = device ? ggml_backend_dev_name(device) : nullptr; ++ const char* description = device ? ggml_backend_dev_description(device) : nullptr; ++ return name && description && std::string(name).find("Vulkan") != std::string::npos && ++ std::string(description).find("Adreno") != std::string::npos; ++ }; ++ bool adreno_attention = backend && is_adreno(ggml_backend_get_device(backend)); ++ // HTP may accept the initial default-precision node, then delegate its ++ // FP32 execution to Vulkan through the scheduler's fallback backend. ++ if (!adreno_attention && sd_backend_is(backend, "HTP")) { ++ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { ++ if (is_adreno(ggml_backend_dev_get(i))) { adreno_attention = true; break; } ++ } ++ } ++ if (mask_in == nullptr && adreno_attention && q_in->ne[1] > 128) { ++ // One owned full-Q buffer lets all query views stay on the GPU. ++ // RoPE may otherwise leave Q aliased to an in-place HTP result. ++ q_in = ggml_cont(ctx, q_in); ++ const int64_t work_per_query = 2 * k_in->ne[1] * ++ (q_in->ne[0] + v_in->ne[0]) * q_in->ne[2] * q_in->ne[3]; ++ const int64_t chunk_rows = std::max(8, std::min(128, ++ (4000000000LL / std::max(1, work_per_query) / 8) * 8)); ++ ggml_tensor* joined = nullptr; ++ for (int64_t start = 0; start < q_in->ne[1]; start += chunk_rows) { ++ const int64_t rows = std::min(chunk_rows, q_in->ne[1] - start); ++ auto query = ggml_cont(ctx, ggml_view_4d(ctx, q_in, ++ q_in->ne[0], rows, q_in->ne[2], q_in->ne[3], ++ q_in->nb[1], q_in->nb[2], q_in->nb[3], start * q_in->nb[1])); ++ auto part = ggml_flash_attn_ext(ctx, query, k_in, v_in, nullptr, scale / kv_scale, 0, 0); ++ if (!ggml_backend_supports_op(backend, part)) return nullptr; ++ ggml_flash_attn_ext_set_prec(part, GGML_PREC_F32); ++ joined = joined ? ggml_concat(ctx, joined, part, 2) : part; ++ } ++ if (kv_scale != 1.0f) joined = ggml_ext_scale(ctx, joined, 1.0f / kv_scale); ++ return joined; ++ } ++#endif + auto out = ggml_flash_attn_ext(ctx, q_in, k_in, v_in, mask_in, scale / kv_scale, 0, 0); + if (!ggml_backend_supports_op(backend, out)) { + return nullptr; +--- a/src/model/diffusion/wan.hpp ++++ b/src/model/diffusion/wan.hpp +@@ -988,7 +988,25 @@ + float vace_strength = 1.f, + const sd::Tensor& audio_embed_tensor = {}, + const sd::Tensor& ref_latent_tensor = {}) { +- ggml_cgraph* gf = new_graph_custom(WAN_GRAPH_SIZE); ++ size_t graph_size = WAN_GRAPH_SIZE; ++#ifdef __ANDROID__ ++ // Query chunks keep full attention but add view/copy/concat nodes. ++ // Reserve graph metadata for that path inside the existing 327680-node arena. ++ if (sd_backend_is(runtime_backend, "Vulkan") || sd_backend_is(runtime_backend, "HTP")) { ++ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { ++ auto device = ggml_backend_dev_get(i); ++ const char* name = ggml_backend_dev_name(device); ++ const char* description = ggml_backend_dev_description(device); ++ if (name && description && std::string(name).find("Vulkan") != std::string::npos && ++ std::string(description).find("Adreno") != std::string::npos) { ++ graph_size = 131072; ++ LOG_DEBUG("Adreno chunked Wan graph capacity: %zu", graph_size); ++ break; ++ } ++ } ++ } ++#endif ++ ggml_cgraph* gf = new_graph_custom(graph_size); + + ggml_tensor* x = make_input(x_tensor); + ggml_tensor* timesteps = make_input(timesteps_tensor); diff --git a/scripts/video/android-attention-query-routing.patch b/scripts/video/android-attention-query-routing.patch new file mode 100644 index 000000000..5e6ddb3e5 --- /dev/null +++ b/scripts/video/android-attention-query-routing.patch @@ -0,0 +1,71 @@ +--- a/src/core/ggml_runner.cpp ++++ b/src/core/ggml_runner.cpp +@@ -210,6 +210,29 @@ + if (sched == nullptr || gf == nullptr) { + return; + } ++#ifdef __ANDROID__ ++ ggml_backend_t attention_backend = nullptr; ++ int attention_nodes_pinned = 0; ++ const int backend_count = ggml_backend_sched_get_n_backends(sched); ++ for (int index = 0; index < backend_count; ++index) { ++ auto candidate = ggml_backend_sched_get_backend(sched, index); ++ auto device = candidate ? ggml_backend_get_device(candidate) : nullptr; ++ const char* description = device ? ggml_backend_dev_description(device) : nullptr; ++ if (sd_backend_is(candidate, "Vulkan") && description && ++ std::string(description).find("Adreno") != std::string::npos) { ++ attention_backend = candidate; ++ break; ++ } ++ } ++ auto is_query_chunk = [](const ggml_tensor* node) { ++ if (!node || node->op != GGML_OP_FLASH_ATTN_EXT || node->src[3] || ++ node->op_params[3] != GGML_PREC_F32) return false; ++ auto query = node->src[0]; ++ auto view = query && query->op == GGML_OP_CONT ? query->src[0] : nullptr; ++ return view && view->op == GGML_OP_VIEW && view->src[0] && ++ query->ne[1] <= 128 && view->ne[1] < view->src[0]->ne[1]; ++ }; ++#endif + ggml_backend_t current = runtime_backend; + const int n_nodes = ggml_graph_n_nodes(gf); + for (int i = 0; i < n_nodes; i++) { +@@ -230,6 +253,28 @@ + node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE) { + continue; + } ++#ifdef __ANDROID__ ++ // Keep query copies and their joined attention output on the same GPU. ++ // Pinning these to HTP would copy each chunk across both devices. ++ if (attention_backend && (is_query_chunk(node) || ++ (node->op == GGML_OP_CONCAT && is_query_chunk(node->src[1]))) && ++ ggml_backend_supports_op(attention_backend, node)) { ++ ggml_backend_sched_set_tensor_backend(sched, node, attention_backend); ++ ++attention_nodes_pinned; ++ if (is_query_chunk(node) && ggml_backend_supports_op(attention_backend, node->src[0])) { ++ ggml_backend_sched_set_tensor_backend(sched, node->src[0], attention_backend); ++ // Query views share this full-Q contiguous producer. Keep it ++ // on GPU too, so the scheduler does not recopy it for each view. ++ auto query_view = node->src[0]->src[0]; ++ auto query_storage = query_view ? query_view->view_src : nullptr; ++ if (query_storage && query_storage->op == GGML_OP_CONT && ++ ggml_backend_supports_op(attention_backend, query_storage)) { ++ ggml_backend_sched_set_tensor_backend(sched, query_storage, attention_backend); ++ } ++ } ++ continue; ++ } ++#endif + // In-place operations must follow the device chosen for their source. + // Its buffer may not exist yet when this graph is assigned. + if (node->view_src != nullptr) continue; +@@ -239,6 +284,9 @@ + ggml_backend_sched_set_tensor_backend(sched, node, current); + } + } ++#ifdef __ANDROID__ ++ if (attention_backend) LOG_DEBUG("Adreno query-chunk GPU nodes pinned: %d", attention_nodes_pinned); ++#endif + } + + size_t GGMLRunner::retained_runtime_buffer_bytes(ggml_backend_t backend) const { diff --git a/scripts/video/android-hardware-fallback.patch b/scripts/video/android-hardware-fallback.patch new file mode 100644 index 000000000..5e2411a4d --- /dev/null +++ b/scripts/video/android-hardware-fallback.patch @@ -0,0 +1,111 @@ +diff --git a/src/core/compute_workspace.cpp b/src/core/compute_workspace.cpp +index 54f0538..9f20b7e 100644 +--- a/src/core/compute_workspace.cpp ++++ b/src/core/compute_workspace.cpp +@@ -18,6 +18,7 @@ namespace sd { + segment_end(); + release(); + ggml_backend_free(cpu_backend_); ++ ggml_backend_free(gpu_fallback_); + } + + void ComputeWorkspace::set_extra_backends(const std::vector& backends) { +@@ -43,6 +44,19 @@ namespace sd { + ggml_backend_sched_t ComputeWorkspace::make_scheduler(size_t graph_size) { + std::vector backends{backend_}; + backends.insert(backends.end(), extra_backends_.begin(), extra_backends_.end()); ++#ifdef __ANDROID__ ++ // Hexagon is preferred; unsupported operations go to Vulkan, then CPU. ++ if (sd_backend_is(backend_, "HTP")) { ++ if (gpu_fallback_ == nullptr) { ++ auto device = ggml_backend_dev_by_name("Vulkan0"); ++ if (device) gpu_fallback_ = ggml_backend_dev_init(device, nullptr); ++ } ++ if (gpu_fallback_ != nullptr && ++ std::find(backends.begin(), backends.end(), gpu_fallback_) == backends.end()) { ++ backends.push_back(gpu_fallback_); ++ } ++ } ++#endif + if (!sd_backend_is_cpu(backend_)) { + if (cpu_backend_ == nullptr) { + cpu_backend_ = sd_backend_cpu_init(); +@@ -207,11 +221,12 @@ namespace sd { + auto scheduler = make_scheduler(graph_size); + if (scheduler != nullptr) { + assign_nodes(scheduler, copy_graph); +- std::vector sizes(extra_backends_.size() + 2); ++ const int count = ggml_backend_sched_get_n_backends(scheduler); ++ std::vector sizes(count); + ggml_backend_sched_reserve_size(scheduler, copy_graph, sizes.data()); +- result.buffers.push_back({backend_, sizes[0]}); +- for (size_t i = 0; i < extra_backends_.size(); ++i) { +- result.buffers.push_back({extra_backends_[i], sizes[i + 1]}); ++ for (int i = 0; i < count; ++i) { ++ auto backend = ggml_backend_sched_get_backend(scheduler, i); ++ if (backend != cpu_backend_) result.buffers.push_back({backend, sizes[i]}); + } + ggml_backend_sched_free(scheduler); + } +diff --git a/src/core/compute_workspace.h b/src/core/compute_workspace.h +index fcf14b8..203d62c 100644 +--- a/src/core/compute_workspace.h ++++ b/src/core/compute_workspace.h +@@ -17,6 +17,7 @@ namespace sd { + ggml_backend_t backend_; + std::vector extra_backends_; + ggml_backend_t cpu_backend_ = nullptr; ++ ggml_backend_t gpu_fallback_ = nullptr; + ggml_gallocr_t allocator_ = nullptr; + ggml_backend_sched_t scheduler_ = nullptr; + size_t scheduler_capacity_ = 0; +diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp +index d87353d..e491eff 100644 +--- a/src/core/ggml_extend.cpp ++++ b/src/core/ggml_extend.cpp +@@ -517,7 +517,11 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx, + } + if (b != nullptr) { + b = ggml_reshape_4d(ctx, b, 1, 1, 1, b->ne[0]); // [OC, 1, 1, 1] +- x = ggml_add_inplace(ctx, x, b); ++ // The convolution may fall back to CPU while ADD runs on an accelerator. ++ // Give ADD its own output so the scheduler can transfer the input. ++ x = backend != nullptr && !ggml_backend_supports_op(backend, x) ++ ? ggml_add(ctx, x, b) ++ : ggml_add_inplace(ctx, x, b); + } + return x; + } +diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp +index 4c156ed..c01c8d0 100644 +--- a/src/core/ggml_runner.cpp ++++ b/src/core/ggml_runner.cpp +@@ -230,7 +230,12 @@ void GGMLRunner::pin_multi_device_nodes(ggml_backend_sched_t sched, ggml_cgraph* + node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE) { + continue; + } +- if (ggml_backend_supports_op(current, node)) { ++ // In-place operations must follow the device chosen for their source. ++ // Its buffer may not exist yet when this graph is assigned. ++ if (node->view_src != nullptr) continue; ++ auto buffer = node->buffer; ++ if (ggml_backend_supports_op(current, node) && ++ (buffer == nullptr || ggml_backend_supports_buft(current, ggml_backend_buffer_get_type(buffer)))) { + ggml_backend_sched_set_tensor_backend(sched, node, current); + } + } +@@ -728,8 +733,12 @@ std::vector GGMLRunner::memory_requests( + const size_t reusable = workspace_.bytes(size.backend); + const size_t cache_bytes = size.backend == runtime_backend ? pending_cache_bytes : 0; + const size_t pending = add_bytes(size.bytes > reusable ? size.bytes - reusable : 0, cache_bytes); +- size_t limit = max_graph_vram_bytes; +- if (is_multi_device()) { ++ // A fallback device has its own physical memory checks. Do not apply ++ // the primary device's address-space cap (for example Hexagon's) to it. ++ const bool configured_backend = size.backend == runtime_backend || ++ std::find(extra_runtime_backends.begin(), extra_runtime_backends.end(), size.backend) != extra_runtime_backends.end(); ++ size_t limit = configured_backend ? max_graph_vram_bytes : 0; ++ if (configured_backend && is_multi_device()) { + size_t index = 0; + if (size.backend != runtime_backend) { + auto position = std::find(extra_runtime_backends.begin(), extra_runtime_backends.end(), size.backend); diff --git a/scripts/video/android-hexagon-buffer-validation.patch b/scripts/video/android-hexagon-buffer-validation.patch new file mode 100644 index 000000000..d2f7a4024 --- /dev/null +++ b/scripts/video/android-hexagon-buffer-validation.patch @@ -0,0 +1,29 @@ +--- a/src/ggml-hexagon/ggml-hexagon.cpp ++++ b/src/ggml-hexagon/ggml-hexagon.cpp +@@ -6474,6 +6474,26 @@ + } + } + ++ // Mixed-backend graphs can retain tensors backed by another device. ++ // Validate the full split before queuing any DSP work: buffer contexts ++ // from CPU/Vulkan are not ggml_hexagon_shared_buffer objects. ++ for (const auto & node : *nodes_ptr) { ++ auto valid_buffer = [&](const ggml_tensor * tensor) { ++ if (!tensor) return true; ++ if (tensor->buffer && ggml_backend_buffer_is_hexagon(tensor->buffer)) return true; ++ GGML_LOG_ERROR("ggml-hex: %s cannot execute %s: tensor %s uses %s buffer\n", ++ sess->c_name(), node.op_name().c_str(), tensor->name, ++ tensor->buffer ? ggml_backend_buffer_name(tensor->buffer) : "missing"); ++ return false; ++ }; ++ for (const auto * tensor : node.get_inputs()) { ++ if (!valid_buffer(tensor)) return GGML_STATUS_FAILED; ++ } ++ for (const auto * tensor : node.get_outputs()) { ++ if (!valid_buffer(tensor)) return GGML_STATUS_FAILED; ++ } ++ } ++ + // Queue and execute + for (const auto & node : *nodes_ptr) { + sess->enqueue_op(node); diff --git a/scripts/video/android-hexagon-precision.patch b/scripts/video/android-hexagon-precision.patch new file mode 100644 index 000000000..fc525fc7d --- /dev/null +++ b/scripts/video/android-hexagon-precision.patch @@ -0,0 +1,30 @@ +--- a/src/ggml-hexagon/ggml-hexagon.cpp ++++ b/src/ggml-hexagon/ggml-hexagon.cpp +@@ -4287,6 +4287,8 @@ + } + + static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { ++ // HMX keeps attention scores and recurrent output in F16 tiles. ++ if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) return false; + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * src2 = op->src[2]; +@@ -5391,6 +5393,9 @@ + } + + static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) { ++ // HMX can lower activations to F16. Honor explicit F32 accumulation ++ // requests (for example video attention) through another backend. ++ if (ggml_get_op_params_i32(dst, 0) == GGML_PREC_F32) return false; + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + +@@ -5481,7 +5486,7 @@ + return false; + } + +- if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { ++ if (ggml_get_op_params_i32(op, 0) == GGML_PREC_F32) { + return false; + } + diff --git a/scripts/video/android-hexagon.patch b/scripts/video/android-hexagon.patch new file mode 100644 index 000000000..5c0e8b2b5 --- /dev/null +++ b/scripts/video/android-hexagon.patch @@ -0,0 +1,32 @@ +diff --git a/src/ggml-hexagon/CMakeLists.txt b/src/ggml-hexagon/CMakeLists.txt +index 845b922..0a9f237 100644 +--- a/src/ggml-hexagon/CMakeLists.txt ++++ b/src/ggml-hexagon/CMakeLists.txt +@@ -1,3 +1,9 @@ ++# Off Grid builds DSP libraries separately so Android hosts can use macOS. ++if(OFFGRID_VIDEO_HEXAGON_CMAKE) ++ include("${OFFGRID_VIDEO_HEXAGON_CMAKE}") ++ return() ++endif() ++ + file(TO_CMAKE_PATH "${HEXAGON_SDK_ROOT}" HEXAGON_SDK_ROOT) + file(TO_CMAKE_PATH "${HEXAGON_TOOLS_ROOT}" HEXAGON_TOOLS_ROOT) + +diff --git a/src/ggml-hexagon/ggml-hexagon.cpp b/src/ggml-hexagon/ggml-hexagon.cpp +index 9a35002..0ca4222 100644 +--- a/src/ggml-hexagon/ggml-hexagon.cpp ++++ b/src/ggml-hexagon/ggml-hexagon.cpp +@@ -3887,7 +3887,12 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n + char session_uri[256]; + { + char htp_uri[256]; +- snprintf(htp_uri, sizeof(htp_uri), "file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch); ++ const char * video_htp_dir = getenv("OFFGRID_VIDEO_HTP_DIR"); ++ if (video_htp_dir && *video_htp_dir) { ++ snprintf(htp_uri, sizeof(htp_uri), "file:///liboffgrid-video-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch); ++ } else { ++ throw std::runtime_error("Video Hexagon runtime directory is not configured"); ++ } + + struct remote_rpc_get_uri u = {}; + u.session_id = this->session_id; diff --git a/scripts/video/android-video-numerics.patch b/scripts/video/android-video-numerics.patch new file mode 100644 index 000000000..e65cb8500 --- /dev/null +++ b/scripts/video/android-video-numerics.patch @@ -0,0 +1,55 @@ +--- a/src/pipeline/video.cpp ++++ b/src/pipeline/video.cpp +@@ -1214,6 +1214,32 @@ + return embeds; + } + ++#if defined(__ANDROID__) ++ // These tensors are already in host memory. Scan once without copying them. ++ static void log_video_tensor_summary(const char* stage, const sd::Tensor& tensor) { ++ int64_t finite = 0; ++ int64_t zero = 0; ++ double minimum = INFINITY; ++ double maximum = -INFINITY; ++ double sum = 0; ++ double squares = 0; ++ for (const float value : tensor.values()) { ++ if (!std::isfinite(value)) continue; ++ ++finite; ++ if (value == 0.0f) ++zero; ++ minimum = std::min(minimum, static_cast(value)); ++ maximum = std::max(maximum, static_cast(value)); ++ sum += value; ++ squares += static_cast(value) * value; ++ } ++ LOG_INFO("OFFGRID_VIDEO_NUMERICS stage=%s values=%lld finite=%lld nonfinite=%lld zero=%lld min=%.9g max=%.9g mean=%.9g rms=%.9g", ++ stage, (long long)tensor.numel(), (long long)finite, ++ (long long)(tensor.numel() - finite), (long long)zero, ++ minimum, maximum, finite ? sum / finite : 0.0, ++ finite ? std::sqrt(squares / finite) : 0.0); ++ } ++#endif ++ + static sd_image_t* decode_video_outputs(StableDiffusionGGML* sd, + const GenerationRequest& request, + const sd::Tensor& final_latent, +@@ -1226,6 +1252,9 @@ + LOG_ERROR("cancelling video decode"); + return nullptr; + } ++#if defined(__ANDROID__) ++ log_video_tensor_summary("diffusion_latent", final_latent); ++#endif + sd::Tensor video_latent = final_latent; + if ((sd_version_is_ltxav(sd->version) || sd_version_is_minimax_h3(sd->version)) && + video_latent.shape()[3] > sd->get_latent_channel()) { +@@ -1254,6 +1283,9 @@ + (int)vid.shape()[1], + (int)vid.shape()[2], + (int)vid.shape()[3]); ++#if defined(__ANDROID__) ++ log_video_tensor_summary("decoded_rgb", vid); ++#endif + if (request.frames > 0 && + vid.shape()[2] > request.frames) { + vid = sd::ops::slice(vid, 2, 0, request.frames); diff --git a/scripts/video/android-vulkan-attention-fallback.patch b/scripts/video/android-vulkan-attention-fallback.patch new file mode 100644 index 000000000..5b6a0ea52 --- /dev/null +++ b/scripts/video/android-vulkan-attention-fallback.patch @@ -0,0 +1,74 @@ +--- a/src/ggml-vulkan/ggml-vulkan.cpp ++++ b/src/ggml-vulkan/ggml-vulkan.cpp +@@ -1259,6 +1259,16 @@ + const uint32_t D_lsb = D ^ (D & (D-1)); // extract lowest set bit + + result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4); ++ ++#ifdef __ANDROID__ ++ if (device->vendor_id == VK_VENDOR_ID_QUALCOMM && f32acc && ++ k_type == GGML_TYPE_F16 && v_type == GGML_TYPE_F16) { ++ // Use the existing shared-memory reduction path instead of the ++ // subgroup shuffle pipeline that loses the Adreno device in Wan. ++ result.disable_subgroups = true; ++ result.block_rows = std::min(result.block_rows, 8u); ++ } ++#endif + + result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0; + +@@ -1982,6 +1992,16 @@ + } + name = aligned ? "flash_attn_f32_f16_aligned" : "flash_attn_f32_f16"; + } ++#ifdef __ANDROID__ ++ if (device->vendor_id == VK_VENDOR_ID_QUALCOMM && f32acc && ++ fa.first.k_type == GGML_TYPE_F16 && fa.first.v_type == GGML_TYPE_F16) { ++ // Preserve FP32 accumulation and avoid the mobile FP16 arithmetic ++ // path. K/V storage stays F16; no extra attention matrix is stored. ++ spv_data = flash_attn_f32_f16_fp32_data; ++ spv_size = flash_attn_f32_f16_fp32_len; ++ name = "flash_attn_adreno_f32_shared"; ++ } ++#endif + ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 8, + sizeof(vk_flash_attn_push_constants), {Br, 1, 1}, + get_fa_spec_constants(fa.first), aligned ? Bc : 1, true, +@@ -14226,6 +14246,13 @@ + uint64_t batch_flops = 0; + uint64_t total_flops = 0; + uint64_t flops_cap = 200'000'000'000ULL; ++#ifdef __ANDROID__ ++ // Keep query chunks in separate submissions on mobile Adreno. A full ++ // Wan attention dispatch exceeds 130 GFLOP and can hit the driver limit. ++ if (ctx->device->vendor_id == VK_VENDOR_ID_QUALCOMM) { ++ flops_cap = 8'000'000'000ULL; ++ } ++#endif + + // On weaker AMD GPUs larger submissions can hit a driver timeout, submit more often to avoid this + if (ctx->device->vendor_id == VK_VENDOR_ID_AMD && ctx->device->shader_core_count > 0) { +@@ -14236,6 +14263,11 @@ + } + } + uint64_t flops_per_submit = std::min(flops_cap, ctx->last_total_flops / 40u); ++#ifdef __ANDROID__ ++ if (ctx->device->vendor_id == VK_VENDOR_ID_QUALCOMM) { ++ flops_per_submit = flops_cap; ++ } ++#endif + + auto const submit_after = [&](int start, int end) { + if (ctx->device->serialize_submissions) { +@@ -14263,6 +14295,11 @@ + batch_flops = 0; + if (submit_count < 3) { + flops_per_submit *= 2; ++#ifdef __ANDROID__ ++ if (ctx->device->vendor_id == VK_VENDOR_ID_QUALCOMM) { ++ flops_per_submit = std::min(flops_cap, flops_per_submit); ++ } ++#endif + } + submit_count++; + }; diff --git a/scripts/video/android-vulkan-device-fault.patch b/scripts/video/android-vulkan-device-fault.patch new file mode 100644 index 000000000..9762ca261 --- /dev/null +++ b/scripts/video/android-vulkan-device-fault.patch @@ -0,0 +1,18 @@ +--- a/src/ggml-vulkan/ggml-vulkan.cpp ++++ b/src/ggml-vulkan/ggml-vulkan.cpp +@@ -146,6 +146,15 @@ + return std::find(lightning_indexer_k_types.begin(), lightning_indexer_k_types.end(), type) != lightning_indexer_k_types.end(); + } + void ggml_vk_print_device_fault_info(const vk_device& device) { ++#ifdef __ANDROID__ ++ // Adreno's fault-info callback can dereference null after device loss ++ // (observed on SM8635). Keep normal error reporting and exception handling ++ // alive instead of crashing while asking the driver for extra diagnostics. ++ if (device->vendor_id == VK_VENDOR_ID_QUALCOMM) { ++ GGML_LOG_ERROR("ggml_vulkan: skipping unsafe Adreno device-fault query\n"); ++ return; ++ } ++#endif + if (!device->device_fault || !device->pfn_vkGetDeviceFaultInfoEXT) { + return; + } diff --git a/scripts/video/android-vulkan-dispatch.patch b/scripts/video/android-vulkan-dispatch.patch new file mode 100644 index 000000000..41a93caff --- /dev/null +++ b/scripts/video/android-vulkan-dispatch.patch @@ -0,0 +1,29 @@ +--- a/src/ggml-vulkan/ggml-vulkan.cpp ++++ b/src/ggml-vulkan/ggml-vulkan.cpp +@@ -4306,7 +4306,7 @@ + device_extensions.push_back("VK_EXT_device_fault"); + } + +- vkGetPhysicalDeviceFeatures2(device->physical_device, &device_features2); ++ VULKAN_HPP_DEFAULT_DISPATCHER.vkGetPhysicalDeviceFeatures2(device->physical_device, &device_features2); + + device->device_fault = device->device_fault && fault_features.deviceFault; + +@@ -4930,7 +4930,7 @@ + } + #endif + +- vkGetPhysicalDeviceFeatures2(physical_device, &device_features2); ++ VULKAN_HPP_DEFAULT_DISPATCHER.vkGetPhysicalDeviceFeatures2(physical_device, &device_features2); + + fp16 = fp16 && vk12_features.shaderFloat16; + +@@ -16170,7 +16170,7 @@ + vk11_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES; + device_features2.pNext = &vk11_features; + +- vkGetPhysicalDeviceFeatures2(vkdev, &device_features2); ++ VULKAN_HPP_DEFAULT_DISPATCHER.vkGetPhysicalDeviceFeatures2(vkdev, &device_features2); + + return vk11_features.storageBuffer16BitAccess; + } diff --git a/scripts/video/android-vulkan-matvec-fallback.patch b/scripts/video/android-vulkan-matvec-fallback.patch new file mode 100644 index 000000000..feaa46db6 --- /dev/null +++ b/scripts/video/android-vulkan-matvec-fallback.patch @@ -0,0 +1,36 @@ +--- a/src/ggml-vulkan/ggml-vulkan.cpp ++++ b/src/ggml-vulkan/ggml-vulkan.cpp +@@ -15333,6 +15333,33 @@ + case GGML_OP_MUL_MAT_ID: + { + ggml_type src0_type = op->src[0]->type; ++#ifdef __ANDROID__ ++ // SM8635's proprietary Adreno compiler crashes while compiling ++ // a floating-point dmmv shader in Wan VAE decode. Reject only ++ // float vector/small-column MUL_MAT shapes so the scheduler can ++ // use another backend. Keep quantized and regular GPU matrices. ++ const bool float_weight = src0_type == GGML_TYPE_F32 || ++ src0_type == GGML_TYPE_F16 || ++ src0_type == GGML_TYPE_BF16; ++ const bool float_input = op->src[1]->type == GGML_TYPE_F32 || ++ op->src[1]->type == GGML_TYPE_F16; ++ const bool vector_shape = op->ne[1] == 1 || ++ (op->ne[1] <= mul_mat_vec_max_cols && ++ op->src[1]->ne[2] * op->src[1]->ne[3] == 1); ++ // The dispatcher can swap a single output row into matvec too. ++ const bool swapped_vector = op->ne[0] == 1 && ++ op->src[0]->ne[2] == 1 && op->src[0]->ne[3] == 1 && ++ op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 1; ++ if (device->vendor_id == VK_VENDOR_ID_QUALCOMM && ++ op->op == GGML_OP_MUL_MAT && float_weight && float_input && ++ (vector_shape || swapped_vector)) { ++ static std::atomic_flag reported = ATOMIC_FLAG_INIT; ++ if (!reported.test_and_set()) { ++ GGML_LOG_WARN("ggml_vulkan: routing Adreno float matvec to another backend to avoid shader compiler crash\n"); ++ } ++ return false; ++ } ++#endif + if (src0_type == GGML_TYPE_I8) { + return op->op == GGML_OP_MUL_MAT && + ggml_vk_can_use_mul_mat_i8_tensorwise(device, op); diff --git a/scripts/video/android-vulkan-pipeline-diagnostics.patch b/scripts/video/android-vulkan-pipeline-diagnostics.patch new file mode 100644 index 000000000..5c7c1410b --- /dev/null +++ b/scripts/video/android-vulkan-pipeline-diagnostics.patch @@ -0,0 +1,24 @@ +--- a/src/ggml-vulkan/ggml-vulkan.cpp ++++ b/src/ggml-vulkan/ggml-vulkan.cpp +@@ -586,6 +586,21 @@ + GGML_ASSERT(parameter_count > 0); + GGML_ASSERT(parameter_count <= MAX_PARAMETER_COUNT); + GGML_ASSERT(wg_denoms[0] > 0 && wg_denoms[1] > 0 && wg_denoms[2] > 0); // NOLINT ++ ++#ifdef __ANDROID__ ++ // The Adreno compiler can terminate the process inside pipeline creation, ++ // before Vulkan can return an error. Record the lazy shader variant first. ++ if (device->vendor_id == VK_VENDOR_ID_QUALCOMM) { ++ std::string constants; ++ for (const auto value : specialization_constants) { ++ if (!constants.empty()) constants += ","; ++ constants += std::to_string(value); ++ } ++ GGML_LOG_INFO("ggml_vulkan: Adreno compiling %s (specialization=[%s], subgroup=%u, full_subgroups=%d, spirv_bytes=%zu)\n", ++ pipeline->name.c_str(), constants.c_str(), required_subgroup_size, ++ require_full_subgroups ? 1 : 0, spv_size); ++ } ++#endif + + vk::ShaderModuleCreateInfo shader_module_create_info({}, spv_size, reinterpret_cast(spv_data)); + diff --git a/scripts/video/build-hexagon.sh b/scripts/video/build-hexagon.sh new file mode 100755 index 000000000..f88c57b02 --- /dev/null +++ b/scripts/video/build-hexagon.sh @@ -0,0 +1,50 @@ +#!/bin/bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +BUILD="$ROOT/.video-build" +SDK_VERSION=6.4.0.2 +SDK_SHA256=b4a57a774795cf12da19a777a5d306e970905bf9758a4c4765e5e4593428ae0b +SDK="$BUILD/hexagon-sdk/$SDK_VERSION" +# Reuse the DSP artifacts when their source revision and SDK match. +REVISION="$(cat "$ROOT/scripts/video/revision")" +READY="$BUILD/hexagon-runtime/ready" +COMPLETE=true +for file in htp_iface_stub.c htp_iface.h; do + [ -s "$BUILD/hexagon-runtime/$file" ] || COMPLETE=false +done +for arch in v73 v75 v79 v81; do + [ -s "$BUILD/hexagon-assets/video-hexagon/liboffgrid-video-htp-$arch.so" ] || COMPLETE=false +done +if $COMPLETE && [ "$(cat "$READY" 2>/dev/null || true)" = "$REVISION:$SDK_VERSION" ]; then exit 0; fi +mkdir -p "$BUILD/hexagon-sdk" "$BUILD/hexagon-runtime" "$BUILD/hexagon-assets/video-hexagon" +if [ ! -f "$SDK/hexagon_sdk.json" ]; then + curl -fL --retry 3 -C - "https://github.com/snapdragon-toolchain/hexagon-sdk/releases/download/v$SDK_VERSION/hexagon-sdk-v$SDK_VERSION-amd64-lnx.tar.xz" -o "$BUILD/hexagon-sdk.tar.xz" + # A bad archive is removed: `curl -C -` would otherwise resume from the same bad bytes every build. + if ! printf '%s %s\n' "$SDK_SHA256" "$BUILD/hexagon-sdk.tar.xz" | shasum -a 256 -c -; then + rm -f "$BUILD/hexagon-sdk.tar.xz" + echo "Hexagon SDK archive failed its checksum and was removed; run the build again." >&2 + exit 1 + fi + tar -xJf "$BUILD/hexagon-sdk.tar.xz" -C "$BUILD/hexagon-sdk" +fi +# The official Hexagon tools run on Linux. Only this build directory is mounted. +docker run --rm --platform linux/amd64 -v "$BUILD:/work" -w /work ubuntu:22.04 bash -euc ' + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ninja-build python3 python3-pip build-essential libncurses5 libtinfo5 + python3 -m pip install --no-cache-dir cmake==3.31.6 + export HEXAGON_SDK_ROOT=/work/hexagon-sdk/6.4.0.2 + export HEXAGON_TOOLS_ROOT=$HEXAGON_SDK_ROOT/tools/HEXAGON_Tools/19.0.04 + export DEFAULT_HLOS_ARCH=64 DEFAULT_TOOLS_VARIANT=toolv19 DEFAULT_NO_QURT_INC=0 + for arch in v73 v75 v79 v81; do + cmake -S source/ggml/src/ggml-hexagon/htp -B hexagon-build/$arch -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=/work/source/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake \ + -DCMAKE_BUILD_TYPE=Release -DHEXAGON_SDK_ROOT=$HEXAGON_SDK_ROOT \ + -DHEXAGON_TOOLS_ROOT=$HEXAGON_TOOLS_ROOT -DDSP_VERSION=$arch \ + -DPREBUILT_LIB_DIR=toolv19_$arch -DHEXAGON_HTP_DEBUG=OFF + cmake --build hexagon-build/$arch --parallel 2 + cp hexagon-build/$arch/libggml-htp-$arch.so hexagon-assets/video-hexagon/liboffgrid-video-htp-$arch.so + done + cp hexagon-build/v73/htp_iface_stub.c hexagon-build/v73/htp_iface.h hexagon-runtime/ +' +cp "$ROOT/scripts/video/revision" "$BUILD/hexagon-runtime/revision" +printf '%s:%s\n' "$REVISION" "$SDK_VERSION" > "$READY" diff --git a/scripts/video/build-shaderc.sh b/scripts/video/build-shaderc.sh new file mode 100644 index 000000000..f0664dbc4 --- /dev/null +++ b/scripts/video/build-shaderc.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +REVISION=d393a813c9b67c709d37b329203d72beb69d719e +SOURCE="$ROOT/.video-build/shaderc-source" +PREFIX="$ROOT/.video-build/shaderc" +if [ -x "$PREFIX/bin/glslc" ] && [ "$(cat "$PREFIX/revision" 2>/dev/null || true)" = "$REVISION" ]; then exit 0; fi +# A host package of this exact release is also suitable for shader compilation. +if command -v glslc >/dev/null && [[ "$(glslc --version)" == "shaderc v2026.4 "* ]]; then + mkdir -p "$PREFIX/bin" + ln -sf "$(command -v glslc)" "$PREFIX/bin/glslc" + printf '%s\n' "$REVISION" > "$PREFIX/revision" + exit 0 +fi +mkdir -p "$ROOT/.video-build" +if [ ! -d "$SOURCE/.git" ]; then + git clone --filter=blob:none --no-checkout https://github.com/google/shaderc.git "$SOURCE" +fi +git -C "$SOURCE" fetch origin "$REVISION" +git -C "$SOURCE" checkout --detach "$REVISION" +(cd "$SOURCE" && python3 utils/git-sync-deps) +cmake -S "$SOURCE" -B "$SOURCE/build-host" -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DSHADERC_SKIP_TESTS=ON -DSHADERC_SKIP_EXAMPLES=ON -DSHADERC_SKIP_COPYRIGHT_CHECK=ON +cmake --build "$SOURCE/build-host" --target glslc --parallel "${VIDEO_BUILD_JOBS:-4}" +mkdir -p "$PREFIX/bin" +cp "$SOURCE/build-host/glslc/glslc" "$PREFIX/bin/glslc" +printf '%s\n' "$REVISION" > "$PREFIX/revision" diff --git a/scripts/video/conditioning-errors.patch b/scripts/video/conditioning-errors.patch new file mode 100644 index 000000000..15b07dea3 --- /dev/null +++ b/scripts/video/conditioning-errors.patch @@ -0,0 +1,16 @@ +diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp +index 7796a40..33ec41e 100644 +--- a/src/conditioning/conditioner.hpp ++++ b/src/conditioning/conditioner.hpp +@@ -1570,7 +1570,10 @@ struct T5CLIPEmbedder : public Conditioner { + input_ids, + t5_attn_mask_chunk, + false); +- GGML_ASSERT(!chunk_hidden_states.empty()); ++ if (chunk_hidden_states.empty()) { ++ LOG_ERROR("T5 encoder returned no output; the backend failed or generation was cancelled"); ++ return {}; ++ } + chunk_hidden_states = apply_token_weights(std::move(chunk_hidden_states), chunk_weights); + + if (zero_out_masked) { diff --git a/scripts/video/exports.txt b/scripts/video/exports.txt new file mode 100644 index 000000000..828cd07cb --- /dev/null +++ b/scripts/video/exports.txt @@ -0,0 +1,12 @@ +_sd_* +_new_sd_ctx +_free_sd_ctx +_free_sd_images +_generate_video +_generate_image +_str_to_sample_method +_str_to_scheduler +_new_upscaler_ctx +_free_upscaler_ctx +_upscale +_get_upscale_factor diff --git a/scripts/video/hexagon-prebuilt.cmake b/scripts/video/hexagon-prebuilt.cmake new file mode 100644 index 000000000..ebb950c18 --- /dev/null +++ b/scripts/video/hexagon-prebuilt.cmake @@ -0,0 +1,18 @@ +# Host-side Hexagon backend. DSP code is built by build-hexagon.sh from the +# same pinned GGML source; do not reuse llama.rn's different DSP ABI. +set(VIDEO_HEXAGON_ROOT "${SD_SOURCE}/../hexagon-runtime") +set(VIDEO_HEXAGON_SDK "${SD_SOURCE}/../hexagon-sdk/6.4.0.2") +if(NOT EXISTS "${VIDEO_HEXAGON_ROOT}/htp_iface_stub.c") + message(FATAL_ERROR "Run scripts/video/build-hexagon.sh before building Android video.") +endif() +add_library(htp_iface OBJECT "${VIDEO_HEXAGON_ROOT}/htp_iface_stub.c") +set_target_properties(htp_iface PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_include_directories(htp_iface PUBLIC + "${VIDEO_HEXAGON_SDK}/incs" "${VIDEO_HEXAGON_SDK}/incs/stddef" + "${VIDEO_HEXAGON_SDK}/ipc/fastrpc/rpcmem/inc" + "${VIDEO_HEXAGON_SDK}/utils/examples" "${CMAKE_CURRENT_SOURCE_DIR}/htp" + "${VIDEO_HEXAGON_ROOT}") +target_link_options(htp_iface PUBLIC -llog -ldl) +ggml_add_backend_library(ggml-hexagon ggml-hexagon.cpp htp-drv.cpp htp-drv.h libdl.h ../../include/ggml-hexagon.h) +target_link_libraries(ggml-hexagon PRIVATE htp_iface) +target_include_directories(ggml-hexagon PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/htp" "${VIDEO_HEXAGON_ROOT}") diff --git a/scripts/video/revision b/scripts/video/revision new file mode 100644 index 000000000..8e5956cae --- /dev/null +++ b/scripts/video/revision @@ -0,0 +1 @@ +3f8527a46c54ecf4cb4ed6003da8e8982283c73c diff --git a/scripts/video/video-decode-observer.patch b/scripts/video/video-decode-observer.patch new file mode 100644 index 000000000..c1ffcafee --- /dev/null +++ b/scripts/video/video-decode-observer.patch @@ -0,0 +1,186 @@ +diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h +index 8e5577d..8713281 100644 +--- a/include/stable-diffusion.h ++++ b/include/stable-diffusion.h +@@ -461,10 +461,16 @@ struct ggml_tensor; + typedef void (*sd_log_cb_t)(enum sd_log_level_t level, const char* text, void* data); + typedef void (*sd_progress_cb_t)(int step, int steps, float time, void* data); + typedef void (*sd_preview_cb_t)(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data); ++// The decoded frame is borrowed for the duration of this callback only. ++typedef void (*sd_video_decode_progress_cb_t)(int completed, int total, void* data); ++typedef void (*sd_video_decoded_frame_cb_t)(const sd_image_t* frame, void* data); + typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data); + + SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data); + SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data); ++// Reports final video decode sections and the first complete frame, without an extra decode. ++// total == 0 means the section count is not yet known. Does not report sampling previews. ++SD_API void sd_set_video_decode_callback(sd_video_decode_progress_cb_t progress, sd_video_decoded_frame_cb_t frame, void* data); + // In each sampling pass, a positive interval previews every Nth denoiser step, while a + // negative interval previews only completed logical step -interval. Zero previews the final + // completed step of the first sampling pass (base-resolution or high-noise). +diff --git a/src/core/util.cpp b/src/core/util.cpp +index b90f70e..0fc1aed 100644 +--- a/src/core/util.cpp ++++ b/src/core/util.cpp +@@ -357,6 +357,46 @@ int32_t sd_get_num_physical_cores() { + return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4; + } + ++static sd_video_decode_progress_cb_t sd_decode_progress_cb = nullptr; ++static sd_video_decoded_frame_cb_t sd_decoded_frame_cb = nullptr; ++static void* sd_decode_cb_data = nullptr; ++static thread_local bool sd_video_decode_active = false; ++static thread_local int sd_decode_spatial_completed = 0; ++static thread_local int sd_decode_spatial_total = 0; ++static thread_local int sd_decode_temporal_total = 1; ++ ++SDVideoDecodeScope::SDVideoDecodeScope() : previous(sd_video_decode_active) { ++ sd_video_decode_active = true; ++ sd_video_decode_progress(0, 0); ++} ++SDVideoDecodeScope::~SDVideoDecodeScope() { sd_video_decode_active = previous; } ++void sd_video_decode_progress(int completed, int total) { ++ if (!sd_video_decode_active) return; ++ // A fresh spatial pass also resets the multiplier on an allocation retry. ++ if (completed == 0) sd_decode_temporal_total = 1; ++ sd_decode_spatial_completed = completed; ++ sd_decode_spatial_total = total; ++ if (sd_decode_progress_cb) ++ sd_decode_progress_cb(completed * sd_decode_temporal_total, ++ total * sd_decode_temporal_total, sd_decode_cb_data); ++} ++void sd_video_decode_temporal_progress(int completed, int total) { ++ if (!sd_video_decode_active || total <= 0) return; ++ sd_decode_temporal_total = total; ++ if (sd_decode_progress_cb) ++ sd_decode_progress_cb(sd_decode_spatial_completed * total + completed, ++ std::max(1, sd_decode_spatial_total) * total, sd_decode_cb_data); ++} ++void sd_video_decoded_frame(const sd_image_t* frame) { ++ if (frame && frame->data && sd_decoded_frame_cb) ++ sd_decoded_frame_cb(frame, sd_decode_cb_data); ++} ++void sd_set_video_decode_callback(sd_video_decode_progress_cb_t progress, sd_video_decoded_frame_cb_t frame, void* data) { ++ sd_decode_progress_cb = progress; ++ sd_decoded_frame_cb = frame; ++ sd_decode_cb_data = data; ++} ++ + static sd_progress_cb_t sd_progress_cb = nullptr; + void* sd_progress_cb_data = nullptr; + +@@ -581,6 +621,8 @@ static void print_progress_line(int step, int steps, const std::string& speed_te + } + + void pretty_progress(int step, int steps, float time) { ++ // Video decode has its own section callback; keep it out of sampling progress. ++ if (sd_video_decode_active && sd_decode_progress_cb) return; + if (sd_progress_cb) { + sd_progress_cb(step, steps, time, sd_progress_cb_data); + return; +@@ -600,6 +642,8 @@ + void pretty_bytes_progress(int step, int steps, uint64_t bytes_processed, float elapsed_seconds) { ++ // Loading counts are not sampling steps, even when their totals happen to match. ++ if (sd_decode_progress_cb) return; + if (sd_progress_cb) { + float time = elapsed_seconds / (step + 1e-6f); + sd_progress_cb(step, steps, time, sd_progress_cb_data); + return; + } +diff --git a/src/core/util.h b/src/core/util.h +index 334224d..64f3351 100644 +--- a/src/core/util.h ++++ b/src/core/util.h +@@ -98,6 +98,19 @@ std::vector> parse_prompt_attention(const std::str + std::vector> split_quotation_attention( + const std::vector>& parsed_attention); + ++class SDVideoDecodeScope { ++public: ++ SDVideoDecodeScope(); ++ ~SDVideoDecodeScope(); ++ SDVideoDecodeScope(const SDVideoDecodeScope&) = delete; ++ SDVideoDecodeScope& operator=(const SDVideoDecodeScope&) = delete; ++private: ++ bool previous; ++}; ++void sd_video_decode_progress(int completed, int total); ++void sd_video_decode_temporal_progress(int completed, int total); ++void sd_video_decoded_frame(const sd_image_t* frame); ++ + sd_progress_cb_t sd_get_progress_callback(); + void* sd_get_progress_callback_data(); + +diff --git a/src/model/vae/wan_vae.hpp b/src/model/vae/wan_vae.hpp +index 9dd30fd..e28887b 100644 +--- a/src/model/vae/wan_vae.hpp ++++ b/src/model/vae/wan_vae.hpp +@@ -1500,6 +1500,9 @@ namespace WAN { + free_cache_ctx_and_buffer(); + ae.clear_cache(); + ++ LOG_INFO("Wan VAE decode backend=%s temporal_chunks=%d latent_frames=%lld", ++ ggml_backend_name(runtime_backend), (int)plan.tiles.size(), (long long)input.shape()[2]); ++ sd_video_decode_temporal_progress(0, (int)plan.tiles.size()); + auto output = process_vae_temporal_tiles(input, plan, [&](const sd::Tensor& input_tile, const VAETemporalTile& tile) { + LOG_VERBOSE("Wan VAE temporal tile %d/%d: latent frames [%lld, %lld)", + tile.index + 1, +@@ -1509,9 +1512,11 @@ namespace WAN { + auto get_graph = [&]() -> ggml_cgraph* { + return build_temporal_tile_graph(input_tile, static_cast(tile.start)); + }; +- return restore_trailing_singleton_dims( ++ auto decoded = restore_trailing_singleton_dims( + GGMLRunner::compute(get_graph, n_threads, false), + static_cast(input.dim())); ++ if (!decoded.empty()) sd_video_decode_temporal_progress(tile.index + 1, (int)plan.tiles.size()); ++ return decoded; + }); + + free_cache_ctx_and_buffer(); +diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp +index 2fcff4c..dc0f30f 100644 +--- a/src/pipeline/video.cpp ++++ b/src/pipeline/video.cpp +@@ -1238,7 +1238,11 @@ namespace sd::pipeline { + (int)video_latent.shape()[3]); + // auto z = sd::load_tensor_from_file_as_tensor("ltx_vae_z.bin"); + int64_t t4 = ggml_time_ms(); +- sd::Tensor vid = sd->decode_first_stage(video_latent, true); ++ sd::Tensor vid; ++ { ++ SDVideoDecodeScope decode_scope; ++ vid = sd->decode_first_stage(video_latent, true); ++ } + int64_t t5 = ggml_time_ms(); + LOG_INFO("decode_first_stage completed, taking %.2fs", (t5 - t4) * 1.0f / 1000); + if (vid.empty()) { +@@ -1265,6 +1269,7 @@ namespace sd::pipeline { + + for (int64_t i = 0; i < vid.shape()[2]; i++) { + result_images[i] = tensor_to_sd_image(vid, static_cast(i)); ++ if (i == 0) sd_video_decoded_frame(&result_images[i]); + } + + return result_images; +diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp +index 38ea4c2..771968c 100644 +--- a/src/runtime/tiling.cpp ++++ b/src/runtime/tiling.cpp +@@ -193,6 +193,7 @@ sd::Tensor process_tiles_2d(const sd::Tensor& input, + } + + int num_tiles = num_tiles_x * num_tiles_y; ++ sd_video_decode_progress(0, num_tiles); + int tile_count = 1; + bool last_y = false; + bool last_x = false; +@@ -254,6 +255,7 @@ sd::Tensor process_tiles_2d(const sd::Tensor& input, + last_time = (t2 - t1) / 1000.0f; + pretty_progress(tile_count, num_tiles, last_time); + } ++ sd_video_decode_progress(tile_count, num_tiles); + tile_count++; + } + last_x = false; diff --git a/src/bootstrap/hookRegistry.ts b/src/bootstrap/hookRegistry.ts index cade6e933..de4ef00f0 100644 --- a/src/bootstrap/hookRegistry.ts +++ b/src/bootstrap/hookRegistry.ts @@ -19,6 +19,11 @@ export function registerHook(name: string, fn: HookFn): () => void { }; } +/** Whether a hook is registered (Pro features present and active). */ +export function hasHook(name: string): boolean { + return name in hooks; +} + /** Call a hook if registered; returns its result, or undefined when absent. */ export function callHook(name: string, ...args: any[]): R | undefined { const fn = hooks[name]; @@ -98,4 +103,26 @@ export const HOOKS = { /** (text: string, timestamp: number) => void — core copied text locally. Pro records it through * the shared clipboard owner instead of waiting for a delayed native clipboard notification. */ clipboardRecordLocalText: 'clipboard.recordLocalText', + /** () => Record — Pro features switched on, for the daily analytics + * `features_snapshot` (tts, mcp_tool, sync, clipboard, meetings, browser_action). */ + analyticsProFeatures: 'analytics.proFeatures', + /** Video is Pro. Core asks these through `videoSeam`; free builds register none of them. */ + /** () => boolean — a video job (clip or film) is running. */ + videoIsBusy: 'video.isBusy', + /** () => boolean — a video is running or requested for after the current reply. */ + videoHasPending: 'video.hasPending', + /** () => Promise — stop the running video and drop requested ones. */ + videoCancel: 'video.cancel', + /** (input: { prompt: string; conversationId: string; initImagePath?: string }) => Promise + * — make one clip in the chat. */ + videoGenerate: 'video.generate', + /** (conversationId: string, input: { prompt: string; film?: boolean; startImage?: 'last_image' }) + * => void — the model asked for a clip (generate_video) or a film (make_film); start it after + * this reply. */ + videoDefer: 'video.defer', + /** (conversationId: string, stopped: boolean) => Promise — the reply ended; start the + * requested video unless the person stopped. */ + videoFinishDeferred: 'video.finishDeferred', + /** (conversationId: string) => void — the reply failed; forget the requested video. */ + videoDiscardDeferred: 'video.discardDeferred', } as const; diff --git a/src/bootstrap/slotRegistry.ts b/src/bootstrap/slotRegistry.ts index d78a39851..4f4718cf9 100644 --- a/src/bootstrap/slotRegistry.ts +++ b/src/bootstrap/slotRegistry.ts @@ -95,4 +95,23 @@ export const SLOTS = { modelsScreenVoiceTab: 'modelsScreen.voiceTab', /** Small Pro-owned Voice availability indicator on the core Auto Setup plan. */ autoSetupVoiceIndicator: 'autoSetup.voiceIndicator', + /** Video is Pro. Each video surface below is empty in free builds, where core shows the upgrade + * screen or nothing. */ + /** Body of the Video tab on the Models screen. Props: none. */ + modelsScreenVideoTab: 'modelsScreen.videoTab', + /** Body of the Video tab in the model picker sheet. Props: { onSelect: () => void }. */ + modelSelectorVideoTab: 'modelSelector.videoTab', + /** Video settings in Chat Settings and Model Settings. Props: none. */ + generationSettingsVideo: 'generationSettings.video', + /** Live status of a running video under the chat or the Gallery. + * Props: { conversationId?: string | null }. */ + videoStatus: 'video.status', + /** Actions under a finished clip or film in chat (Continue, New shot, Make it a story). + * Props: { message: Message; conversationId: string }. */ + messageVideoActions: 'message.videoActions', + /** "Animate this photo" on an image. Props: { imagePath: string; conversationId?: string; + * onStarted?: () => void }. */ + imageAnimateAction: 'image.animateAction', + /** Films and characters at the top of the Gallery. Props: none. */ + galleryVideoSection: 'gallery.videoSection', } as const; diff --git a/src/bootstrap/videoSeam.ts b/src/bootstrap/videoSeam.ts new file mode 100644 index 000000000..ac48e18d3 --- /dev/null +++ b/src/bootstrap/videoSeam.ts @@ -0,0 +1,34 @@ +import { callHook, hasHook, HOOKS } from './hookRegistry'; + +/** + * Core's only way to reach video. Video is Pro: Pro registers the hooks, and free builds (or Pro + * that ended) get the quiet defaults here - nothing is busy, nothing is pending, nothing to stop. + */ +export const videoSeam = { + available: (): boolean => hasHook(HOOKS.videoGenerate), + isBusy: (): boolean => callHook(HOOKS.videoIsBusy) === true, + hasPending: (): boolean => callHook(HOOKS.videoHasPending) === true, + cancel: (): Promise => + callHook>(HOOKS.videoCancel) ?? Promise.resolve(), + generate: (input: { + prompt: string; + conversationId: string; + initImagePath?: string; + }): Promise => + callHook>(HOOKS.videoGenerate, input) ?? + Promise.reject(new Error('Video needs Off Grid AI Pro.')), + defer: ( + conversationId: string, + input: { prompt: string; film?: boolean; startImage?: 'last_image' }, + ): void => { + if (!hasHook(HOOKS.videoDefer)) + throw new Error('Video needs Off Grid AI Pro.'); + callHook(HOOKS.videoDefer, conversationId, input); + }, + finishDeferred: (conversationId: string, stopped: boolean): Promise => + callHook>(HOOKS.videoFinishDeferred, conversationId, stopped) ?? + Promise.resolve(), + discardDeferred: (conversationId: string): void => { + callHook(HOOKS.videoDiscardDeferred, conversationId); + }, +}; diff --git a/src/components/AnalyticsConsentPrompt.tsx b/src/components/AnalyticsConsentPrompt.tsx new file mode 100644 index 000000000..070d661f6 --- /dev/null +++ b/src/components/AnalyticsConsentPrompt.tsx @@ -0,0 +1,40 @@ +import React from 'react'; +import { CenteredAlert } from './CenteredAlert'; +import { + closeConsentPrompt, + useConsentPromptStore, +} from '../services/analytics/consentPrompt'; +import { setAnalyticsConsent } from '../services/analytics/analytics'; +import { ignore } from '../services/analytics/values'; + +const COPY = { + failure: { title: 'Help us fix this?', source: 'failure_prompt' }, + first_chat: { title: 'Help us fix what breaks', source: 'first_chat_prompt' }, +} as const; + +const BODY = 'Send anonymous crash reports and usage. Your chats stay on this device.'; + +/** The one consent prompt shown outside Auto Setup. Mounted once at the app root. */ +export const AnalyticsConsentPrompt: React.FC = () => { + const kind = useConsentPromptStore(s => s.kind); + if (!kind) return null; + const copy = COPY[kind]; + return ( + { + setAnalyticsConsent({ usage: true, crashes: true }, copy.source).catch(ignore); + }, + }, + ]} + /> + ); +}; diff --git a/src/components/Button.tsx b/src/components/Button.tsx index 47517b4bd..e4c2d3037 100644 --- a/src/components/Button.tsx +++ b/src/components/Button.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { TouchableOpacity, Text, ViewStyle, TextStyle } from 'react-native'; -import type { AccessibilityState } from 'react-native'; +import type { AccessibilityRole, AccessibilityState } from 'react-native'; import { useTheme, useThemedStyles } from '../theme'; import type { ThemeColors, ThemeShadows } from '../theme'; import { SPACING, TYPOGRAPHY } from '../constants'; @@ -24,6 +24,7 @@ interface ButtonProps { testID?: string; accessibilityLabel?: string; accessibilityState?: AccessibilityState; + accessibilityRole?: AccessibilityRole; } export const Button: React.FC = ({ @@ -40,6 +41,7 @@ export const Button: React.FC = ({ testID, accessibilityLabel, accessibilityState, + accessibilityRole = 'button', }) => { const { colors } = useTheme(); const styles = useThemedStyles(createStyles); @@ -69,7 +71,7 @@ export const Button: React.FC = ({ disabled={disabled || loading} activeOpacity={0.7} testID={testID} - accessibilityRole="button" + accessibilityRole={accessibilityRole} accessibilityLabel={accessibilityLabel ?? title} accessibilityState={accessibilityState} > diff --git a/src/components/ChatInput/Popovers.tsx b/src/components/ChatInput/Popovers.tsx index 8673e95e4..09b95893b 100644 --- a/src/components/ChatInput/Popovers.tsx +++ b/src/components/ChatInput/Popovers.tsx @@ -77,6 +77,7 @@ interface QuickSettingsPopoverProps { anchorX: number; imageMode: ImageModeState; onImageModeToggle: () => void; + onVideoModeToggle?: () => void; imageModelLoaded: boolean; supportsThinking: boolean; supportsToolCalling: boolean; @@ -88,7 +89,8 @@ interface QuickSettingsPopoverProps { function getImageModeBadge(mode: ImageModeState, colors: any) { if (mode === 'force') return { label: 'ON', bg: colors.primary }; - if (mode === 'disabled') return { label: 'OFF', bg: colors.textMuted }; + if (mode === 'disabled' || mode === 'video') + return { label: 'OFF', bg: colors.textMuted }; return { label: 'Auto', bg: `${colors.textMuted}80` }; } @@ -111,10 +113,20 @@ function getToolsStyle(supported: boolean, count: number, colors: any) { } export const QuickSettingsPopover: React.FC = ({ - visible, onClose, anchorY, anchorX, - imageMode, onImageModeToggle, imageModelLoaded, supportsThinking, - supportsToolCalling, enabledToolCount, onToolsPress, - mcpToolCount = 0, onMcpPress, + visible, + onClose, + anchorY, + anchorX, + imageMode, + onImageModeToggle, + onVideoModeToggle, + imageModelLoaded, + supportsThinking, + supportsToolCalling, + enabledToolCount, + onToolsPress, + mcpToolCount = 0, + onMcpPress, }) => { const { colors } = useTheme(); const thinkingEnabled = useAppStore(state => state.settings.thinkingEnabled); @@ -139,7 +151,12 @@ export const QuickSettingsPopover: React.FC = ({ const mcpBadgeBg = showMcpWarning ? TOOL_WARNING_COLOR : mcpDefaultBg; return ( - + {/* accessible={false} on the SCRIM too. The inner wrapper was fixed first, but this outer dismiss layer wraps the whole popover and merges it just the same - iOS reported one control named ", Image Gen, Auto, , Thinking, ON, , Voice, Chat, , Tools, 1, Pro Tools, 6" @@ -155,21 +172,92 @@ export const QuickSettingsPopover: React.FC = ({ reach one setting. Android exposes the rows individually; this is what made iOS differ. */} - + { triggerHaptic('impactLight'); onImageModeToggle(); }} + onPress={() => { + triggerHaptic('impactLight'); + onImageModeToggle(); + }} + > + + + Image Gen + + + + {imgBadge.label} + + + + + { + triggerHaptic('impactLight'); + onVideoModeToggle?.(); + }} > - - Image Gen - - {imgBadge.label} + + + Video Gen + + + + {imageMode === 'video' ? 'ON' : 'OFF'} + @@ -182,12 +270,32 @@ export const QuickSettingsPopover: React.FC = ({ updateSettings({ thinkingEnabled: !thinkingEnabled }); }} > - - Thinking - - + + + Thinking + + + {thinkingEnabled ? 'ON' : 'OFF'} @@ -206,9 +314,25 @@ export const QuickSettingsPopover: React.FC = ({ }} > - Tools - - {tools.badgeLabel} + + Tools + + + + {tools.badgeLabel} + @@ -221,11 +345,29 @@ export const QuickSettingsPopover: React.FC = ({ onMcpPress?.(); }} > - - Pro Tools + + + Pro Tools + {mcpToolCount > 0 && ( - - {mcpToolCount} + + + {mcpToolCount} + )} @@ -250,15 +392,25 @@ interface AttachPickerPopoverProps { } export const AttachPickerPopover: React.FC = ({ - visible, onClose, anchorY, anchorX, - supportsVision, onPhoto, onDocument, + visible, + onClose, + anchorY, + anchorX, + supportsVision, + onPhoto, + onDocument, }) => { const { colors } = useTheme(); if (!visible) return null; return ( - + {/* accessible={false} on the SCRIM too. The inner wrapper was fixed first, but this outer dismiss layer wraps the whole popover and merges it just the same - iOS reported one control named ", Image Gen, Auto, , Thinking, ON, , Voice, Chat, , Tools, 1, Pro Tools, 6" @@ -274,24 +426,41 @@ export const AttachPickerPopover: React.FC = ({ reach one setting. Android exposes the rows individually; this is what made iOS differ. */} - + { onClose(); onPhoto(); }} + onPress={() => { + onClose(); + onPhoto(); + }} > - - Photo + + + Photo + { onClose(); onDocument(); }} + onPress={() => { + onClose(); + onDocument(); + }} > Document diff --git a/src/components/ChatInput/Voice.ts b/src/components/ChatInput/Voice.ts index 3c5087641..967e004f2 100644 --- a/src/components/ChatInput/Voice.ts +++ b/src/components/ChatInput/Voice.ts @@ -69,6 +69,27 @@ function createWhisperReadiness( }); } +/** How long a one-line transcription notice stays in the composer. */ +const NOTICE_MS = 5000; + +/** The latest one-line notice about a transcription (a Qwen3-ASR failure or its Whisper fallback). */ +function useTranscriptionNotice(): string | null { + const [notice, setNotice] = useState(null); + useEffect(() => { + let timer: ReturnType | undefined; + const unsubscribe = whisperService.onNotice(message => { + setNotice(message); + if (timer) clearTimeout(timer); + timer = setTimeout(() => setNotice(null), NOTICE_MS); + }); + return () => { + unsubscribe(); + if (timer) clearTimeout(timer); + }; + }, []); + return notice; +} + function useRemoteTranscriptionAvailable(): boolean { return useRemoteServerStore(state => { const server = state.servers.find(item => item.id === state.activeRemoteMediaServerIds.transcription); @@ -124,6 +145,7 @@ export function useVoiceInput({ conversationId, interfaceMode, onTranscript, onA const [isTranscribingFile, setIsTranscribingFile] = useState(false); const [directError, setDirectError] = useState(null); + const transcriptionNotice = useTranscriptionNotice(); const supportsDirectAudio = (): boolean => activeModelService.supportsAudioInput() && audioRecorderService.supportsDirectAudioInput(); @@ -448,6 +470,7 @@ export function useVoiceInput({ conversationId, interfaceMode, onTranscript, onA isTranscribing, partialResult, error, + transcriptionNotice, voiceAvailable, // INTENTS, not mechanics: the controller's registered handlers own the session decisions // (userStart out of stopped, userStop on a deliberate hands-free stop). Handing out the raw diff --git a/src/components/ChatInput/index.tsx b/src/components/ChatInput/index.tsx index 09a8140be..c5acecacd 100644 --- a/src/components/ChatInput/index.tsx +++ b/src/components/ChatInput/index.tsx @@ -18,8 +18,11 @@ import { buildVoiceNoteHandlers } from './voiceNoteSend'; import { QuickSettingsPopover, AttachPickerPopover } from './Popovers'; import { useKeyboardAwarePopover } from './useKeyboardAwarePopover'; import { useAppStore } from '../../stores'; +import { useRemoteServerStore } from '../../stores/remoteServerStore'; +import { remoteServerCapabilities } from '../../types/remoteServer'; import { useUiModeStore } from '../../stores'; import { getSlot, SLOTS } from '../../bootstrap/slotRegistry'; +import { videoSeam } from '../../bootstrap/videoSeam'; import { AppSheet } from '../AppSheet'; import { Button } from '../Button'; @@ -36,6 +39,10 @@ interface ChatInputProps { conversationId?: string | null; imageModelLoaded?: boolean; onImageModeChange?: (mode: ImageModeState) => void; + /** Video is Pro. Called instead of turning video on when Pro is not active. */ + onVideoUpgrade?: () => void; + /** Opens the screen where a computer is paired. Offered when no video model is reachable. */ + onFindComputer?: () => void; onOpenSettings?: () => void; queue?: QueueView; queueActions?: QueueActions; @@ -120,7 +127,15 @@ const buildNoVisionAlert = (opts: { [ { text: 'Cancel', onPress: opts.dismiss }, ...(opts.onRepairVision - ? [{ text: 'Go to Download Manager', onPress: () => { opts.dismiss(); opts.onRepairVision!(); } }] + ? [ + { + text: 'Go to Download Manager', + onPress: () => { + opts.dismiss(); + opts.onRepairVision!(); + }, + }, + ] : [{ text: 'OK' }]), ], ); @@ -145,6 +160,8 @@ export const ChatInput: React.FC = ({ conversationId, imageModelLoaded = false, onImageModeChange, + onVideoUpgrade, + onFindComputer, onOpenSettings: _onOpenSettings, queue, queueActions, @@ -222,13 +239,27 @@ export const ChatInput: React.FC = ({ addAudioAttachment, clearAttachments, onHaptic: () => triggerHaptic('impactMedium'), - appendTranscript: (text) => setMessage(prev => { - const prefix = prev.trim() ? `${prev.trim()} ` : ''; - return prefix + text; - }), + appendTranscript: text => + setMessage(prev => { + const prefix = prev.trim() ? `${prev.trim()} ` : ''; + return prefix + text; + }), }); - const { isRecording, isModelLoading, isStartingRecording, isTranscribing, partialResult, error, voiceAvailable, isAwaitingSpeech, startRecording, stopRecording, cancelRecording } = useVoiceInput({ + const { + isRecording, + isModelLoading, + isStartingRecording, + isTranscribing, + partialResult, + error, + transcriptionNotice, + voiceAvailable, + isAwaitingSpeech, + startRecording, + stopRecording, + cancelRecording, + } = useVoiceInput({ conversationId, interfaceMode, onTranscript: voiceHandlers.onTranscript, @@ -281,7 +312,7 @@ export const ChatInput: React.FC = ({ setMessage(''); clearAttachments(); inputRef.current?.focus(); - if (imageMode === 'force') { + if (imageMode === 'force' || imageMode === 'video') { setImageMode('auto'); onImageModeChange?.('auto'); } @@ -305,6 +336,61 @@ export const ChatInput: React.FC = ({ onImageModeChange?.(newMode); }; + const handleVideoModeToggle = () => { + if (imageMode !== 'video' && !videoSeam.available()) { + quickSettings.hide(); + onVideoUpgrade?.(); + return; + } + if (imageMode !== 'video') { + const { activeVideoModelId, downloadedVideoModels, setActiveVideoModelId } = + useAppStore.getState(); + const remoteVideoId = + useRemoteServerStore.getState().activeRemoteMediaServerIds.video; + const localVideoSelected = downloadedVideoModels.some( + model => model.id === activeVideoModelId, + ); + if (!remoteVideoId && !localVideoSelected) { + if (downloadedVideoModels.length === 0) { + const remote = useRemoteServerStore.getState(); + const computer = [ + ...remote.servers.filter(server => server.id === remote.activeServerId), + ...remote.servers, + ].find(server => remoteServerCapabilities(server).videoGeneration); + if (computer) { + remote.setActiveRemoteMediaServerId('video', computer.id); + } else { + setAlertState( + showAlert( + 'Videos are made on your computer', + 'Find your computer running Off Grid AI Desktop, and videos are made there.', + onFindComputer + ? [ + { text: 'Not now' }, + { + text: 'Find my computer', + onPress: () => { + setAlertState(hideAlert()); + onFindComputer(); + }, + }, + ] + : [{ text: 'OK' }], + ), + ); + quickSettings.hide(); + return; + } + } else { + setActiveVideoModelId(downloadedVideoModels[0].id); + } + } + } + const newMode: ImageModeState = imageMode === 'video' ? 'auto' : 'video'; + setImageMode(newMode); + onImageModeChange?.(newMode); + }; + const handleVisionPress = () => { if (!supportsVision) { setAlertState(buildNoVisionAlert({ isRemote, needsRepair: visionNeedsRepair, onRepairVision, dismiss: () => setAlertState(hideAlert()) })); @@ -343,26 +429,27 @@ export const ChatInput: React.FC = ({ }, ANIM_DURATION_IN); }; - const assistantGate = assistantAvailability === 'no-pro' - ? { - title: 'Assistant requires Pro', - body: 'Assistant uses Web Use and Computer Use on a connected Desktop.', - action: 'View Pro', - onPress: onAssistantUpgrade, - } - : assistantAvailability === 'unavailable' + const assistantGate = + assistantAvailability === 'no-pro' ? { - title: 'Desktop tasks unavailable', - body: 'This Desktop is connected, but Web Use and Computer Use are not available. Allow remote tasks on the Desktop and try again.', - action: 'OK', - onPress: undefined, + title: 'Assistant requires Pro', + body: 'Assistant uses Web Use and Computer Use on a connected Desktop.', + action: 'View Pro', + onPress: onAssistantUpgrade, } - : { - title: 'Connect a Desktop', - body: 'Set up Sync and connect Off Grid AI Desktop to use Web Use and Computer Use.', - action: 'Set up Sync', - onPress: onAssistantSetupSync, - }; + : assistantAvailability === 'unavailable' + ? { + title: 'Desktop tasks unavailable', + body: 'This Desktop is connected, but Web Use and Computer Use are not available. Allow remote tasks on the Desktop and try again.', + action: 'OK', + onPress: undefined, + } + : { + title: 'Connect a Desktop', + body: 'Set up Sync and connect Off Grid AI Desktop to use Web Use and Computer Use.', + action: 'Set up Sync', + onPress: onAssistantSetupSync, + }; const handleAttachPress = () => { logger.log(`[COMPOSER-SM] attach pressed platform=${Platform.OS} supportsVision=${supportsVision}`); @@ -372,7 +459,7 @@ export const ChatInput: React.FC = ({ : ['Document', 'Cancel']; ActionSheetIOS.showActionSheetWithOptions( { options, cancelButtonIndex: options.length - 1 }, - (index) => { + index => { if (supportsVision) { if (index === 0) handleVisionPress(); else if (index === 1) handlePickDocument(); @@ -420,6 +507,7 @@ export const ChatInput: React.FC = ({ onCancelRecording={cancelRecording} onStop={onStop} onImageModeToggle={handleImageModeToggle} + onVideoModeToggle={handleVideoModeToggle} onThinkingToggle={handleThinkingToggle} onToolsPress={onToolsPress} onMcpPress={onMcpPress} @@ -498,6 +586,11 @@ export const ChatInput: React.FC = ({ + {transcriptionNotice ? ( + + {transcriptionNotice} + + ) : null} {showVoiceStatus ? ( @@ -514,7 +607,10 @@ export const ChatInput: React.FC = ({ size="small" testID="assistant-toggle" accessibilityLabel="Assistant" - accessibilityState={{ selected: assistantEnabled, busy: assistantTransitioning }} + accessibilityState={{ + selected: assistantEnabled, + busy: assistantTransitioning, + }} onPress={handleAssistantPress} active={assistantEnabled} loading={assistantTransitioning} @@ -533,7 +629,9 @@ export const ChatInput: React.FC = ({ style={styles.pillInput} value={message} onChangeText={setMessage} - placeholder={placeholder} + placeholder={ + imageMode === 'video' ? 'Describe a video' : placeholder + } placeholderTextColor={colors.textMuted} multiline scrollEnabled @@ -578,6 +676,7 @@ export const ChatInput: React.FC = ({ anchorX={quickSettings.anchor.x} imageMode={imageMode} onImageModeToggle={handleImageModeToggle} + onVideoModeToggle={handleVideoModeToggle} imageModelLoaded={imageModelLoaded} supportsThinking={supportsThinking} supportsToolCalling={supportsToolCalling} diff --git a/src/components/ChatInput/styles.ts b/src/components/ChatInput/styles.ts index ec06dd3c3..0b39b75f7 100644 --- a/src/components/ChatInput/styles.ts +++ b/src/components/ChatInput/styles.ts @@ -136,6 +136,11 @@ export const createStyles = (colors: ThemeColors, shadows: ThemeShadows) => ({ padding: 4, }, // Main input row (pill + circular button) + transcriptionNotice: { + ...TYPOGRAPHY.bodySmall, + color: colors.textSecondary, + marginBottom: SPACING.xs, + }, mainRow: { flexDirection: 'row' as const, alignItems: 'center' as const, diff --git a/src/components/ChatMessage/components/GenerationMeta.tsx b/src/components/ChatMessage/components/GenerationMeta.tsx index 6abd19f30..bf044ebd2 100644 --- a/src/components/ChatMessage/components/GenerationMeta.tsx +++ b/src/components/ChatMessage/components/GenerationMeta.tsx @@ -33,6 +33,9 @@ function formatOptionalMeta(meta: NonNullable, tps: n ['steps', m.steps == null ? undefined : `${m.steps} steps`], ['cfg', m.guidanceScale == null ? undefined : `cfg ${m.guidanceScale}`], ['res', m.resolution], + ['frames', m.frames == null ? undefined : `${m.frames} frames`], + ['fps', m.fps == null ? undefined : `${m.fps} fps`], + ['seed', m.seed == null ? undefined : `Seed: ${m.seed}`], ['cache', m.cacheType ? `KV ${m.cacheType}` : undefined], ]; return entries diff --git a/src/components/ChatMessage/components/MessageAttachments.tsx b/src/components/ChatMessage/components/MessageAttachments.tsx index 619639f62..e4f0489a7 100644 --- a/src/components/ChatMessage/components/MessageAttachments.tsx +++ b/src/components/ChatMessage/components/MessageAttachments.tsx @@ -10,8 +10,14 @@ import Animated, { useSharedValue, useAnimatedStyle, withTiming, + useReducedMotion, } from 'react-native-reanimated'; import Icon from 'react-native-vector-icons/Feather'; +import VideoPlayer, { type VideoPlayerRef } from 'react-native-video-player'; +import { ResizeMode } from 'react-native-video'; +import { useTheme, useThemedStyles, type ThemeColors, type ThemeShadows } from '../../../theme'; +import { SPACING, TYPOGRAPHY } from '../../../constants'; +import { Button } from '../../Button'; // Imported directly, not through the barrel: a component that reaches its sibling via the index // resolves undefined at render time. import { LoadingDots } from '../../LoadingDots'; @@ -26,13 +32,14 @@ interface FadeInImageProps { testID?: string; wrapperTestID?: string; onPress?: () => void; + accessibilityLabel?: string; } function resolveMediaUri(uri: string): string { return uri.includes('/Documents/') ? `file://${resolveDocumentPath(uri)}` : uri; } -function FadeInImage({ uri, imageStyle, testID, wrapperTestID, onPress }: FadeInImageProps) { +export function FadeInImage({ uri, imageStyle, testID, wrapperTestID, onPress, accessibilityLabel }: FadeInImageProps) { const displayUri = resolveMediaUri(uri); const opacity = useSharedValue(0); const [loaded, setLoaded] = React.useState(false); @@ -44,10 +51,11 @@ function FadeInImage({ uri, imageStyle, testID, wrapperTestID, onPress }: FadeIn testID={wrapperTestID} style={fadeInImageStyles.wrapper} onPress={onPress} + disabled={!onPress} activeOpacity={0.8} - accessibilityRole="button" + accessibilityRole={onPress ? 'button' : 'image'} accessibilityLabel={ - isGeneratedImage ? `Generated image ${loaded ? 'loaded' : 'loading'}` : undefined + accessibilityLabel ?? (isGeneratedImage ? `Generated image ${loaded ? 'loaded' : 'loading'}` : undefined) } > ({ + container: { + width: '100%' as const, + gap: SPACING.xs, + padding: SPACING.sm, + borderRadius: 8, + backgroundColor: colors.surface, + ...shadows.small, + }, + pictureInPicture: { width: SPACING.xl * 2, height: SPACING.xl * 2, paddingHorizontal: 0 }, +}); + const fadeInImageStyles = StyleSheet.create({ wrapper: { borderRadius: 12, @@ -98,6 +119,85 @@ interface MessageAttachmentsProps { onImagePress?: (uri: string) => void; } +/** The library owns playback and controls; this adapter supplies attachment data and theme. */ +function VideoAttachment({ attachment }: { attachment: MediaAttachment }) { + const { colors } = useTheme(); + const videoAttachmentStyles = useThemedStyles(createVideoAttachmentStyles); + const player = React.useRef(null); + const reducedMotion = useReducedMotion(); + const [size, setSize] = React.useState({ + width: attachment.width || 16, + height: attachment.height || 9, + }); + const [loaded, setLoaded] = React.useState(false); + const [error, setError] = React.useState(null); + return ( + + } + style={videoAttachmentStyles.pictureInPicture} + onPress={async () => { + try { + await player.current?.enterPictureInPicture(); + } catch { + setError('Picture in picture is not available on this device.'); + } + }} + /> + } + pauseOnPress + animationDuration={reducedMotion ? 0 : 150} + playInBackground + playWhenInactive + ignoreSilentSwitch="ignore" + enterPictureInPictureOnLeave + onRestoreUserInterfaceForPictureInPictureStop={() => { + player.current?.restoreUserInterfaceForPictureInPictureStopCompleted(true); + }} + onLoad={({ naturalSize }) => { + setLoaded(true); + setError(null); + if (naturalSize.width > 0 && naturalSize.height > 0) { + setSize({ width: naturalSize.width, height: naturalSize.height }); + } + }} + onError={() => setError('This video could not be played.')} + customStyles={{ + wrapper: { width: '100%', borderRadius: SPACING.sm, overflow: 'hidden' }, + controls: { backgroundColor: colors.surface, height: SPACING.xl * 2, marginTop: 0 }, + controlButton: { width: SPACING.xl * 2, height: SPACING.xl * 2, padding: SPACING.sm, alignItems: 'center', justifyContent: 'center' }, + controlIcon: { tintColor: colors.text, width: SPACING.lg, height: SPACING.lg }, + playArrow: { tintColor: colors.text, width: SPACING.lg, height: SPACING.lg, marginLeft: 0 }, + playButton: { backgroundColor: colors.surface, width: SPACING.xl * 2, height: SPACING.xl * 2, borderRadius: SPACING.sm }, + seekBar: { flex: 1, minWidth: 0, paddingHorizontal: SPACING.xs, marginLeft: 0, marginRight: SPACING.sm }, + seekBarProgress: { backgroundColor: colors.primary }, + seekBarKnob: { backgroundColor: colors.primary, width: SPACING.sm, height: SPACING.sm, marginHorizontal: -SPACING.xs, marginVertical: 0 }, + seekBarBackground: { backgroundColor: colors.border }, + durationText: { ...TYPOGRAPHY.meta, color: colors.textSecondary, flexShrink: 0 }, + }} + /> + {error && {error}} + + + ); +} + function AudioAttachment({ index, isUser, @@ -197,6 +297,8 @@ export function MessageAttachments({ styles={styles} colors={colors} /> + ) : attachment.type === 'video' ? ( + ) : attachment.type === 'audio' ? ( = ({ onLongPress, onMenuOpen, }) => { + const savedVideo = useAppStore(state => + message.role === 'assistant' + ? state.generatedVideos.find(video => + message.attachments?.some(attachment => + attachment.type === 'video' && attachment.id === video.id, + ), + ) + : undefined, + ); + const generationMeta = savedVideo + ? { ...videoGenerationMeta(savedVideo), ...message.generationMeta } + : message.generationMeta; const timelineHasThinking = Boolean( message.timeline?.some(entry => entry.kind === 'thinking'), ); @@ -237,6 +252,18 @@ const MessageBubble: React.FC = ({ const answerParsedContent = hasAssistantWork ? { ...parsedContent, thinking: '' } : parsedContent; + const isGeneratedVideo = Boolean( + !isUser && + message.turnKind === 'video' && + message.attachments?.length && + message.attachments.every(attachment => attachment.type === 'video'), + ); + // Video is Pro: Continue / New shot / Make it a story, and film cards, fill this slot. + const VideoActions = useSlot(SLOTS.messageVideoActions); + const showVideoActions = Boolean( + VideoActions && !isUser && !isStreaming && (isGeneratedVideo || message.videoStoryId), + ); + // With Pro, the slot draws a generated clip as its own card; core's player is the free fallback. const hasVisibleAnswer = Boolean( hasAttachments || answerParsedContent.response.trim(), ); @@ -321,10 +348,16 @@ const MessageBubble: React.FC = ({ )} - {(!hideProse || hasAttachments) && showAnswerBubble && ( + {(!hideProse || hasAttachments) && showAnswerBubble && !(showVideoActions && isGeneratedVideo) && ( {hasAttachments && ( = ({ /> )} - + {!isGeneratedVideo && ( + + )} )} + {showVideoActions && VideoActions && } + {!message.isThinking && !isStreaming && !hideProse && hasVisibleAnswer && ( = ({ )} - {showTurnFooter && showGenerationDetails && message.generationMeta && ( + {showTurnFooter && showGenerationDetails && generationMeta && ( diff --git a/src/components/ChatMessage/styles.ts b/src/components/ChatMessage/styles.ts index 56ed186ff..e0cd56784 100644 --- a/src/components/ChatMessage/styles.ts +++ b/src/components/ChatMessage/styles.ts @@ -38,6 +38,10 @@ const createBubbleStyles = (colors: ThemeColors, shadows: ThemeShadows) => ({ width: MESSAGE_MAX_WIDTH, alignSelf: 'flex-start' as const, }, + videoMessage: { + width: MESSAGE_MAX_WIDTH, + alignSelf: 'flex-start' as const, + }, toolCallPreText: { alignSelf: 'flex-start' as const, paddingBottom: 6, diff --git a/src/components/DebugLogsScreen/index.tsx b/src/components/DebugLogsScreen/index.tsx index db6ebd35e..0691671c8 100644 --- a/src/components/DebugLogsScreen/index.tsx +++ b/src/components/DebugLogsScreen/index.tsx @@ -24,9 +24,11 @@ interface DebugLogsScreenProps { visible: boolean; onClose: () => void; syncOnly?: boolean; + /** Report a problem: the person reads the log here, then sends it to support. */ + onSend?: () => void; } -export const DebugLogsScreen: React.FC = ({ visible, onClose, syncOnly = false }) => { +export const DebugLogsScreen: React.FC = ({ visible, onClose, syncOnly = false, onSend }) => { const theme = useTheme(); const styles = useThemedStyles(createStyles); const { logs, clearLogs } = useDebugLogsStore(); @@ -106,7 +108,7 @@ export const DebugLogsScreen: React.FC = ({ visible, onClo {syncOnly ? 'Sync Debug Logs' : 'Debug Logs'} {visibleLogs.length} log entries - + @@ -129,6 +131,20 @@ export const DebugLogsScreen: React.FC = ({ visible, onClo )} + {onSend && ( + + + + Send to support + + + )} + {/* Action Buttons */} @@ -141,7 +157,8 @@ export const DebugLogsScreen: React.FC = ({ visible, onClo Share - {!syncOnly && + {/* Report a problem sends the log file; clearing only this view would hide what is sent. */} + {!syncOnly && !onSend && Clear } diff --git a/src/components/GenerationSettingsModal/ImageGenerationSection.tsx b/src/components/GenerationSettingsModal/ImageGenerationSection.tsx index 08130a395..cd46a39ae 100644 --- a/src/components/GenerationSettingsModal/ImageGenerationSection.tsx +++ b/src/components/GenerationSettingsModal/ImageGenerationSection.tsx @@ -208,7 +208,9 @@ export const ImageGenerationSection: React.FC = () => { - + + + diff --git a/src/components/GenerationSettingsModal/ImageQualitySliders.tsx b/src/components/GenerationSettingsModal/ImageQualitySliders.tsx index f0c6607e9..b6853ceae 100644 --- a/src/components/GenerationSettingsModal/ImageQualitySliders.tsx +++ b/src/components/GenerationSettingsModal/ImageQualitySliders.tsx @@ -1,12 +1,11 @@ import React from 'react'; import { View, Text, Switch, Platform, TouchableOpacity } from 'react-native'; +import { Button } from '../Button'; import { SliderSetting } from '../SliderSetting'; import { useTheme, useThemedStyles } from '../../theme'; import { useAppStore } from '../../stores'; -import { DEFAULT_SETTINGS } from '../../stores/appStore'; -import { useClearGpuCache } from '../../hooks/useImageGenerationSettings'; +import { useClearGpuCache, useImageParameterSettings } from '../../hooks/useImageGenerationSettings'; import { - defaultImageSteps, MAX_IMAGE_STEPS, SWEET_SPOT_SIZE, } from '../../utils/imageGenAdvice'; @@ -32,7 +31,7 @@ const ClearGPUCacheButton: React.FC = () => { /** Basic controls: Image Steps + Image Size */ export const ImageQualityBasicSliders: React.FC = () => { - const { settings, updateSettings } = useAppStore(); + const { parameters, defaults, maxSize, applyDefaults, setParameter } = useImageParameterSettings(); return ( <> @@ -41,27 +40,29 @@ export const ImageQualityBasicSliders: React.FC = () => { compact label="Image Steps" description="4-8 steps for speed, 20-50 for quality" - value={settings.imageSteps || defaultImageSteps(Platform.OS)} + value={parameters.steps} min={4} max={MAX_IMAGE_STEPS} step={1} - onChange={(value) => updateSettings({ imageSteps: value })} + onChange={(value) => setParameter('steps', value)} /> `${v}x${v}`} - onChange={(value) => updateSettings({ imageWidth: value, imageHeight: value })} + onChange={(value) => setParameter('size', value)} /> +