From 8d0978419381e4cf41412d2fcb530eb418c9f218 Mon Sep 17 00:00:00 2001 From: Aizat Hawari Date: Mon, 28 Sep 2026 10:04:57 +0100 Subject: [PATCH 1/5] feat: external API, saved directory maps, and import progress External API - The Settings page's External API card sets memoryProvider, memoryApiUrl, memoryModel, and memoryApiKey. The key comes from an environment variable, an existing key file, or a pasted key saved to ~/.config/omms/secrets/.key, readable only by the user. A literal key is never written to the config. A Test button makes one small call. - `external` is a new value for opencodeModel, piModel, opencodeBackfillModel, and piBackfillModel. The rule lives only in live-model-choice.ts and is the same on both hosts. - An env:// or file:// key that does not resolve in a process counts as not set, instead of stopping the config from loading. Directory maps - importPathMaps is a global list of saved directory maps. Every import surface uses it; a run's --map wins for the same folder. - The page lists unresolved directories with session counts and suggests an existing target. Import progress - Every model-calling import records its progress in import_runs and takes the host's lock, so a backfill, a web import, a CLI import, and a slash command cannot overlap. Progress counts only work that needs a model call. - The page shows a progress bar, percentage, and minutes left, with Run now, Pause, and Resume for each host's backfill. A pause survives host starts. Run now works in the login web app with the external API. Also - om-memory-system --version and -v; the Web app section compares the running version with the global command. - google-gemini is selectable on the External API card. - Web UI: keyword badges and keyword filter, tooltips on Cleanup and Deduplicate, collapsible sidebar with profile sections, a shared Select component, and restyled diagnostics tables. OpenSpec change: external-api-backfill-maps-progress. --- .../.openspec.yaml | 2 + .../design.md | 97 ++++++ .../proposal.md | 45 +++ .../specs/auto-backfill/spec.md | 65 ++++ .../specs/host-neutral-memory-core/spec.md | 43 +++ .../specs/import-directory-maps/spec.md | 54 ++++ .../specs/import-progress/spec.md | 65 ++++ .../specs/web-autostart/spec.md | 61 ++++ .../specs/web-settings/spec.md | 160 ++++++++++ .../tasks.md | 58 ++++ package.json | 5 + src/adapters/opencode/backfill-models.ts | 7 +- src/adapters/opencode/backfill-startup.ts | 10 + src/adapters/opencode/import-command.ts | 1 + src/adapters/pi/backfill-models.ts | 10 +- src/adapters/pi/extension.ts | 10 + src/adapters/pi/import-command.ts | 1 + src/cli/index.ts | 14 +- src/config.ts | 35 +- src/importer/auto-backfill.ts | 50 ++- src/importer/backfill-controls.ts | 135 ++++++++ src/importer/backfill-lock.ts | 4 +- src/importer/backfill-model.ts | 3 +- src/importer/external-api-test.ts | 63 ++++ src/importer/external-backfill-models.ts | 28 ++ src/importer/import-path-maps.ts | 58 ++++ src/importer/import-progress.ts | 66 ++++ src/importer/import-runs.ts | 207 ++++++++++++ src/importer/import-sessions.ts | 20 +- src/importer/importer.ts | 15 +- src/importer/map-suggestions.ts | 141 ++++++++ src/importer/opencode-import.ts | 8 +- src/importer/run-import.ts | 107 ++++++- src/importer/web-import-api.ts | 3 + src/importer/web-import-jobs.ts | 1 + src/index.ts | 15 +- src/services/ai/live-model-choice.ts | 22 +- src/services/backfill-state.ts | 70 ++++ src/services/capture-diagnostics.ts | 72 +---- src/services/global-config-writer.ts | 48 ++- src/services/global-version.ts | 114 +++++++ src/services/memory-key-source.ts | 122 +++++++ src/services/package-version.ts | 22 ++ src/services/private-path.ts | 67 ++++ src/services/settings-snapshot.ts | 72 ++++- src/services/web-server.ts | 98 +++++- tests/auto-backfill.test.ts | 51 +++ tests/backfill-model.test.ts | 24 ++ tests/backfill-state.test.ts | 31 ++ tests/capture-diagnostics.test.ts | 2 +- tests/external-api-test.test.ts | 35 ++ tests/global-config-writer.test.ts | 33 ++ tests/global-version.test.ts | 93 ++++++ tests/import-path-maps.test.ts | 53 +++ tests/import-progress.test.ts | 54 ++++ tests/import-runs.test.ts | 205 ++++++++++++ tests/live-model-choice.test.ts | 51 +++ tests/map-suggestions.test.ts | 78 +++++ tests/memory-timeline-orphan.test.ts | 59 ++++ tests/omms-config.test.ts | 53 ++- tests/opencode-backfill-model.test.ts | 27 ++ tests/opencode-capture-diagnostics.test.ts | 13 + tests/pi-backfill-model.test.ts | 11 + tests/web-external-settings.test.ts | 131 ++++++++ tests/web-settings-api.test.ts | 143 ++++++++- web/src/App.tsx | 72 ++++- web/src/app.css | 11 + web/src/hooks/useMemoriesExplorer.ts | 27 +- web/src/lib/auto-import-settings.ts | 41 ++- .../lib/components/explorer/AppSidebar.tsx | 129 +++++++- .../lib/components/explorer/KeywordBadge.tsx | 44 +++ .../lib/components/explorer/MemoryCard.tsx | 104 ++++-- .../lib/components/explorer/MemoryList.tsx | 12 +- .../lib/components/explorer/ProfileView.tsx | 12 +- .../components/settings/AutoImportSection.tsx | 125 +++++++- .../settings/DiagnosticsSection.tsx | 106 +++--- .../settings/DirectoryMapsSection.tsx | 170 ++++++++++ .../settings/ExternalApiSection.tsx | 301 ++++++++++++++++++ .../lib/components/settings/ImportSection.tsx | 13 +- .../lib/components/settings/ModelsSection.tsx | 72 +++-- .../lib/components/settings/SettingsView.tsx | 6 +- .../lib/components/settings/WebAppSection.tsx | 32 ++ web/src/lib/components/ui/select.tsx | 232 ++++++++++++++ web/src/lib/components/ui/tooltip.tsx | 30 ++ web/src/lib/external-api-settings.ts | 90 ++++++ web/src/lib/i18n/settings.ts | 115 +++++++ web/src/lib/i18n/translations.ts | 32 ++ 87 files changed, 5007 insertions(+), 290 deletions(-) create mode 100644 openspec/changes/external-api-backfill-maps-progress/.openspec.yaml create mode 100644 openspec/changes/external-api-backfill-maps-progress/design.md create mode 100644 openspec/changes/external-api-backfill-maps-progress/proposal.md create mode 100644 openspec/changes/external-api-backfill-maps-progress/specs/auto-backfill/spec.md create mode 100644 openspec/changes/external-api-backfill-maps-progress/specs/host-neutral-memory-core/spec.md create mode 100644 openspec/changes/external-api-backfill-maps-progress/specs/import-directory-maps/spec.md create mode 100644 openspec/changes/external-api-backfill-maps-progress/specs/import-progress/spec.md create mode 100644 openspec/changes/external-api-backfill-maps-progress/specs/web-autostart/spec.md create mode 100644 openspec/changes/external-api-backfill-maps-progress/specs/web-settings/spec.md create mode 100644 openspec/changes/external-api-backfill-maps-progress/tasks.md create mode 100644 src/importer/backfill-controls.ts create mode 100644 src/importer/external-api-test.ts create mode 100644 src/importer/external-backfill-models.ts create mode 100644 src/importer/import-path-maps.ts create mode 100644 src/importer/import-progress.ts create mode 100644 src/importer/import-runs.ts create mode 100644 src/importer/map-suggestions.ts create mode 100644 src/services/global-version.ts create mode 100644 src/services/memory-key-source.ts create mode 100644 src/services/package-version.ts create mode 100644 src/services/private-path.ts create mode 100644 tests/external-api-test.test.ts create mode 100644 tests/global-version.test.ts create mode 100644 tests/import-path-maps.test.ts create mode 100644 tests/import-progress.test.ts create mode 100644 tests/import-runs.test.ts create mode 100644 tests/map-suggestions.test.ts create mode 100644 tests/web-external-settings.test.ts create mode 100644 web/src/lib/components/explorer/KeywordBadge.tsx create mode 100644 web/src/lib/components/settings/DirectoryMapsSection.tsx create mode 100644 web/src/lib/components/settings/ExternalApiSection.tsx create mode 100644 web/src/lib/components/ui/select.tsx create mode 100644 web/src/lib/components/ui/tooltip.tsx create mode 100644 web/src/lib/external-api-settings.ts diff --git a/openspec/changes/external-api-backfill-maps-progress/.openspec.yaml b/openspec/changes/external-api-backfill-maps-progress/.openspec.yaml new file mode 100644 index 00000000..ee7c5448 --- /dev/null +++ b/openspec/changes/external-api-backfill-maps-progress/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-28 diff --git a/openspec/changes/external-api-backfill-maps-progress/design.md b/openspec/changes/external-api-backfill-maps-progress/design.md new file mode 100644 index 00000000..7e2adb3a --- /dev/null +++ b/openspec/changes/external-api-backfill-maps-progress/design.md @@ -0,0 +1,97 @@ +# Design + +## Context + +See proposal.md for the motivation and the specs for the required behaviour. The current code shapes the approach in these ways: + +- `src/services/ai/live-model-choice.ts` holds the live-model rule as pure functions of the config. `inherit` is the only special host model value. `parseBackfillModel` in `src/importer/backfill-model.ts` accepts `inherit` or `provider/model`. +- `memoryApiKey` is resolved once, when the config loads, by `resolveSecretValue` (`src/services/secret-resolver.ts`), which supports literal, `env://NAME`, and `file://path`. A login item started by launchd, systemd, or the Windows Startup folder does not load a shell profile, so `env://` names set only in `~/.zshrc` do not resolve there. +- `src/services/global-config-writer.ts` edits `omms.jsonc` with `jsonc-parser`, keeps comments, checks a revision, and only writes keys in an allow-list. +- `import-ledger.db` in the store already holds `import_ledger`, `backfill_state` (one row per host, numeric counts only), and `backfill_locks` (one row per host with the holder's PID and a token, replaced when the PID is gone). Manual imports use only an in-process guard (`manual-import-guard.ts`), so a CLI run and a host backfill can overlap today. +- `runHistoryImport` accepts `onProgress(processed, total, promptPreview)`. The total is known before the first model call. +- `resolveImportProject` takes an explicit map list and matches `from` exactly. +- The web server keeps one in-memory import job slot (`SettingsImportJobs`). Import readiness (`import-readiness.ts`) already reports whether the external API is usable in the server's process. + +## Goals / Non-Goals + +**Goals:** + +- One model rule change, in one module, shared by both hosts, the importer, and the web server. +- One lock and one progress record per host that every surface (host backfill, web import, Run now, CLI, slash command) uses, so they cannot overlap and all show up on the page. +- Key values stay out of config, logs, traces, and responses. + +**Non-Goals:** + +- A menu bar or tray item. +- Showing or editing secrets other than `memoryApiKey`. +- Per-project directory maps or per-project external APIs. +- Changing the ledger's unit identity or the import report format. + +## Decisions + +### D1. `external` is a host model value, not a new key + +`opencodeModel`, `piModel`, `opencodeBackfillModel`, and `piBackfillModel` accept `external`, like the existing `inherit`. `getAutoCaptureProviderStatus`, `resolveOpencodeHostModel`, and `resolvePiLiveModel` return the manual mode for `external` when the external API is ready and an unready status naming the missing settings when it is not. The fallback-on-failure path is skipped for `external`, because the external API is already the primary call. `parseBackfillModel` returns `"external"` as a third variant, and the backfill model resolvers on both adapters map it to the shared external provider. + +Alternative: a new `opencodeModelSource`/`piModelSource` key. Rejected because it adds a second key that must agree with the first, and existing configs and the Settings cards already treat the model value as the switch (`inherit`). + +### D2. Key sources and the private key file + +The External API card sends one of three shapes: `{ source: "env", name }`, `{ source: "file", path }`, or `{ source: "paste", value, name }`. The server validates the name (`[A-Za-z_][A-Za-z0-9_]*`) or path, and for `paste` writes `~/.config/omms/secrets/.key` with an exclusive create or, after confirmation, an atomic replace. Permissions: create the folder `700` and the file `600` on POSIX; on Windows, move the user-only ACL code that protects capture traces out of `src/services/capture-diagnostics.ts` into a shared module and use it for both. The config gets `file://` with the absolute path. An `env://` or `file://` key that does not resolve in a process is treated as not set there (`resolveMemoryApiKey` in `src/config.ts`) instead of stopping the config from loading, so a login web app without the variable still starts and reports the key as missing. The request body is never logged; error messages go through the existing API key redaction. The global config writer gains `memoryProvider`, `memoryApiUrl`, `memoryModel`, `memoryApiKey`, and `importPathMaps` in its allow-list, and rejects a `memoryApiKey` that is not an `env://` or `file://` reference. + +Alternative: store the pasted key in the OS keychain. Rejected for now: it needs native code on three platforms, and `file://` already works in every OMMS process, including the login item. + +The **Test** button builds the provider from the saved settings in the web server's process and sends one short prompt with a small output limit, reusing the Health section's model test. + +### D3. Saved maps merge before resolution + +`importPathMaps` is an array of `{ from, to }`, global only. Paths are expanded (`~`) and normalised by the config loader. `run-import.ts` builds the run's map list as the saved maps with run maps (`--map` or the web request's maps) applied on top by `from`, then passes it to the existing `resolveImportProject`. Resolution rules do not change. + +Unresolved directories come from the import report and from the page's session listing. They are stored per host in their own `unresolved_directories` table in `import-ledger.db` (paths and session counts only, capped at 200 entries), so the page can show them without running a listing. They are not stored in the `backfill_state` row, because creating that row fixes a host's backfill cutoff, and a listing must not do that. Suggestions are computed on request by the web server: + +1. For each ancestor of the missing directory, from the directory itself upwards, take its name and find the longest existing sibling of that ancestor, or the longest existing directory under the user's other recorded project roots, whose name is a prefix of that name ending at a `-` or `/` boundary and which is a Git repository or has a project marker. For `~/orca/workspaces/app/feat-x` this also tries the segment `app` against known project roots. +2. For OpenCode sessions, the project `worktree` recorded in OpenCode's `project` table, read through the existing read-only reader. + +Only existing directories are suggested. Nothing is saved until the user saves. + +### D4. One run record and one lock for every surface + +A new `import_runs` table in `import-ledger.db` has one row per host: `host`, `surface` (`auto`, `web`, `cli`, `slash`), `state`, `pid`, `started_at`, `total`, `done`, `imported`, `skipped`, `failed`, `error`, `updated_at`, a JSON array of recent `(time, done)` samples (last 20), and `paused`. The existing `backfill_state` row stays for the cutoff and backfill-specific fields; the page reads both. + +Every model-calling run acquires the existing `backfill_locks` row for its host before it starts (today only automatic backfill does), updates `import_runs` from `onProgress` at most once a second, and releases the lock in `finally`. `promptPreview` is never stored. A run whose PID is gone is reported as stopped. This replaces the in-process `manual-import-guard` as the cross-process guard; the guard stays for the in-process fast path. + +Progress counts only units that need a model call. The importer reports each unit as it starts, with the number of ledger hits so far; `workProgress` in `import-progress.ts` turns that into finished work units. A backfill passes its dry-run count as the fixed total. Samples are taken at most every 15 seconds, so the 20-sample window spans about 5 minutes. + +Time left is `(total - done) / rate`, where `rate` is units per minute across the sample window, and is shown only after 5 samples spanning at least 60 seconds. + +Alternative: parse the CLI's printed report. Rejected: a second process cannot read another process's stdout, and the store is already the shared place both processes use. + +### D5. Pause, resume, and Run now + +`paused` is a flag on the host's `import_runs` row. Pause sets it and aborts the running job's `AbortController` when the run is in the web server's process. When the run is in another process (a host or the CLI), that process checks the flag between exchanges, the same way it checks `autoBackfill` today, and stops. Host start-up skips the automatic run while the flag is set. Resume clears the flag, then behaves like Run now. + +Run now starts the host's backfill inside the web server process using `scheduleAutoBackfill` with no start-up delay and the `web` surface. In the login web app there is no host runtime, so it is available only when the host's backfill model resolves to `external`; otherwise the endpoint answers with the reason. Inside an OpenCode-served web app, OpenCode's connected models remain available for the OpenCode backfill. Pi's backfill needs the Pi SDK for reading sessions, which the web server already loads for Pi imports (`piReader` readiness). + +### D6. `--version` and the version check + +`src/cli/index.ts` handles `--version` and `-v` by printing the version from the package's `package.json`. The web server finds `om-memory-system` on its own `PATH`, runs it with `--version` and a 3-second timeout without a shell (the pattern used by the Windows Git wrapper fix), caches the result for 10 minutes, and compares it with its own version. + +## Risks / Trade-offs + +- [A login web app still cannot see `env://` keys] → The card says so and suggests a key file; readiness reports the key as missing in that process. +- [A pasted key lands on disk] → User-only permissions, a dedicated folder, no logging, and the same trust model as the existing `file://` support. Saving a pasted key is refused on a network-bound server without Basic Auth. +- [`external` hides fallback] → A failure of the external API is not retried elsewhere. The capture attempt log records the failure, as it does for other failures. +- [Suggestions can be wrong] → They are only suggestions; the user confirms each map, and imports never apply an unconfirmed suggestion. +- [Progress writes add store traffic] → Throttled to one write a second per run, and only numbers are written. +- [A crashed run leaves a running row] → PID liveness turns it into stopped; the lock is already replaced the same way. +- [A CLI run started before the upgrade has no progress row] → Only runs from the new version report progress; older runs still write the ledger. + +## Migration Plan + +- New config keys are optional; existing configs behave as before. +- `import_runs` is created on first use with `CREATE TABLE IF NOT EXISTS`; older versions ignore it. +- Rolling back to an older version: `external` values would fail validation there, so the rollback note in the changelog tells users to switch those settings back first. Saved key files and `importPathMaps` are ignored by older versions. + +## Open Questions + +- The exact wording and layout of the Directory maps list and the suggestion controls can be settled during implementation without changing the specs. diff --git a/openspec/changes/external-api-backfill-maps-progress/proposal.md b/openspec/changes/external-api-backfill-maps-progress/proposal.md new file mode 100644 index 00000000..20c35c9b --- /dev/null +++ b/openspec/changes/external-api-backfill-maps-progress/proposal.md @@ -0,0 +1,45 @@ +# Proposal + +## Why + +Setting up an external model (for example a Z.ai GLM Coding Plan endpoint) today means editing `omms.jsonc` by hand, and that model can only act as a fallback: a host cannot choose it as its main capture or backfill model. History from deleted worktrees can only be imported with repeated `--map` flags on the CLI, because automatic backfill and the Settings page have no saved directory maps. Long imports show counts but no percentage or time left, and a CLI run is invisible on the page. The CLI guide also lacks a full command list and a global install step. + +## What Changes + +- Add an **External API** card to the Settings page. It edits `memoryProvider`, `memoryApiUrl`, `memoryModel`, and `memoryApiKey`. The user picks the key source: + - **Environment variable**: types a variable name, saved as `env://NAME`. + - **Key file**: types the path of an existing key file, saved as `file://path`. + - **Save key to a private file**: pastes the key once. OMMS writes it to `~/.config/omms/secrets/.key`, readable only by the user, and saves `file://` with that path. The key never goes into `omms.jsonc`, the log, or any response, and the page never shows it again. + + A **Test** button makes one small call and reports success or a redacted error. The card shows whether the key resolves in the web app's own process, and warns that a login web app does not see variables set only in a shell profile, so a key file suits the login web app. + +- Let each host choose the external API as its model. The value `external` for `opencodeModel`/`piModel` sends live capture and profile learning to the external API; the same value for `opencodeBackfillModel`/`piBackfillModel` sends that host's backfill there. The model rule changes only in `live-model-choice.ts`, identically for both hosts. +- Add a saved directory map list, `importPathMaps`, in the global config. Automatic backfill, web imports, and CLI and slash-command imports all use it. A `--map` flag adds to it for that run and wins for the same source directory. +- On the Settings page, list the directories that cannot be resolved, with session counts and a suggested target where one can be found (for example, a deleted worktree's main repository). The user confirms, edits, or removes each map and saves. +- Record progress for every import run (automatic, web, and CLI) in the store: totals, done, percentage, and minutes left from the recent rate. The page shows a progress bar for each host and for a running CLI or web import. +- Add **Run now**, **Pause**, and **Resume** for each host's backfill on the Settings page. They work in the login web app without Pi or OpenCode open, using the external API. A paused backfill stays paused across host starts until the user resumes it. +- Add `om-memory-system --version`. The Web app section shows the global command's version and warns when it differs from the running OMMS version. +- Docs: a complete command reference in `docs/cli.md` (every terminal command and subcommand, and both hosts' slash commands), global install as optional but highly recommended (`npm i -g` or `bun add -g`, upgrading, checking the version) in `docs/web-ui.md` and the setup docs, and a note that terminal proxies such as Orca's `*.orca.localhost` addresses can differ from the printed `127.0.0.1` URL. + +No breaking changes: existing configs keep their behaviour; `external` and `importPathMaps` are opt-in. + +## Capabilities + +### New Capabilities + +- `import-directory-maps`: saved directory maps shared by every import surface, and suggestions for unresolved directories. +- `import-progress`: a progress record for every import run, with percentage and time left, and Run now, Pause, and Resume for backfills. + +### Modified Capabilities + +- `host-neutral-memory-core`: the live-capture model rule gains the `external` host model value. +- `auto-backfill`: backfill models accept `external`; backfill uses saved maps; a paused state survives host starts. +- `web-settings`: the External API card; the page may write a key reference or save a pasted key to a private key file, but never reads or shows a key value; the Automatic import section gains progress, maps, and run controls. +- `web-autostart`: the login web app can run a user-started backfill; `--version` and the version-mismatch warning. + +## Impact + +- Code: `src/services/ai/live-model-choice.ts`, `src/config.ts` (new keys and validation), `src/services/global-config-writer.ts` (editable keys), `src/importer/` (maps, progress record, backfill controls, suggestions), `src/services/backfill-state.ts`, `src/services/api-handlers.ts` and `web-server.ts` (new endpoints), `src/cli/index.ts` (`--version`), both host adapters (backfill model `external`, pause state), and the web UI Settings page. +- Store: new progress and control rows in `import-ledger.db`; no change to memory shards. +- Docs: `docs/cli.md`, `docs/web-ui.md`, `docs/configuration.md`, `README.md` setup, and the history import guides. +- Tests: model rule parity, config validation, map merging, progress maths, pause across restart, endpoint access control, and secret handling. diff --git a/openspec/changes/external-api-backfill-maps-progress/specs/auto-backfill/spec.md b/openspec/changes/external-api-backfill-maps-progress/specs/auto-backfill/spec.md new file mode 100644 index 00000000..1361a3ec --- /dev/null +++ b/openspec/changes/external-api-backfill-maps-progress/specs/auto-backfill/spec.md @@ -0,0 +1,65 @@ +## MODIFIED Requirements + +### Requirement: Past chats are imported automatically when a host starts + +When `autoBackfill` is `true`, which is the default, each host SHALL start a background import of its own chat history after it starts: Pi imports Pi sessions from its default sessions folder, and OpenCode imports sessions from its default database. The run SHALL cover every project whose directory can be resolved with the importer's project rules, including the saved directory maps in `importPathMaps`. It SHALL use the shared importer, its ledger, and the live capture pipeline, and it SHALL also record the imported prompts and build the user profile from them. The run SHALL start after a start-up delay and SHALL NOT delay session start, prompt handling, retrieval, or live capture. When `autoBackfill` is `false`, or when the user has paused that host's backfill, no automatic run SHALL start. `autoBackfill` SHALL be read from the global config only. + +#### Scenario: A new machine with existing history + +- **WHEN** Pi starts with `autoBackfill` on, and the Pi sessions folder holds sessions that were never imported +- **THEN** OMMS SHALL import their exchanges in the background +- **AND** the user profile SHALL be built from the imported prompts + +#### Scenario: Automatic backfill is off + +- **WHEN** a host starts with `autoBackfill` set to `false` +- **THEN** no automatic import SHALL start and no model call SHALL be made for past chats + +#### Scenario: Start-up is not delayed + +- **WHEN** a host starts with pending history +- **THEN** the session SHALL become usable without waiting for the backfill +- **AND** retrieval and live capture SHALL keep working while the backfill runs + +#### Scenario: A project config sets the switch + +- **WHEN** a project config sets `autoBackfill` +- **THEN** the value SHALL be ignored and the global value SHALL apply + +#### Scenario: Sessions from a mapped directory + +- **WHEN** a saved directory map covers sessions recorded in a deleted worktree +- **THEN** the backfill SHALL import them into the map's target project + +#### Scenario: The backfill is paused + +- **WHEN** the user has paused Pi's backfill and Pi starts +- **THEN** no Pi backfill SHALL start + +### Requirement: Each host's backfill model is configurable + +`opencodeBackfillModel` and `piBackfillModel` SHALL choose the model for each host's automatic backfill. The value `inherit`, which is the default, SHALL use the model that the host's live capture would use under the live-model rule. The value `external` SHALL use the external API (`memoryProvider`, `memoryModel`, `memoryApiUrl`, `memoryApiKey`). A `provider/model` value SHALL use that model from the host's signed-in models. When the chosen model cannot be resolved, including an `external` value while the external API is not fully configured, the run SHALL NOT start and the status SHALL say why. The setting SHALL NOT change the model of live capture or of manual imports. + +#### Scenario: A cheaper model for Pi's backfill + +- **WHEN** `piBackfillModel` is `zai/glm-5-turbo` and `piModel` is another model +- **THEN** the Pi backfill SHALL call `zai/glm-5-turbo` +- **AND** Pi's live capture SHALL keep using its own model + +#### Scenario: The chosen model is not signed in + +- **WHEN** `opencodeBackfillModel` names a provider that OpenCode has not connected +- **THEN** no OpenCode backfill SHALL start +- **AND** the status SHALL say that the model is not available + +#### Scenario: Backfill through the external API + +- **WHEN** `opencodeBackfillModel` is `external` and the external API is fully configured +- **THEN** the OpenCode backfill SHALL call the external API +- **AND** OpenCode's live capture SHALL keep using its own model rule + +#### Scenario: The external API is not configured + +- **WHEN** `piBackfillModel` is `external` and `memoryModel` is not set +- **THEN** no Pi backfill SHALL start +- **AND** the status SHALL say that `memoryModel` is missing diff --git a/openspec/changes/external-api-backfill-maps-progress/specs/host-neutral-memory-core/spec.md b/openspec/changes/external-api-backfill-maps-progress/specs/host-neutral-memory-core/spec.md new file mode 100644 index 00000000..917bc4a7 --- /dev/null +++ b/openspec/changes/external-api-backfill-maps-progress/specs/host-neutral-memory-core/spec.md @@ -0,0 +1,43 @@ +## MODIFIED Requirements + +### Requirement: Live capture uses one model rule on both hosts + +OpenCode and Pi SHALL choose the model for automatic capture and profile learning in the same order: + +1. the host model: `opencodeProvider`/`opencodeModel` on OpenCode, `piProvider`/`piModel` on Pi, where the model value `inherit` means the session's model and the model value `external` means the external API +2. the external API (`memoryModel`, `memoryApiUrl`, `memoryApiKey`), when no host model is set +3. the session's own model, when neither is set + +When the host model is `external`, the provider value SHALL be ignored and every call SHALL go to the external API. When the host model is `external` and the external API is not fully configured, automatic capture on that host SHALL be disabled and the missing settings SHALL be reported. When another host model fails and the external API is fully configured, the call SHALL use the external API and the user SHALL be notified. When the external API is only partly configured and no host model is set, automatic capture SHALL be disabled and the missing settings SHALL be reported. + +#### Scenario: Nothing is configured + +- **WHEN** no host model and no external API settings exist +- **THEN** automatic capture on either host SHALL use the session's model + +#### Scenario: The pinned host model fails + +- **WHEN** the configured host model call fails and the external API is fully configured +- **THEN** the capture SHALL be retried through the external API +- **AND** a "Using fallback provider" notification SHALL be shown + +#### Scenario: The external API is half configured + +- **WHEN** `memoryModel` is set but `memoryApiKey` is not, and no host model is set +- **THEN** automatic capture SHALL be disabled and the missing settings SHALL be reported + +#### Scenario: A host chooses the external API + +- **WHEN** `piModel` is `external` and the external API is fully configured +- **THEN** Pi's automatic capture and profile learning SHALL call the external API +- **AND** OpenCode SHALL keep using its own host model + +#### Scenario: The same choice on both hosts + +- **WHEN** `opencodeModel` is `external` on OpenCode and `piModel` is `external` on Pi, with the same external API settings +- **THEN** both hosts SHALL call the same external model + +#### Scenario: A host chooses an unconfigured external API + +- **WHEN** `opencodeModel` is `external` and `memoryApiUrl` is not set +- **THEN** OpenCode's automatic capture SHALL be disabled and `memoryApiUrl` SHALL be reported as missing diff --git a/openspec/changes/external-api-backfill-maps-progress/specs/import-directory-maps/spec.md b/openspec/changes/external-api-backfill-maps-progress/specs/import-directory-maps/spec.md new file mode 100644 index 00000000..de74a621 --- /dev/null +++ b/openspec/changes/external-api-backfill-maps-progress/specs/import-directory-maps/spec.md @@ -0,0 +1,54 @@ +## Purpose + +Let the user save directory maps once, so that history recorded in deleted or moved directories is imported into the right project by automatic backfill, web imports, and CLI and slash-command imports alike. + +## ADDED Requirements + +### Requirement: Saved directory maps apply to every import + +The global config SHALL accept `importPathMaps`, a list of maps, each with a source directory `from` and a target directory `to`. Automatic backfill, web imports, and CLI and slash-command imports on both hosts SHALL resolve project directories with these maps, using the existing resolution order: an exact directory map first, then the recorded directory, then, for OpenCode only, the project worktree. A `--map` flag or a web import's own map SHALL add to the saved maps for that run, and SHALL win over a saved map with the same source directory. A map whose target directory does not exist SHALL leave its sessions unresolved. `importPathMaps` SHALL be read from the global config only. Invalid entries SHALL be rejected by the same validation used at startup. + +#### Scenario: Automatic backfill uses a saved map + +- **WHEN** `importPathMaps` maps `/Users/me/code/app-feat-x` to `/Users/me/code/app` and Pi's backfill finds sessions recorded in `/Users/me/code/app-feat-x`, which no longer exists +- **THEN** those sessions SHALL be imported into the project of `/Users/me/code/app` + +#### Scenario: A CLI map overrides a saved map + +- **WHEN** a saved map sends `/old` to `/a` and the user runs an import with `--map /old=/b` +- **THEN** that run SHALL use `/b` for sessions recorded in `/old` +- **AND** the saved map SHALL be unchanged + +#### Scenario: The target directory is missing + +- **WHEN** a saved map points at a directory that does not exist +- **THEN** its sessions SHALL be reported as unresolved and SHALL NOT be assigned to any project + +#### Scenario: A project config sets maps + +- **WHEN** a project config sets `importPathMaps` +- **THEN** the value SHALL be ignored and the global value SHALL apply + +### Requirement: The Settings page manages directory maps + +The Settings page SHALL have a **Directory maps** section. It SHALL list the saved maps and the directories that the latest listing, preview, or backfill could not resolve, for each host, with the number of sessions in each. For each unresolved directory it SHALL offer a suggested target when one can be found, and SHALL let the user accept, edit, or reject it. Saving SHALL write `importPathMaps` to the global config with the same safe-save rules as other settings. The page SHALL say that a change applies to the next import or backfill run. It SHALL NOT show conversation content. + +A suggestion SHALL be an existing directory. The page SHALL suggest, in this order: the main repository of a deleted Git worktree, found as an existing directory whose name is the longest leading part of the missing directory's name or of one of its parent directories' names (for example `app` for `app-feat-x` or for `workspaces/app/feat-x`); then, for OpenCode sessions, the project directory that OpenCode recorded for the session's project, read without writing to OpenCode's database. When no candidate exists, the page SHALL show no suggestion and SHALL let the user type a target or leave the directory unmapped. + +#### Scenario: Accepting a suggested target + +- **WHEN** Pi sessions are recorded in `/Users/me/code/app-feat-x`, which no longer exists, and `/Users/me/code/app` is a Git repository +- **THEN** the page SHALL suggest `/Users/me/code/app` +- **AND** accepting it and saving SHALL add the map to `importPathMaps` + +#### Scenario: No candidate exists + +- **WHEN** sessions are recorded in a temporary directory that no longer exists and no candidate is found +- **THEN** the page SHALL show the directory with no suggestion +- **AND** its sessions SHALL stay unresolved unless the user types a target + +#### Scenario: Removing a map + +- **WHEN** the user removes a saved map and saves +- **THEN** `importPathMaps` SHALL no longer contain it +- **AND** memories already imported through it SHALL remain diff --git a/openspec/changes/external-api-backfill-maps-progress/specs/import-progress/spec.md b/openspec/changes/external-api-backfill-maps-progress/specs/import-progress/spec.md new file mode 100644 index 00000000..e521adde --- /dev/null +++ b/openspec/changes/external-api-backfill-maps-progress/specs/import-progress/spec.md @@ -0,0 +1,65 @@ +## Purpose + +Show how far every history import has progressed and how long it has left, whether it runs as an automatic backfill, from the web page, or from the terminal, and let the user start, pause, and resume each host's backfill from the Settings page. + +## ADDED Requirements + +### Requirement: Every import run records its progress + +Each import run that makes model calls, whether an automatic backfill, a web import, or a CLI or slash-command import, SHALL record a progress entry in the memory store. The entry SHALL hold the host, the surface that started it, the state (running, paused, stopped, done, or failed), the start time, the total number of memory units and profile batches to process, the number processed so far with imported, skipped, and failed counts, the last update time, and the last error with secrets removed. The total SHALL count only units that need a model call, not units already in the ledger. For an automatic backfill, and for Run now and Resume, the total SHALL be the backfill's dry-run count, known before the first model call. For a CLI, slash-command, or web import, which has no dry run, the total SHALL start from the units found and SHALL be refined as the run passes units already in the ledger. The entry SHALL NOT contain prompts, replies, or other conversation content. Dry runs SHALL NOT record progress entries. A run that stops without updating its entry SHALL be shown as stopped once its owning process is gone. + +#### Scenario: A CLI import is visible on the page + +- **WHEN** the user runs `om-memory-system import-pi-history` in a terminal and opens the Settings page +- **THEN** the page SHALL show that Pi import as running, started from the CLI, with its counts + +#### Scenario: Most units are already imported + +- **WHEN** a Pi backfill finds 736 units, of which 729 are already in the ledger +- **THEN** the page SHALL show a total of 7 and count only those 7 as pending + +#### Scenario: The terminal is closed during a CLI import + +- **WHEN** the CLI process ends without finishing its run +- **THEN** the page SHALL show that run as stopped +- **AND** rerunning the same command SHALL continue from the ledger + +### Requirement: Progress shows percentage and time left + +For a running import, the Settings page SHALL show a progress bar, the percentage done, the number done out of the total, and the estimated minutes left. The estimate SHALL be computed from the processing rate over the recent part of the run, not from the whole run, and SHALL be shown as unknown until enough units have finished to measure a rate. The values SHALL refresh without a page reload while the run is active. + +#### Scenario: Watching a long import + +- **WHEN** 400 of 1,000 units are done and the recent rate is 5 units a minute +- **THEN** the page SHALL show 40%, 400 of 1,000, and about 120 minutes left + +#### Scenario: A run has just started + +- **WHEN** fewer units than needed to measure a rate have finished +- **THEN** the time left SHALL be shown as unknown + +### Requirement: The user can run, pause, and resume a host's backfill + +The Settings page SHALL offer **Run now**, **Pause**, and **Resume** for each host's backfill. Run now SHALL start that host's backfill at once, with the same cutoff, maps, model rule, ledger, and one-run-per-host lock as an automatic backfill. It SHALL be available in the login web app and in `om-memory-system web` without Pi or OpenCode open when the host's backfill model resolves to the external API, and SHALL otherwise say which model setting it needs. Pause SHALL stop the run after its current exchange and record the paused state. A paused backfill SHALL NOT start automatically at a host start until the user resumes it. Resume SHALL clear the paused state and start the run, continuing from the ledger. Run now SHALL be refused, with the reason, while another import for the same host runs. These controls SHALL follow the same origin and authentication rules as other Settings changes. + +#### Scenario: Running a backfill with no host open + +- **WHEN** the login web app runs, Pi is closed, `piBackfillModel` is `external`, and the user clicks Run now for Pi +- **THEN** the Pi backfill SHALL start in the web app's process and its progress SHALL be shown + +#### Scenario: Pausing across a restart + +- **WHEN** the user pauses the OpenCode backfill and later starts OpenCode +- **THEN** no OpenCode backfill SHALL start +- **AND** the page SHALL show it as paused until the user clicks Resume + +#### Scenario: A second run for the same host + +- **WHEN** a Pi CLI import is running and the user clicks Run now for Pi +- **THEN** the page SHALL refuse with the reason that a Pi import is already running + +#### Scenario: No usable model outside a host + +- **WHEN** the login web app runs and the Pi backfill model is a Pi signed-in model +- **THEN** Run now for Pi SHALL be unavailable +- **AND** the page SHALL say to open Pi or choose the external API for Pi's backfill diff --git a/openspec/changes/external-api-backfill-maps-progress/specs/web-autostart/spec.md b/openspec/changes/external-api-backfill-maps-progress/specs/web-autostart/spec.md new file mode 100644 index 00000000..0f3f462e --- /dev/null +++ b/openspec/changes/external-api-backfill-maps-progress/specs/web-autostart/spec.md @@ -0,0 +1,61 @@ +## ADDED Requirements + +### Requirement: The web app reports the global command's version + +The Settings page's **Web app** section SHALL show the running OMMS version and the version of the `om-memory-system` command found on the web app's `PATH`, or that the command is not installed globally. When the two versions differ, the section SHALL warn about it and show the command that upgrades the global install. It SHALL say that a global install is optional but recommended, because it lets the login item and the terminal commands run without `npx`. + +#### Scenario: The global command is older + +- **WHEN** OMMS 3.4.0 runs and the global `om-memory-system --version` prints `3.3.1` +- **THEN** the section SHALL warn that the versions differ and show the upgrade command + +#### Scenario: No global install + +- **WHEN** `om-memory-system` is not on the web app's `PATH` +- **THEN** the section SHALL say that it is not installed globally and show the install command + +## MODIFIED Requirements + +### Requirement: The web app runs without a host session + +The web app started by the login item or by `om-memory-system web` SHALL serve the same pages and API as the web app inside OpenCode, with the same port, bind address, origin rules, API token, and Basic Auth settings. When another OMMS process already serves the web app on the configured port, it SHALL follow the existing port ownership and takeover rules instead of starting a second server. Features that need a host, such as OpenCode's connected-model list, SHALL report that they are unavailable, with the existing reasons. It SHALL NOT start a backfill on its own. It SHALL run a host's backfill when the user starts or resumes it on the Settings page and that host's backfill model resolves to the external API. + +#### Scenario: Opening the web app with no session open + +- **WHEN** the user logs in and opens the web app URL without starting Pi or OpenCode +- **THEN** the memories, profile, and Settings pages SHALL load + +#### Scenario: OpenCode starts while the login web app runs + +- **WHEN** the login web app owns the port and OpenCode starts +- **THEN** OpenCode SHALL use the running web app instead of starting a second one + +#### Scenario: The login web app does not backfill by itself + +- **WHEN** the login web app starts with pending history and `autoBackfill` on +- **THEN** no backfill SHALL start until the user clicks Run now or Resume + +### Requirement: The terminal can start and manage the web app + +The package SHALL provide `om-memory-system web`, which starts the web app in the foreground until it is stopped, and `om-memory-system web install`, `web uninstall`, and `web status`. `install` SHALL set `webServerAutoStart` to `true` in the global config and install the login item. `uninstall` SHALL set it to `false` and remove the item. `status` SHALL report the setting, whether the item is installed, what it starts, and whether a web app answers on the configured port. `web` SHALL refuse to start, with the reason, when `webServerEnabled` is `false`. `om-memory-system --version` SHALL print the package version and exit with code `0`. + +#### Scenario: Starting the web app by hand + +- **WHEN** the user runs `om-memory-system web` +- **THEN** the web app SHALL start and print its URL + +#### Scenario: Removing the login item from the terminal + +- **WHEN** the user runs `om-memory-system web uninstall` +- **THEN** the login item SHALL be removed and `webServerAutoStart` SHALL be `false` in the global config +- **AND** a later host start SHALL NOT install the item again + +#### Scenario: The web server is disabled + +- **WHEN** `webServerEnabled` is `false` and the user runs `om-memory-system web` +- **THEN** the command SHALL exit with a message that the web server is disabled + +#### Scenario: Printing the version + +- **WHEN** the user runs `om-memory-system --version` +- **THEN** the command SHALL print the installed package version and exit with code `0` diff --git a/openspec/changes/external-api-backfill-maps-progress/specs/web-settings/spec.md b/openspec/changes/external-api-backfill-maps-progress/specs/web-settings/spec.md new file mode 100644 index 00000000..64ad99c5 --- /dev/null +++ b/openspec/changes/external-api-backfill-maps-progress/specs/web-settings/spec.md @@ -0,0 +1,160 @@ +## ADDED Requirements + +### Requirement: The page configures the external API + +The Settings page SHALL have an **External API** card that edits `memoryProvider`, `memoryApiUrl`, `memoryModel`, and `memoryApiKey` in the global config. The provider SHALL be chosen from the providers OMMS supports. The card SHALL offer three key sources: + +- **Environment variable**: the user types a variable name, and the page saves `env://NAME`. +- **Key file**: the user types the path of an existing file, and the page saves `file://` with that path. +- **Save key to a private file**: the user pastes the key once. The server SHALL write it to a file under `~/.config/omms/secrets/`, create the folder if needed, restrict the file to the current user (mode `600` on macOS and Linux, a user-only access list on Windows), and save `file://` with that path. It SHALL replace an existing key file only after the user confirms. + +The key value SHALL NOT be written to `omms.jsonc`, the log, the capture trace, or any response, and the page SHALL NOT show it after saving. The card SHALL show the saved key source type and reference, whether the key resolves in the web app's own process, and, for an environment variable, that a login web app does not see variables set only in a shell profile. A **Test** button SHALL make one small call with the saved settings and report success or an error with the key redacted. + +#### Scenario: Using an environment variable + +- **WHEN** the user chooses Environment variable, types `ZAI_API_KEY`, and saves +- **THEN** the global config SHALL have `memoryApiKey` set to `env://ZAI_API_KEY` + +#### Scenario: Saving a pasted key + +- **WHEN** the user pastes a key, chooses Save key to a private file, and saves +- **THEN** the key SHALL be written to a user-only file under `~/.config/omms/secrets/` +- **AND** `memoryApiKey` SHALL be set to `file://` with that path +- **AND** no response or log line SHALL contain the key + +#### Scenario: The variable is missing in the login web app + +- **WHEN** `memoryApiKey` is `env://ZAI_API_KEY` and the login web app's process has no such variable +- **THEN** the card SHALL say that the key does not resolve in the web app and suggest a key file + +#### Scenario: Testing the endpoint + +- **WHEN** the user clicks Test and the endpoint rejects the key +- **THEN** the card SHALL show the error with the key redacted + +## MODIFIED Requirements + +### Requirement: Each host's capture model can be chosen on the page + +The Settings page SHALL show one model card for OpenCode and one for Pi. Each card SHALL offer **Session model**, **Manual model**, and **External API**. Choosing Session model SHALL save the host's model as `inherit`. Choosing External API SHALL save the host's model as `external`. Choosing Manual model SHALL save the selected provider and model to that host's settings (`opencodeProvider`/`opencodeModel` or `piProvider`/`piModel`). The manual picker SHALL list the host's signed-in models when the server can read them. Otherwise it SHALL accept a typed `provider/model` value and say that the list is not available. External API SHALL be selectable only when the external API is fully configured, and otherwise SHALL say which setting is missing. Each card SHALL show, read-only, the external API fallback and which model the live-model rule would choose now. The page SHALL NOT change the order of the live-model rule. + +#### Scenario: Switching OpenCode to the session model + +- **WHEN** the user chooses Session model on the OpenCode card and saves +- **THEN** the global config SHALL have `opencodeModel` set to `inherit` +- **AND** the next OpenCode capture SHALL use the session's model + +#### Scenario: Pinning a manual Pi model + +- **WHEN** the user picks provider `zai` and model `glm-5.3` on the Pi card and saves +- **THEN** the global config SHALL have `piProvider` `zai` and `piModel` `glm-5.3` +- **AND** the next Pi capture SHALL use that model + +#### Scenario: Choosing the external API for Pi + +- **WHEN** the external API is fully configured and the user chooses External API on the Pi card and saves +- **THEN** the global config SHALL have `piModel` set to `external` +- **AND** the next Pi capture SHALL call the external API + +#### Scenario: The Pi model list is not available + +- **WHEN** the server cannot load the Pi SDK +- **THEN** the Pi card SHALL accept a typed `provider/model` value +- **AND** it SHALL say that the list of signed-in models is not available + +#### Scenario: A project config overrides the host model + +- **WHEN** the current project's config sets the same model keys +- **THEN** the card SHALL say that the project value takes precedence for that project + +### Requirement: Settings are saved safely to the global config + +Saving on the Settings page SHALL write only the changed keys to the global config file that OMMS is reading. When that file is the legacy `~/.config/opencode/opencode-mem.jsonc`, the first save SHALL create `~/.config/omms/omms.jsonc` as a copy of it, comments included, apply the change there, and tell the user that OMMS now reads the new file. The legacy file SHALL NOT be written. Saves SHALL run one at a time, and a save SHALL be rejected without writing when the file changed after the page read it. Saving SHALL keep comments, key order, and all other keys. It SHALL reject values that fail the same validation used at startup, and SHALL leave the file unchanged when it rejects them. Running OpenCode and Pi processes SHALL use the saved values from their next capture without a restart. The page SHALL NOT write project config files. It SHALL NOT read or show secret values; it SHALL show only whether a secret is set, its source type (literal, `env://`, or `file://`), and, for `env://` and `file://`, the variable name or file path. The only secret the page SHALL change is `memoryApiKey`, and only to an `env://` or `file://` reference, including one created by saving a pasted key to a private key file. It SHALL NOT save a literal key to the config. + +#### Scenario: A commented config file is edited + +- **WHEN** the user saves a new Pi model and the config file has comments +- **THEN** the comments and every other key SHALL remain unchanged + +#### Scenario: The install still uses the legacy config file + +- **WHEN** only `~/.config/opencode/opencode-mem.jsonc` exists and the user saves a new Pi model +- **THEN** `~/.config/omms/omms.jsonc` SHALL be created with every key and comment from the legacy file plus the new Pi model +- **AND** the legacy file SHALL be unchanged +- **AND** the page SHALL say that OMMS now reads `~/.config/omms/omms.jsonc` + +#### Scenario: The file changed while the page was open + +- **WHEN** the config file is edited by hand, or by another process, after the page read it and before the page saves +- **THEN** the save SHALL be rejected without writing +- **AND** the page SHALL reload the current settings and ask the user to save again + +#### Scenario: An invalid value is saved + +- **WHEN** the user saves a retention of `0` days +- **THEN** the save SHALL be rejected with the reason, and the file SHALL be unchanged + +#### Scenario: The running host picks up a change + +- **WHEN** Pi is running and the user saves a new Pi model on the page +- **THEN** Pi's next capture SHALL use the new model without a restart + +#### Scenario: A secret is configured + +- **WHEN** `memoryApiKey` is set to `env://OMMS_KEY` +- **THEN** the page SHALL show that the key is set from the environment variable `OMMS_KEY` and SHALL NOT show its value + +#### Scenario: A literal key is submitted as a reference + +- **WHEN** a request tries to save `memoryApiKey` as a value that is not an `env://` or `file://` reference +- **THEN** the save SHALL be rejected and the file SHALL be unchanged + +### Requirement: Changes from the page are access-controlled + +Every Settings endpoint that changes config, saves a key file, deletes trace files, validates or browses an import source, lists import sessions, starts, pauses, resumes, or cancels an import or backfill, or makes a model test call SHALL require a JSON request body and SHALL reject requests whose origin is not allowed by the web server's origin rules. When the web server is bound to a non-loopback host, these endpoints SHALL also require the existing API token or Basic Auth credentials. Turning `captureTrace` on, and saving a pasted key to a key file, SHALL be rejected when the server is bound to a non-loopback host without Basic Auth. + +#### Scenario: A request from another website + +- **WHEN** a page on another origin sends a request to change settings +- **THEN** the server SHALL reject it and the config SHALL be unchanged + +#### Scenario: A cross-site request tries to list sessions + +- **WHEN** Basic Auth is on and a page on another site sends a request to list sessions without a JSON body +- **THEN** the server SHALL reject it before reading any history source or copying any file + +#### Scenario: Turning on tracing over the network + +- **WHEN** the server is bound to `0.0.0.0` without Basic Auth and a request turns tracing on +- **THEN** the server SHALL reject it + +#### Scenario: Saving a key over the network + +- **WHEN** the server is bound to `0.0.0.0` without Basic Auth and a request saves a pasted key +- **THEN** the server SHALL reject it and write no key file + +### Requirement: The page controls automatic import + +The Settings page SHALL have an **Automatic import** section with a switch for `autoBackfill` and, for each host, a model choice for `opencodeBackfillModel` or `piBackfillModel`: **Same as live capture** (saves `inherit`), **External API** (saves `external`), or a manual `provider/model` chosen the same way as the host's capture model. For each host the section SHALL show the backfill state, including paused, the counts of imported, skipped, failed, and pending exchanges, the number of sessions whose project cannot be resolved with a link to the Directory maps section, the model used, the cutoff, the last error, and the progress bar, percentage, and time left defined by the import progress capability. It SHALL offer Run now, Pause, and Resume for each host. The counts SHALL refresh while a run is active. It SHALL say that a model change takes effect at the next run, except that turning the switch off also stops a running backfill after its current exchange. It SHALL say that automatic import makes model calls. + +#### Scenario: Turning automatic import off + +- **WHEN** the user turns off the Automatic import switch and saves while a Pi backfill runs +- **THEN** the global config SHALL have `autoBackfill` set to `false` +- **AND** the Pi backfill SHALL stop after its current exchange + +#### Scenario: Choosing a backfill model for Pi + +- **WHEN** the user picks provider `zai` and model `glm-5-turbo` for Pi's backfill and saves +- **THEN** the global config SHALL have `piBackfillModel` set to `zai/glm-5-turbo` +- **AND** `piProvider` and `piModel` SHALL be unchanged + +#### Scenario: Choosing the external API for OpenCode's backfill + +- **WHEN** the user chooses External API for OpenCode's backfill and saves +- **THEN** the global config SHALL have `opencodeBackfillModel` set to `external` + +#### Scenario: Watching progress + +- **WHEN** a backfill runs while the section is open +- **THEN** the counts, progress bar, percentage, and time left SHALL update without a page reload diff --git a/openspec/changes/external-api-backfill-maps-progress/tasks.md b/openspec/changes/external-api-backfill-maps-progress/tasks.md new file mode 100644 index 00000000..f0e36c5e --- /dev/null +++ b/openspec/changes/external-api-backfill-maps-progress/tasks.md @@ -0,0 +1,58 @@ +# Tasks + +## 1. Config and model rule + +- [x] 1.1 Accept `external` for `opencodeModel`, `piModel`, `opencodeBackfillModel`, and `piBackfillModel`, and add `importPathMaps` (global only, `~` expanded, entries validated) in `src/config.ts`; verify with config validation tests for valid and invalid values and that a project config's `importPathMaps` is ignored +- [x] 1.2 Handle `external` in `live-model-choice.ts` (ready → manual mode, unready → missing settings, no fallback retry) for both hosts; verify with parity tests that OpenCode and Pi give the same result for the same config, including a half-configured external API +- [x] 1.3 Return an `external` variant from `parseBackfillModel` and map it to the external provider in both adapters' backfill model resolvers; verify with backfill model tests that an unconfigured external API stops the run with the missing setting named +- [x] 1.4 Wire the OpenCode and Pi live capture paths to the `external` choice; verify with adapter tests that each host calls the external provider and that the other host is unaffected + +## 2. Directory maps + +- [x] 2.1 Merge saved `importPathMaps` with run maps (run maps win by `from`) in `run-import.ts` for CLI, slash-command, web, and automatic runs; verify with tests for merge order and a missing target staying unresolved +- [x] 2.2 Store a bounded list of unresolved directories with session counts in the backfill state; verify with a test that paths and counts are stored, capped at 200, and no conversation content is stored +- [x] 2.3 Implement target suggestions (longest existing prefix repository across ancestor names, then OpenCode's recorded project worktree read-only); verify with fixture tests for `app-feat-x`, `workspaces/app/feat-x`, a temporary directory with no suggestion, and a check that OpenCode's database is opened read-only + +## 3. Progress, lock, and controls + +- [x] 3.1 Add the `import_runs` table and a throttled progress writer fed by `onProgress` (numbers only, recent samples, PID); verify with tests that `promptPreview` is never stored and writes are throttled +- [x] 3.2 Make every model-calling run (automatic, web, CLI, slash command) acquire the host's `backfill_locks` row and report stale runs as stopped; verify with a two-process test that a second run for the same host is refused and that a dead PID's run shows as stopped +- [x] 3.3 Compute percentage and minutes left from the sample window, unknown until 5 samples over 60 seconds; verify with unit tests for the 400/1,000 at 5 per minute example and the early unknown case +- [x] 3.4 Add the paused flag: pause aborts an in-process run or is honoured between exchanges by another process, host start skips a paused backfill, resume clears it and starts a run; verify with tests for pause across a simulated restart on both hosts +- [x] 3.5 Add Run now, Pause, and Resume endpoints that start the backfill in the web server process, refuse when a run for the host is active, and are unavailable without a host unless the backfill model resolves to `external`; verify with endpoint tests including origin and auth rejection + +## 4. External API settings and key handling + +- [x] 4.1 Extend the global config writer allow-list with `memoryProvider`, `memoryApiUrl`, `memoryModel`, `memoryApiKey`, and `importPathMaps`, and reject a literal `memoryApiKey`; verify with writer tests that comments survive and a literal key is refused without writing +- [x] 4.2 Move the Windows user-only ACL code from `capture-diagnostics.ts` into a shared module and keep capture traces protected; verify with the existing capture trace ACL tests +- [x] 4.3 Add the key-source endpoint (env name, existing file path, pasted key saved to `~/.config/omms/secrets/.key` with folder `700` and file `600`, or the Windows ACL, confirmation before replace, refusal on a network bind without Basic Auth); verify with tests that the key never appears in responses, logs, or config and that permissions are user-only +- [x] 4.4 Report the key source type, reference, and whether it resolves in the web server process, with the shell-profile warning for `env://`; verify with a readiness test where the variable is missing in the server process +- [x] 4.5 Add the Test call for the saved external API with a small output limit and redacted errors; verify with a test that a rejected key produces a redacted error + +## 5. Web UI + +- [x] 5.1 Build the External API card with the three key sources and the Test button; verify with web component tests and a manual check in the running web app +- [x] 5.2 Add the External API option to both hosts' capture model cards and backfill model choices, disabled with the missing setting when not configured; verify with web tests +- [x] 5.3 Build the Directory maps section (saved maps, unresolved directories per host with counts, suggestions to accept, edit, or reject, save); verify with web tests and a manual check against a fixture history +- [x] 5.4 Add progress bars, percentage, done/total, minutes left, surface, and Run now/Pause/Resume to the Automatic import section, refreshing while a run is active; verify with web tests and by watching a CLI run on the page +- [x] 5.5 Show the running and global command versions with the mismatch warning and install or upgrade command in the Web app section; verify with web tests for equal, different, and missing global versions +- [x] 5.6 Add the new strings to every language the Settings page supports; verify with the existing i18n completeness test + +## 6. CLI + +- [x] 6.1 Add `--version` and `-v` to `om-memory-system`; verify with a CLI test that it prints the `package.json` version and exits `0` +- [x] 6.2 Implement the web server's global version lookup (no shell, 3-second timeout, 10-minute cache); verify with tests for found, missing, and timed-out commands, including on Windows paths + +## 7. Documentation + +- [x] 7.1 Rewrite `docs/cli.md` with a complete command reference: every terminal command, subcommand, and option, `--version`, and both hosts' slash commands; verify by comparing it against `src/cli/index.ts` and both adapters' command registrations +- [x] 7.2 Document global install as optional but highly recommended (`npm i -g om-memory-system` or `bun add -g om-memory-system`, upgrading, `om-memory-system --version`, the mismatch warning) in `docs/web-ui.md`, `README.md` setup, and `docs/upgrading.md`; verify the commands run +- [x] 7.3 Add the terminal proxy note (for example Orca's `*.orca.localhost` addresses differ from the printed `127.0.0.1` URL) to `docs/web-ui.md` +- [x] 7.4 Document the External API card, key sources, `external` model values, `importPathMaps`, progress, and Run now/Pause/Resume in `docs/configuration.md`, `docs/web-ui.md`, and both history import guides; verify with `bun run check` +- [x] 7.5 Record the key-file decision in `docs/adr/` and add it to the index + +## 8. Verification + +- [x] 8.1 Run the boundary tests (`host-neutral-capture-boundary`, `pi-adapter-boundary`, `plugin-bundle-boundary`) and confirm no new host imports in `src/core/` or `src/services/` +- [x] 8.2 Run `bun run ci:local` and `bun run check:package`, and run the security scan on changed files +- [x] 8.3 End-to-end: in the login web app with Pi and OpenCode closed, configure the external API with a key file, set Pi's backfill to `external`, add a suggested map, Run now, pause, resume, and confirm progress and the ledger match diff --git a/package.json b/package.json index 4106339d..d71d8f99 100644 --- a/package.json +++ b/package.json @@ -138,5 +138,10 @@ "*.{css,html,json,jsonc,md,yml,yaml}": [ "prettier --write" ] + }, + "allowScripts": { + "onnxruntime-node": true, + "msgpackr-extract": true, + "protobufjs": true } } diff --git a/src/adapters/opencode/backfill-models.ts b/src/adapters/opencode/backfill-models.ts index cef0c152..82e40f93 100644 --- a/src/adapters/opencode/backfill-models.ts +++ b/src/adapters/opencode/backfill-models.ts @@ -1,5 +1,6 @@ import { CONFIG } from "../../config.js"; import { parseBackfillModel } from "../../importer/backfill-model.js"; +import { resolveExternalBackfillModels } from "../../importer/external-backfill-models.js"; import { selectImportModel } from "../../importer/model-selection.js"; import type { HistoryImportModels } from "../../importer/run-import.js"; import { getAutoCaptureProviderStatus } from "../../services/ai/live-model-choice.js"; @@ -32,6 +33,9 @@ export async function resolveOpencodeBackfillModels( context: OpenCodeBackfillContext ): Promise<{ model: string; models: HistoryImportModels }> { const choice = parseBackfillModel(config, "opencode"); + if (choice === "external") { + return resolveExternalBackfillModels("opencode", config, context.externalModels); + } let ref: OpencodeModelRef; if (choice === "inherit") { const live = getAutoCaptureProviderStatus(config); @@ -49,7 +53,8 @@ export async function resolveOpencodeBackfillModels( : await context.configModel(); if (!selected) throw new Error("OpenCode backfill: no configured default model"); const parsed = parseBackfillModel({ opencodeBackfillModel: selected }, "opencode"); - if (parsed === "inherit") throw new Error("OpenCode backfill: no configured default model"); + if (typeof parsed === "string") + throw new Error("OpenCode backfill: no configured default model"); ref = { providerID: parsed.provider, modelID: parsed.model }; } else { ref = { providerID: choice.provider, modelID: choice.model }; diff --git a/src/adapters/opencode/backfill-startup.ts b/src/adapters/opencode/backfill-startup.ts index 02522b03..b01d68a5 100644 --- a/src/adapters/opencode/backfill-startup.ts +++ b/src/adapters/opencode/backfill-startup.ts @@ -1,5 +1,6 @@ import { CONFIG } from "../../config.js"; import { scheduleAutoBackfill } from "../../importer/auto-backfill.js"; +import { registerHostBackfillModels } from "../../importer/backfill-controls.js"; import { resolveOpencodeBackfillModels } from "./backfill-models.js"; /** Start after connected providers load, without delaying plugin initialisation. */ @@ -16,3 +17,12 @@ export function startOpencodeBackfill(input: { notify: input.notify, }); } + +/** Let Run now on a Settings page served by this process use OpenCode's models. */ +export function registerOpencodeBackfillModels(input: { + directory: string; + connected: string[]; + configModel: () => Promise; +}): void { + registerHostBackfillModels("opencode", () => resolveOpencodeBackfillModels(CONFIG, input)); +} diff --git a/src/adapters/opencode/import-command.ts b/src/adapters/opencode/import-command.ts index 4065ddd6..1b545ca3 100644 --- a/src/adapters/opencode/import-command.ts +++ b/src/adapters/opencode/import-command.ts @@ -93,6 +93,7 @@ export async function runOpencodeImportCommand(input: OpencodeImportCommandInput const report = await runHistoryImport("opencode", parsed, { cwd: input.directory, models, + track: { surface: "slash" }, onProgress: (processed, total, promptPreview) => { if (processed - lastProgress >= 25 || processed === total) { lastProgress = processed; diff --git a/src/adapters/pi/backfill-models.ts b/src/adapters/pi/backfill-models.ts index faa57d96..e8492de8 100644 --- a/src/adapters/pi/backfill-models.ts +++ b/src/adapters/pi/backfill-models.ts @@ -1,5 +1,6 @@ import { CONFIG } from "../../config.js"; import { parseBackfillModel } from "../../importer/backfill-model.js"; +import { resolveExternalBackfillModels } from "../../importer/external-backfill-models.js"; import type { HistoryImportModels } from "../../importer/run-import.js"; import { createPiLiveModels } from "./live-model.js"; import { adaptPiProfileModel } from "./profile.js"; @@ -8,9 +9,16 @@ import { createPiCaptureProvider, resolveImportModel, type PiModelContext } from /** Resolve the Pi backfill independently of the manual import's session-model rule. */ export async function resolvePiBackfillModels( ctx: PiModelContext, - config: { piBackfillModel?: string } = CONFIG + config: { + piBackfillModel?: string; + memoryProvider?: string; + memoryModel?: string; + memoryApiUrl?: string; + memoryApiKey?: string; + } = CONFIG ): Promise<{ model: string; models: HistoryImportModels }> { const choice = parseBackfillModel(config, "pi"); + if (choice === "external") return resolveExternalBackfillModels("pi", config); if (choice === "inherit") { const live = createPiLiveModels(ctx); const profile = live.profile(); diff --git a/src/adapters/pi/extension.ts b/src/adapters/pi/extension.ts index e9f827d1..8ff8dad3 100644 --- a/src/adapters/pi/extension.ts +++ b/src/adapters/pi/extension.ts @@ -135,6 +135,16 @@ export default function ommsPiExtension(pi: ExtensionAPI): void { ); } + // Run now on a Settings page served by this process can use Pi's signed-in models. + void Promise.all([ + import("../../importer/backfill-controls.js"), + import("./backfill-models.js"), + ]) + .then(([{ registerHostBackfillModels }, { resolvePiBackfillModels }]) => + registerHostBackfillModels("pi", () => resolvePiBackfillModels(ctx)) + ) + .catch(() => {}); + const globalScope = globalThis as any; const sessionId = ctx.sessionManager.getSessionId(); if ( diff --git a/src/adapters/pi/import-command.ts b/src/adapters/pi/import-command.ts index 898f7c2d..cdc7e895 100644 --- a/src/adapters/pi/import-command.ts +++ b/src/adapters/pi/import-command.ts @@ -114,6 +114,7 @@ export function registerPiHistoryImportCommand( let lastProgressNotify = 0; const report = await runHistoryImport("pi", parsed, { cwd: ctx.cwd, + track: { surface: "slash" }, models: selectedModel && !parsed.dryRun ? { diff --git a/src/cli/index.ts b/src/cli/index.ts index 9eeadc5d..aba54159 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -22,6 +22,9 @@ Commands: web Start the web app in the foreground web install|uninstall|status Manage the web app login item +Options: + --version, -v Print the installed version + Import commands take the same options; run one with --help to see them. Inside a session, /memory-import-opencode-history and /memory-import-pi-history use that session's model instead of an API key.`; @@ -36,6 +39,11 @@ export function parseImportArgs(argv: string[]): { host: ImportHost; args: Histo } export async function runCli(argv: string[]): Promise { + if (argv[0] === "--version" || argv[0] === "-v") { + const { packageVersion } = await import("../services/package-version.js"); + console.log(packageVersion()); + return 0; + } if (argv.length === 0 || argv[0] === "--help" || argv[0] === "-h") { console.log(usage); return 0; @@ -82,7 +90,11 @@ export async function runCli(argv: string[]): Promise { } const { formatHistoryImportReport, historyImportFailed, runHistoryImport } = await import("../importer/run-import.js"); - const report = await runHistoryImport(host, args, { cwd: process.cwd(), models }); + const report = await runHistoryImport(host, args, { + cwd: process.cwd(), + models, + track: { surface: "cli" }, + }); console.log(formatHistoryImportReport(host, report, model)); return historyImportFailed(report) ? 1 : 0; } catch (error) { diff --git a/src/config.ts b/src/config.ts index e5d16af9..34b6e5a0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,6 +7,7 @@ import { resolveSecretValue } from "./services/secret-resolver.js"; import { isPlaceholderApiKey } from "./services/ai/api-key-placeholder.js"; import { getAutoCaptureProviderStatus } from "./services/ai/live-model-choice.js"; import { parseBackfillModel } from "./importer/backfill-model.js"; +import { parseImportPathMaps } from "./importer/import-path-maps.js"; import { resolveDefaultStoragePath, runLegacyStoreMigration, @@ -82,6 +83,8 @@ interface OmmsConfig { autoBackfill?: boolean; opencodeBackfillModel?: string; piBackfillModel?: string; + /** Saved directory maps for every history import. Global config only. */ + importPathMaps?: Array<{ from: string; to: string }>; webServerAutoStart?: boolean; webServerEnabled?: boolean; webServerPort?: number; @@ -202,6 +205,7 @@ const DEFAULTS: Required< autoBackfill: true, opencodeBackfillModel: "inherit", piBackfillModel: "inherit", + importPathMaps: [], webServerAutoStart: true, webServerEnabled: true, webServerPort: 4747, @@ -352,8 +356,10 @@ export const CONFIG_TEMPLATE = `{ // Start a background import of past chats on each host's next start. "autoBackfill": true, - // "piBackfillModel": "inherit", // or "provider/model" - // "opencodeBackfillModel": "inherit", // or "provider/model" + // "piBackfillModel": "inherit", // or "external", or "provider/model" + // "opencodeBackfillModel": "inherit", // or "external", or "provider/model" + // Directory maps for history recorded in moved or deleted directories. + // "importPathMaps": [{ "from": "~/code/app-feat-x", "to": "~/code/app" }], // Register the web app to start when you log in. "webServerAutoStart": true, @@ -370,7 +376,7 @@ export const CONFIG_TEMPLATE = `{ // "webServerAuthPassword": "", // "webServerAuthUsername": "", - // Required when webServerHost is not loopback. Protects /api/* with Bearer / X-Opencode-Mem-Token. + // Required when webServerHost is not loopback. Protects /api/* with Bearer / X-Omms-Token (legacy X-Opencode-Mem-Token still works). // "webServerApiToken": "env://OMMS_WEB_TOKEN", // ============================================ @@ -409,7 +415,8 @@ export const CONFIG_TEMPLATE = `{ // Which model summarises your work, in this order: // 1. The host model below: opencodeProvider/opencodeModel in OpenCode, // piProvider/piModel in Pi. Set the model to "inherit" to follow - // whatever model the session is using. + // whatever model the session is using, or to "external" to send + // every call to the external API below. // 2. If no host model is set: the external API (memoryModel/memoryApiUrl/ // memoryApiKey further down). // 3. If neither is set: the session's own model. @@ -719,8 +726,22 @@ export function normalizeAutoCleanupRetentionDays(value: number): number { return value; } +/** + * The external API key, or undefined when its `env://` or `file://` source does + * not resolve in this process (a login web app does not see shell-profile + * variables). Readiness then reports the key as missing instead of the whole + * config failing to load. + */ +function resolveMemoryApiKey(value: string | undefined): string | undefined { + try { + return resolveSecretValue(value); + } catch { + return undefined; + } +} + function buildConfig(fileConfig: OmmsConfig) { - const memoryApiKey = resolveSecretValue(fileConfig.memoryApiKey); + const memoryApiKey = resolveMemoryApiKey(fileConfig.memoryApiKey); const embeddingDimensions = fileConfig.embeddingDimensions ?? getEmbeddingDimensions(fileConfig.embeddingModel ?? DEFAULTS.embeddingModel); @@ -731,6 +752,7 @@ function buildConfig(fileConfig: OmmsConfig) { fileConfig.userProfileAutoCleanupInterval ?? DEFAULTS.userProfileAutoCleanupInterval; parseBackfillModel(fileConfig, "pi"); parseBackfillModel(fileConfig, "opencode"); + const importPathMaps = parseImportPathMaps(fileConfig.importPathMaps); for (const key of ["autoBackfill", "webServerAutoStart"] as const) { if (fileConfig[key] !== undefined && typeof fileConfig[key] !== "boolean") { throw new Error(`Invalid ${key} config`); @@ -791,6 +813,7 @@ function buildConfig(fileConfig: OmmsConfig) { autoCaptureProviderStatus: getAutoCaptureProviderStatus({ opencodeProvider: fileConfig.opencodeProvider, opencodeModel: fileConfig.opencodeModel, + memoryProvider: fileConfig.memoryProvider, memoryModel: fileConfig.memoryModel, memoryApiUrl: fileConfig.memoryApiUrl, memoryApiKey, @@ -807,6 +830,7 @@ function buildConfig(fileConfig: OmmsConfig) { autoBackfill: fileConfig.autoBackfill ?? DEFAULTS.autoBackfill, opencodeBackfillModel: fileConfig.opencodeBackfillModel ?? DEFAULTS.opencodeBackfillModel, piBackfillModel: fileConfig.piBackfillModel ?? DEFAULTS.piBackfillModel, + importPathMaps, webServerAutoStart: fileConfig.webServerAutoStart ?? DEFAULTS.webServerAutoStart, webServerEnabled: fileConfig.webServerEnabled ?? DEFAULTS.webServerEnabled, webServerPort: fileConfig.webServerPort ?? DEFAULTS.webServerPort, @@ -995,6 +1019,7 @@ export function initConfig(directory: string, options: { strict?: boolean } = {} delete projectOverrides.autoBackfill; delete projectOverrides.opencodeBackfillModel; delete projectOverrides.piBackfillModel; + delete projectOverrides.importPathMaps; delete projectOverrides.webServerAutoStart; delete projectOverrides.webServerEnabled; const merged: OmmsConfig = { ...globalConfig, ...projectOverrides }; diff --git a/src/importer/auto-backfill.ts b/src/importer/auto-backfill.ts index 41c17b96..46e4fd42 100644 --- a/src/importer/auto-backfill.ts +++ b/src/importer/auto-backfill.ts @@ -5,6 +5,8 @@ import type { BackfillHost } from "./backfill-model.js"; import { tryAcquireBackfillLock } from "./backfill-lock.js"; import { getBackfillCutoff, + recordUnresolvedDirectories, + unresolvedDirectoriesOf, updateBackfillStatus, type BackfillStatus, } from "../services/backfill-state.js"; @@ -15,6 +17,8 @@ import { } from "./run-import.js"; import type { HistoryImportArgs } from "./import-args.js"; import { isManualImportRunning } from "./manual-import-guard.js"; +import { isBackfillPaused, type ImportSurface } from "./import-runs.js"; +import { workProgress } from "./import-progress.js"; type Counts = BackfillStatus["counts"]; type Run = typeof runHistoryImport; @@ -28,6 +32,10 @@ export interface AutoBackfillOptions { now?: () => number; enabled?: () => boolean; signal?: AbortSignal; + /** `web` for Run now and Resume on the Settings page. */ + surface?: ImportSurface; + /** Skip the start-up delay and the autoBackfill switch: the user asked for this run. */ + userStarted?: boolean; } const zero = (): Counts => ({ imported: 0, skipped: 0, failed: 0, pending: 0, unresolved: 0 }); @@ -54,16 +62,15 @@ const delay = (signal?: AbortSignal) => /** Schedule one host's history import after startup; callers do not await it. */ export async function scheduleAutoBackfill(options: AutoBackfillOptions): Promise { - await (options.wait ?? (() => delay(options.signal)))(); + if (!options.userStarted) await (options.wait ?? (() => delay(options.signal)))(); if (options.signal?.aborted) return; - if ( - !( - options.enabled ?? - (() => CONFIG.autoBackfill && process.env.OMMS_DISABLE_AUTO_BACKFILL !== "1") - )() - ) - return; + const enabled = + options.enabled ?? + (() => CONFIG.autoBackfill && process.env.OMMS_DISABLE_AUTO_BACKFILL !== "1"); + if (!options.userStarted && !enabled()) return; if (isManualImportRunning(options.host)) return; + // A paused backfill waits for Resume, across host starts. + if (await isBackfillPaused(options.host).catch(() => false)) return; const release = await tryAcquireBackfillLock(options.host); if (!release) return; const clock = options.now ?? Date.now; @@ -99,6 +106,13 @@ export async function scheduleAutoBackfill(options: AutoBackfillOptions): Promis signal: controller.signal, }); current = counts(dry); + await recordUnresolvedDirectories(options.host, unresolvedDirectoriesOf(dry)).catch( + (error: unknown) => + log("Backfill unresolved directories write failed", { + host: options.host, + error: error instanceof Error ? error.message : String(error), + }) + ); if (controller.signal.aborted) { state = "stopped"; return; @@ -152,14 +166,20 @@ export async function scheduleAutoBackfill(options: AutoBackfillOptions): Promis cwd: options.cwd, models, signal: controller.signal, - onProgress: (done, total) => { - processed = done; + track: { + surface: options.surface ?? "auto", + lockHeld: true, + expectedTotal: dry.unitsWouldImport, + }, + onProgress: (done, total, _preview, alreadyHandled = 0) => { + // Count only units that needed work; ledger hits are not pending work. + processed = workProgress(done, total, alreadyHandled, dry.unitsWouldImport).done; const now = clock(); if (now - checkedAt < 5_000) return; checkedAt = now; refreshConfigIfChanged(options.cwd); - if (!CONFIG.autoBackfill) controller.abort(); - current = { ...current, pending: Math.max(0, total - done) }; + if (!options.userStarted && !CONFIG.autoBackfill) controller.abort(); + current = { ...current, pending: Math.max(0, dry.unitsWouldImport - processed) }; pendingWrite = pendingWrite .then(() => updateBackfillStatus(options.host, { state: "running", model, counts: current }) @@ -174,8 +194,10 @@ export async function scheduleAutoBackfill(options: AutoBackfillOptions): Promis } ); await pendingWrite; - current = { ...counts(real), pending: Math.max(0, dry.unitsWouldImport - processed) }; - state = controller.signal.aborted ? "stopped" : real.unitsFailed ? "failed" : "done"; + const finished = real.unitsImported + real.unitsSkipped + real.unitsFailed; + current = { ...counts(real), pending: Math.max(0, dry.unitsWouldImport - finished) }; + const paused = await isBackfillPaused(options.host).catch(() => false); + state = controller.signal.aborted || paused ? "stopped" : real.unitsFailed ? "failed" : "done"; if (!options.signal?.aborted) { options.notify( `${options.host} automatic import ${state}: ${current.imported} imported, ${current.failed} failed. autoBackfill controls the next run.` diff --git a/src/importer/backfill-controls.ts b/src/importer/backfill-controls.ts new file mode 100644 index 00000000..5520e5cd --- /dev/null +++ b/src/importer/backfill-controls.ts @@ -0,0 +1,135 @@ +import { CONFIG } from "../config.js"; +import { log } from "../services/logger.js"; +import { + getAutoCaptureProviderStatus, + resolvePiLiveModel, +} from "../services/ai/live-model-choice.js"; +import { parseBackfillModel, type BackfillHost } from "./backfill-model.js"; +import { readImportRun, setBackfillPaused, type ImportRun } from "./import-runs.js"; +import type { HistoryImportModels } from "./run-import.js"; + +type ResolveModels = () => Promise<{ model: string; models: HistoryImportModels }>; + +/** A host runtime in this process that can resolve its signed-in backfill models. */ +const hostResolvers = new Map(); + +export function registerHostBackfillModels(host: BackfillHost, resolve: ResolveModels): void { + hostResolvers.set(host, resolve); +} + +/** A control request the page cannot carry out; `status` is the HTTP status. */ +export class BackfillControlError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message); + this.name = "BackfillControlError"; + } +} + +const label = (host: BackfillHost) => (host === "pi" ? "Pi" : "OpenCode"); + +/** True when the host's backfill model is the external API, directly or through the live rule. */ +export function backfillUsesExternal(host: BackfillHost): boolean { + const choice = parseBackfillModel(CONFIG, host); + if (choice === "external") return true; + if (choice !== "inherit") return false; + if (host === "pi") return resolvePiLiveModel(CONFIG).kind === "manual"; + const live = getAutoCaptureProviderStatus(CONFIG); + return live.ready && live.mode === "manual"; +} + +/** Why Run now is not available for the host in this process, or null when it is. */ +export function runNowUnavailableReason(host: BackfillHost): string | null { + if (hostResolvers.has(host)) return null; + if (!backfillUsesExternal(host)) { + return `Open ${label(host)}, or choose the external API for ${label(host)}'s backfill`; + } + return null; +} + +async function resolveModels( + host: BackfillHost +): Promise<{ model: string; models: HistoryImportModels }> { + const hostResolver = hostResolvers.get(host); + if (hostResolver && !backfillUsesExternal(host)) return hostResolver(); + const { resolveExternalBackfillModels } = await import("./external-backfill-models.js"); + return resolveExternalBackfillModels(host, CONFIG); +} + +export interface BackfillHostStatus { + run: ImportRun | null; + runNowUnavailable: string | null; +} + +/** Run now, Pause, and Resume for each host's backfill, run in this process. */ +export class BackfillControls { + private readonly controllers = new Map(); + + async status(): Promise> { + const entry = async (host: BackfillHost) => ({ + run: await readImportRun(host), + runNowUnavailable: runNowUnavailableReason(host), + }); + return { pi: await entry("pi"), opencode: await entry("opencode") }; + } + + async runNow(host: BackfillHost, cwd: string): Promise<{ started: true }> { + const reason = runNowUnavailableReason(host); + if (reason) throw new BackfillControlError(reason, 409); + const current = await readImportRun(host); + if (current?.state === "running" || this.controllers.has(host)) { + throw new BackfillControlError( + `${host === "pi" ? "A" : "An"} ${label(host)} import is already running`, + 409 + ); + } + if (current?.paused) { + throw new BackfillControlError(`${label(host)}'s backfill is paused; resume it first`, 409); + } + // Fail fast with the missing setting instead of starting a run that cannot call a model. + if (backfillUsesExternal(host) || !hostResolvers.has(host)) { + const { externalModelIssues } = await import("../services/ai/live-model-choice.js"); + const issues = externalModelIssues(CONFIG); + if (issues.length > 0) throw new BackfillControlError(issues.join("; "), 409); + } + const controller = new AbortController(); + this.controllers.set(host, controller); + const { scheduleAutoBackfill } = await import("./auto-backfill.js"); + void scheduleAutoBackfill({ + host, + cwd, + signal: controller.signal, + surface: "web", + userStarted: true, + resolveModels: () => resolveModels(host), + notify: (message) => + log("Backfill from the Settings page", { host, message: message.slice(0, 200) }), + }) + .catch((error: unknown) => + log("Backfill from the Settings page failed", { + host, + error: error instanceof Error ? error.name : "unknown", + }) + ) + .finally(() => { + if (this.controllers.get(host) === controller) this.controllers.delete(host); + }); + return { started: true }; + } + + /** Stop after the current exchange; a run in another process sees the flag at its next write. */ + async pause(host: BackfillHost): Promise<{ paused: true }> { + await setBackfillPaused(host, true); + this.controllers.get(host)?.abort(); + return { paused: true }; + } + + async resume(host: BackfillHost, cwd: string): Promise<{ started: true }> { + const reason = runNowUnavailableReason(host); + if (reason) throw new BackfillControlError(reason, 409); + await setBackfillPaused(host, false); + return this.runNow(host, cwd); + } +} diff --git a/src/importer/backfill-lock.ts b/src/importer/backfill-lock.ts index 0b65e454..8fc54597 100644 --- a/src/importer/backfill-lock.ts +++ b/src/importer/backfill-lock.ts @@ -6,7 +6,7 @@ import type { BackfillHost } from "./backfill-model.js"; type LockState = { pid: number; token: string }; -function live(pid: number): boolean { +export function isProcessAlive(pid: number): boolean { if (!Number.isSafeInteger(pid) || pid <= 0) return false; try { process.kill(pid, 0); @@ -37,7 +37,7 @@ export async function tryAcquireBackfillLock( [host] ); if (!holder) continue; - if (live(holder.pid)) return null; + if (isProcessAlive(holder.pid)) return null; const replaced = await db.run( "UPDATE backfill_locks SET pid = ?, token = ? WHERE host = ? AND pid = ? AND token = ?", [state.pid, state.token, host, holder.pid, holder.token] diff --git a/src/importer/backfill-model.ts b/src/importer/backfill-model.ts index d6a73a2c..1ed7446b 100644 --- a/src/importer/backfill-model.ts +++ b/src/importer/backfill-model.ts @@ -1,5 +1,5 @@ export type BackfillHost = "pi" | "opencode"; -export type BackfillModel = "inherit" | { provider: string; model: string }; +export type BackfillModel = "inherit" | "external" | { provider: string; model: string }; /** Parse the global backfill model without importing the runtime config. */ export function parseBackfillModel( @@ -8,6 +8,7 @@ export function parseBackfillModel( ): BackfillModel { const value = host === "pi" ? config.piBackfillModel : config.opencodeBackfillModel; if (value === undefined || value === "inherit") return "inherit"; + if (value === "external") return "external"; if (typeof value !== "string") throw new Error(`Invalid ${host}BackfillModel config`); const slash = value.indexOf("/"); if (slash <= 0 || slash === value.length - 1) { diff --git a/src/importer/external-api-test.ts b/src/importer/external-api-test.ts new file mode 100644 index 00000000..6796667b --- /dev/null +++ b/src/importer/external-api-test.ts @@ -0,0 +1,63 @@ +import { randomUUID } from "node:crypto"; +import { CONFIG } from "../config.js"; +import { safeHealthError } from "../services/safe-health-error.js"; + +const OUTPUT_LIMIT = 64; +const TIMEOUT_MS = 20_000; + +/** Provider-specific output caps; Anthropic-style providers read `maxTokens`. */ +function outputLimitParams(provider: string): Record { + if (provider === "openai-chat" || provider === "orcarouter") return { max_tokens: OUTPUT_LIMIT }; + if (provider === "openai-responses") return { max_output_tokens: OUTPUT_LIMIT }; + return {}; +} + +export type ExternalApiTestResult = { ok: true; model: string } | { ok: false; error: string }; + +/** + * One small call to the saved external API from this process, with a small + * output limit. The error never contains the key. + */ +export async function testExternalApi( + config: typeof CONFIG = CONFIG, + createProvider?: ( + provider: string, + providerConfig: Record + ) => { executeToolCall: (...args: any[]) => Promise<{ success: boolean; error?: string }> } +): Promise { + const secrets = [config.memoryApiKey]; + try { + const { buildMemoryProviderConfig } = await import("../services/ai/provider-config.js"); + const base = buildMemoryProviderConfig(config, { + maxIterations: 1, + iterationTimeout: TIMEOUT_MS, + }); + const providerConfig = { + ...base, + maxTokens: OUTPUT_LIMIT, + extraParams: { ...base.extraParams, ...outputLimitParams(config.memoryProvider) }, + }; + const create = + createProvider ?? + (await import("../services/ai/ai-provider-factory.js")).AIProviderFactory.createProvider; + const provider = create(config.memoryProvider as never, providerConfig as never); + const tool = { + type: "function" as const, + function: { + name: "acknowledge", + description: "Confirm the connection works", + parameters: { type: "object", properties: { ok: { type: "boolean" } }, required: ["ok"] }, + }, + }; + const result = await provider.executeToolCall( + "You are testing a connection. Call the acknowledge tool with ok set to true.", + "Acknowledge.", + tool, + `external-api-test-${randomUUID()}` + ); + if (!result.success) throw new Error(result.error ?? "The external API call failed"); + return { ok: true, model: `${config.memoryProvider}/${config.memoryModel ?? "default"}` }; + } catch (error) { + return { ok: false, error: safeHealthError(error, secrets).slice(0, 500) }; + } +} diff --git a/src/importer/external-backfill-models.ts b/src/importer/external-backfill-models.ts new file mode 100644 index 00000000..05c23b8b --- /dev/null +++ b/src/importer/external-backfill-models.ts @@ -0,0 +1,28 @@ +import { externalModelIssues } from "../services/ai/live-model-choice.js"; +import type { BackfillHost } from "./backfill-model.js"; +import type { SelectedImportModel } from "./model-selection.js"; +import type { HistoryImportModels } from "./run-import.js"; + +interface ExternalConfig { + memoryProvider?: string; + memoryModel?: string; + memoryApiUrl?: string; + memoryApiKey?: string; +} + +/** A backfill set to `external`: the external API, or an error naming the missing settings. */ +export async function resolveExternalBackfillModels( + host: BackfillHost, + config: ExternalConfig, + externalModels?: (flags: Record) => SelectedImportModel +): Promise<{ model: string; models: HistoryImportModels }> { + const issues = externalModelIssues(config); + const label = host === "pi" ? "Pi" : "OpenCode"; + if (issues.length > 0) throw new Error(`${label} backfill: ${issues.join("; ")}`); + const select = externalModels ?? (await import("./model-selection.js")).selectImportModel; + const external = select({}); + return { + model: `${external.provider}/${external.modelId}`, + models: { capture: external.capture, profile: external.profile }, + }; +} diff --git a/src/importer/import-path-maps.ts b/src/importer/import-path-maps.ts new file mode 100644 index 00000000..7c533c6f --- /dev/null +++ b/src/importer/import-path-maps.ts @@ -0,0 +1,58 @@ +import { homedir } from "node:os"; +import { isAbsolute, join, resolve } from "node:path"; +import type { ImportPathMap } from "./importer.js"; + +function expandHome(path: string): string { + if (path === "~") return homedir(); + if (path.startsWith("~/") || path.startsWith("~\\")) return join(homedir(), path.slice(2)); + return path; +} + +/** + * Validate and normalise `importPathMaps` from the global config. Each entry + * needs absolute `from` and `to` directories after `~` expansion. Throws on + * the first invalid entry, like the rest of config validation. + */ +export function parseImportPathMaps(value: unknown): ImportPathMap[] { + if (value === undefined) return []; + if (!Array.isArray(value)) throw new Error("Invalid importPathMaps config: must be a list"); + return value.map((entry, index) => { + const from = (entry as { from?: unknown } | null)?.from; + const to = (entry as { to?: unknown } | null)?.to; + if (typeof from !== "string" || typeof to !== "string" || !from.trim() || !to.trim()) { + throw new Error(`Invalid importPathMaps config: entry ${index} needs "from" and "to"`); + } + const expanded = { from: expandHome(from.trim()), to: expandHome(to.trim()) }; + if (!isAbsolute(expanded.from) || !isAbsolute(expanded.to)) { + throw new Error(`Invalid importPathMaps config: entry ${index} needs absolute paths`); + } + return { from: resolve(expanded.from), to: resolve(expanded.to) }; + }); +} + +/** Saved maps with the run's own maps on top; a run map wins for the same `from`. */ +export function mergeImportPathMaps( + saved: readonly ImportPathMap[], + run: readonly ImportPathMap[] +): ImportPathMap[] { + const byFrom = new Map(); + for (const map of saved) byFrom.set(map.from, map); + for (const map of run) byFrom.set(map.from, map); + return [...byFrom.values()]; +} + +/** + * The maps one import run uses: the saved `importPathMaps` with the run's + * maps on top, run targets resolved against `cwd`. + */ +export async function runPathMaps( + runMaps: readonly ImportPathMap[], + cwd: string, + saved?: readonly ImportPathMap[] +): Promise { + const savedMaps = saved ?? (await import("../config.js")).CONFIG.importPathMaps ?? []; + return mergeImportPathMaps( + savedMaps, + runMaps.map((map) => ({ from: map.from, to: resolve(cwd, map.to) })) + ); +} diff --git a/src/importer/import-progress.ts b/src/importer/import-progress.ts new file mode 100644 index 00000000..8f5eccb4 --- /dev/null +++ b/src/importer/import-progress.ts @@ -0,0 +1,66 @@ +/** One `(time, done)` reading of a running import. */ +export interface ProgressSample { + at: number; + done: number; +} + +export const MAX_PROGRESS_SAMPLES = 20; +const MIN_SAMPLES = 5; +const MIN_WINDOW_MS = 60_000; + +/** Keep the most recent samples; a sample with no new units only moves the clock. */ +export function addProgressSample( + samples: readonly ProgressSample[], + sample: ProgressSample +): ProgressSample[] { + return [...samples, sample].slice(-MAX_PROGRESS_SAMPLES); +} + +export interface ProgressEstimate { + percent: number; + /** Null until the recent window has 5 samples spanning a minute and a non-zero rate. */ + minutesLeft: number | null; + /** Units a minute over the recent window, or null while unknown. */ + ratePerMinute: number | null; +} + +/** Percentage done and minutes left from the recent sample window, not the whole run. */ +export function estimateProgress( + total: number, + done: number, + samples: readonly ProgressSample[] +): ProgressEstimate { + const percent = total > 0 ? Math.min(100, Math.floor((done / total) * 100)) : 0; + if (samples.length < MIN_SAMPLES) return { percent, minutesLeft: null, ratePerMinute: null }; + const first = samples[0]!; + const last = samples[samples.length - 1]!; + const span = last.at - first.at; + if (span < MIN_WINDOW_MS || last.done <= first.done) { + return { percent, minutesLeft: null, ratePerMinute: null }; + } + const ratePerMinute = (last.done - first.done) / (span / 60_000); + return { + percent, + minutesLeft: Math.max(0, Math.round((total - done) / ratePerMinute)), + ratePerMinute, + }; +} + +/** + * Turn the importer's progress callback into finished work units. The + * importer reports a unit as it starts, and `alreadyHandled` (units found in + * the ledger, which need no model call) does not yet include that unit, so + * finished work is `processed - 1 - alreadyHandled`. + */ +export function workProgress( + processed: number, + total: number, + alreadyHandled = 0, + expectedTotal?: number +): { done: number; total: number } { + const workTotal = Math.max(0, expectedTotal ?? total - alreadyHandled); + return { + done: Math.min(workTotal, Math.max(0, processed - 1 - alreadyHandled)), + total: workTotal, + }; +} diff --git a/src/importer/import-runs.ts b/src/importer/import-runs.ts new file mode 100644 index 00000000..0425c236 --- /dev/null +++ b/src/importer/import-runs.ts @@ -0,0 +1,207 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { CONFIG } from "../config.js"; +import { safeHealthError } from "../services/safe-health-error.js"; +import { tursoConnectionManager } from "../services/turso/connection-manager.js"; +import type { BackfillHost } from "./backfill-model.js"; +import { isProcessAlive } from "./backfill-lock.js"; +import { + addProgressSample, + estimateProgress, + type ProgressEstimate, + type ProgressSample, +} from "./import-progress.js"; + +export type ImportSurface = "auto" | "web" | "cli" | "slash"; +export type ImportRunState = "running" | "paused" | "stopped" | "done" | "failed"; + +export interface ImportRunCounts { + total: number; + done: number; + imported: number; + skipped: number; + failed: number; +} + +export interface ImportRun extends ImportRunCounts, ProgressEstimate { + host: BackfillHost; + surface: ImportSurface | null; + state: ImportRunState | null; + pid: number | null; + startedAt: number | null; + updatedAt: number | null; + error: string | null; + /** The user paused this host's backfill; automatic runs wait for Resume. */ + paused: boolean; +} + +const dbPath = (storagePath = CONFIG.storagePath) => join(storagePath, "import-ledger.db"); + +async function table(storagePath?: string) { + const db = await tursoConnectionManager.getConnection(dbPath(storagePath)); + await db.run(`CREATE TABLE IF NOT EXISTS import_runs ( + host TEXT PRIMARY KEY, surface TEXT, state TEXT, pid INTEGER, started_at INTEGER, + total INTEGER NOT NULL DEFAULT 0, done INTEGER NOT NULL DEFAULT 0, + imported INTEGER NOT NULL DEFAULT 0, skipped INTEGER NOT NULL DEFAULT 0, + failed INTEGER NOT NULL DEFAULT 0, error TEXT, updated_at INTEGER, + samples TEXT, paused INTEGER NOT NULL DEFAULT 0 + )`); + // A row per host keeps every later statement a plain UPDATE. + await db.run("INSERT OR IGNORE INTO import_runs (host) VALUES ('pi'), ('opencode')"); + return db; +} + +const count = (value: number) => (Number.isSafeInteger(value) && value >= 0 ? value : 0); + +/** Read a host's run without creating the store; a running row whose process is gone reads as stopped. */ +export async function readImportRun(host: BackfillHost): Promise { + if (!existsSync(dbPath())) return null; + const db = await tursoConnectionManager.getConnection(dbPath()); + const exists = await db.get( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'import_runs'" + ); + if (!exists) return null; + const row = await db.get>("SELECT * FROM import_runs WHERE host = ?", [ + host, + ]); + if (!row) return null; + const pid = row.pid === null ? null : Number(row.pid); + let state = (row.state ?? null) as ImportRunState | null; + if (state === "running" && (pid === null || !isProcessAlive(pid))) state = "stopped"; + const total = Number(row.total); + const done = Number(row.done); + const samples = row.samples ? (JSON.parse(String(row.samples)) as ProgressSample[]) : []; + return { + host, + surface: (row.surface ?? null) as ImportSurface | null, + state, + pid, + startedAt: row.started_at === null ? null : Number(row.started_at), + updatedAt: row.updated_at === null ? null : Number(row.updated_at), + total, + done, + imported: Number(row.imported), + skipped: Number(row.skipped), + failed: Number(row.failed), + error: row.error === null ? null : String(row.error), + paused: Number(row.paused) === 1, + ...(state === "running" + ? estimateProgress(total, done, samples) + : { + percent: total > 0 ? Math.min(100, Math.floor((done / total) * 100)) : 0, + minutesLeft: null, + ratePerMinute: null, + }), + }; +} + +export async function isBackfillPaused(host: BackfillHost): Promise { + return (await readImportRun(host))?.paused ?? false; +} + +export async function setBackfillPaused(host: BackfillHost, paused: boolean): Promise { + const db = await table(); + await db.run("UPDATE import_runs SET paused = ?, updated_at = ? WHERE host = ?", [ + paused ? 1 : 0, + Date.now(), + host, + ]); +} + +export interface ImportRunRecorder { + /** Feed from `onProgress`; writes at most once per interval and checks the paused flag. */ + progress(done: number, total: number): void; + /** True after a progress write saw the paused flag. */ + readonly pauseRequested: boolean; + finish( + state: Exclude, + counts: Partial, + error?: unknown + ): Promise; +} + +/** + * Start a host's run record. Only numbers, the surface, and the PID are + * stored; `promptPreview` and all other conversation content never are. + */ +export async function startImportRun( + host: BackfillHost, + surface: ImportSurface, + options: { + now?: () => number; + intervalMs?: number; + sampleIntervalMs?: number; + onPause?: () => void; + } = {} +): Promise { + const clock = options.now ?? Date.now; + const interval = options.intervalMs ?? 1_000; + const sampleInterval = options.sampleIntervalMs ?? 15_000; + const db = await table(); + const startedAt = clock(); + await db.run( + `UPDATE import_runs SET surface = ?, state = 'running', pid = ?, started_at = ?, total = 0, + done = 0, imported = 0, skipped = 0, failed = 0, error = NULL, updated_at = ?, samples = '[]' + WHERE host = ?`, + [surface, process.pid, startedAt, startedAt, host] + ); + let samples: ProgressSample[] = []; + let lastWrite = Number.NEGATIVE_INFINITY; + let lastSample = Number.NEGATIVE_INFINITY; + let latest = { done: 0, total: 0 }; + let pauseRequested = false; + let writing: Promise = Promise.resolve(); + const write = () => { + const now = clock(); + lastWrite = now; + // Sample less often than we write, so the window spans minutes, not seconds. + if (now - lastSample >= sampleInterval) { + lastSample = now; + samples = addProgressSample(samples, { at: now, done: latest.done }); + } + const snapshot = { ...latest, samples: JSON.stringify(samples) }; + writing = writing + .then(async () => { + await db.run( + "UPDATE import_runs SET total = ?, done = ?, samples = ?, updated_at = ? WHERE host = ?", + [count(snapshot.total), count(snapshot.done), snapshot.samples, now, host] + ); + const row = await db.get<{ paused: number }>( + "SELECT paused FROM import_runs WHERE host = ?", + [host] + ); + if (Number(row?.paused) === 1 && !pauseRequested) { + pauseRequested = true; + options.onPause?.(); + } + }) + .catch(() => {}); + }; + return { + progress(done, total) { + latest = { done, total }; + if (clock() - lastWrite >= interval) write(); + }, + get pauseRequested() { + return pauseRequested; + }, + async finish(state, counts, error) { + await writing; + await db.run( + `UPDATE import_runs SET state = ?, total = ?, done = ?, imported = ?, skipped = ?, + failed = ?, error = ?, updated_at = ? WHERE host = ?`, + [ + state, + count(counts.total ?? latest.total), + count(counts.done ?? latest.done), + count(counts.imported ?? 0), + count(counts.skipped ?? 0), + count(counts.failed ?? 0), + error == null ? null : safeHealthError(error, [CONFIG.memoryApiKey]).slice(0, 500), + clock(), + host, + ] + ); + }, + }; +} diff --git a/src/importer/import-sessions.ts b/src/importer/import-sessions.ts index 37a59c59..de1d660f 100644 --- a/src/importer/import-sessions.ts +++ b/src/importer/import-sessions.ts @@ -127,16 +127,11 @@ export async function matchImportSessions( options.scope === "current-project" ? projectFilterTag(resolve(options.cwd, options.project ?? ".")) : null; - // Map targets resolve against the working directory, as `runHistoryImport` does. + // Saved maps plus the request's own, as `runHistoryImport` uses them. + const { runPathMaps } = await import("./import-path-maps.js"); const rows = await readRows( identity, - { - ...options, - pathMaps: options.pathMaps.map((map) => ({ - from: map.from, - to: resolve(options.cwd, map.to), - })), - }, + { ...options, pathMaps: await runPathMaps(options.pathMaps, options.cwd) }, snapshotMode ); const unresolved = rows.filter((row) => row.directory === null); @@ -239,6 +234,15 @@ export async function listImportSessions( // A refresh takes a fresh copy of a live database; paging reuses the current one. const listedAt = Date.now(); const matched = await matchImportSessions(identity, options, request.refresh ? "fresh" : "any"); + // Feed the Directory maps list; a failure here must not break the listing. + const { recordUnresolvedDirectories, summarizeUnresolvedDirectories } = + await import("../services/backfill-state.js"); + await recordUnresolvedDirectories( + options.host, + summarizeUnresolvedDirectories( + matched.unresolved.map((row) => ({ directory: row.recordedDirectory })) + ) + ).catch(() => {}); const rows = [ ...matched.matching.map((row) => ({ ...row, selectable: true })), ...(options.scope === "all-projects" diff --git a/src/importer/importer.ts b/src/importer/importer.ts index 248aa596..416e7b16 100644 --- a/src/importer/importer.ts +++ b/src/importer/importer.ts @@ -105,7 +105,13 @@ export interface ImportReport { export interface ImporterDeps { loadSession: (file: string) => LoadedPiSession; provider: CaptureSummaryProvider; - onProgress?: (processed: number, total: number, promptPreview: string) => void; + /** `alreadyHandled`: units found already done so far, which need no model call. */ + onProgress?: ( + processed: number, + total: number, + promptPreview: string, + alreadyHandled?: number + ) => void; signal?: AbortSignal; ledger?: PiImportLedger; profile?: { model?: ModelPort; batchSize?: number }; @@ -306,7 +312,12 @@ export async function importHistorySource( const { sessionId, directory, sourceFile, hash, window } = candidate; const key = buildImportKey(sessionId, window, host)!; processed++; - deps.onProgress?.(processed, report.unitsTotal, preview(window.userPrompt)); + deps.onProgress?.( + processed, + report.unitsTotal, + preview(window.userPrompt), + report.unitsAlreadyHandled + ); const unit: ImportUnitReport = { key, diff --git a/src/importer/map-suggestions.ts b/src/importer/map-suggestions.ts new file mode 100644 index 00000000..297fb6ad --- /dev/null +++ b/src/importer/map-suggestions.ts @@ -0,0 +1,141 @@ +import { existsSync, readdirSync, statSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; +import type { UnresolvedDirectory } from "../services/backfill-state.js"; + +const PROJECT_MARKERS = [".git", "package.json", "pyproject.toml", "Cargo.toml", "go.mod"]; + +function isDirectory(path: string): boolean { + try { + return statSync(path).isDirectory(); + } catch { + return false; + } +} + +function isProject(path: string): boolean { + return PROJECT_MARKERS.some((marker) => existsSync(join(path, marker))); +} + +function childDirectories(parent: string): string[] { + try { + return readdirSync(parent, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => join(parent, entry.name)); + } catch { + return []; + } +} + +/** `app` is a leading part of `app`, `app-feat-x`, and `app-feat-x-2`, not of `apple`. */ +function isLeadingPart(candidate: string, name: string): boolean { + return name === candidate || name.startsWith(`${candidate}-`); +} + +export interface MapSuggestionContext { + /** Existing project directories the user has history for; their parents are searched too. */ + knownProjects?: readonly string[]; + /** OpenCode's recorded project worktree for a session directory, read without writing. */ + opencodeWorktree?: (directory: string) => string | null | undefined; +} + +/** + * Suggest an existing target for a directory that no longer exists: first the + * main repository of a deleted worktree, by the longest leading part of the + * directory's name or one of its parents' names, then OpenCode's recorded + * project worktree. Returns null when there is no candidate. + */ +export function suggestMapTarget( + missing: string, + context: MapSuggestionContext = {} +): string | null { + const searchRoots = new Set(); + for (const project of context.knownProjects ?? []) { + if (!isDirectory(project)) continue; + searchRoots.add(dirname(project)); + } + let ancestor = missing; + while (dirname(ancestor) !== ancestor) { + const name = basename(ancestor); + const parents = new Set([dirname(ancestor), ...searchRoots]); + let best: string | null = null; + for (const parent of parents) { + for (const candidate of childDirectories(parent)) { + if (candidate === ancestor || !isLeadingPart(basename(candidate), name)) continue; + if (!isProject(candidate)) continue; + if (!best || basename(candidate).length > basename(best).length) best = candidate; + } + } + if (best) return best; + ancestor = dirname(ancestor); + } + const worktree = context.opencodeWorktree?.(missing); + if (worktree && worktree !== "/" && isDirectory(worktree)) return worktree; + return null; +} + +export interface SuggestedDirectory extends UnresolvedDirectory { + suggestion: string | null; +} + +export function suggestMapTargets( + directories: readonly UnresolvedDirectory[], + context: MapSuggestionContext = {} +): SuggestedDirectory[] { + return directories.map((item) => ({ + ...item, + suggestion: suggestMapTarget(item.directory, context), + })); +} + +/** + * OpenCode's recorded project worktree for each session directory, through + * the importer's read-only reader. OpenCode's database is never written. + */ +export async function readOpencodeWorktrees(dbPath: string): Promise> { + const { openOpencodeHistory } = await import("./opencode-reader.js"); + const history = await openOpencodeHistory(dbPath); + try { + const worktrees = new Map(); + for (const session of history.sessions) { + if (session.recordedDirectory && session.projectWorktree) { + worktrees.set(session.recordedDirectory, session.projectWorktree); + } + } + return worktrees; + } finally { + await history.close(); + } +} + +/** + * The Directory maps list: saved maps and each host's unresolved directories + * from its latest run, with suggestions. Paths and counts only. + */ +export async function directoryMapsView( + options: { opencodeDbPath?: string; knownProjects?: readonly string[] } = {} +) { + const { CONFIG } = await import("../config.js"); + const { readUnresolvedDirectories } = await import("../services/backfill-state.js"); + const { DEFAULT_OPENCODE_DB } = await import("./opencode-reader.js"); + const saved = new Set(CONFIG.importPathMaps.map((map) => map.from)); + let worktrees = new Map(); + const dbPath = options.opencodeDbPath ?? DEFAULT_OPENCODE_DB; + if (existsSync(dbPath)) { + worktrees = await readOpencodeWorktrees(dbPath).catch(() => new Map()); + } + const knownProjects = [ + ...(options.knownProjects ?? []), + ...CONFIG.importPathMaps.map((map) => map.to), + ...worktrees.values(), + ]; + const host = async (name: "pi" | "opencode") => { + const pending = (await readUnresolvedDirectories(name)).filter( + (item) => !saved.has(item.directory) + ); + return suggestMapTargets(pending, { + knownProjects, + ...(name === "opencode" ? { opencodeWorktree: (dir: string) => worktrees.get(dir) } : {}), + }); + }; + return { saved: CONFIG.importPathMaps, pi: await host("pi"), opencode: await host("opencode") }; +} diff --git a/src/importer/opencode-import.ts b/src/importer/opencode-import.ts index c69e39d2..a826f90a 100644 --- a/src/importer/opencode-import.ts +++ b/src/importer/opencode-import.ts @@ -45,7 +45,13 @@ export interface OpencodeImportOptions { profileBatch?: number; provider?: CaptureSummaryProvider; profileModel?: ModelPort; - onProgress?: (processed: number, total: number, promptPreview: string) => void; + /** `alreadyHandled`: units found already done so far, which need no model call. */ + onProgress?: ( + processed: number, + total: number, + promptPreview: string, + alreadyHandled?: number + ) => void; signal?: AbortSignal; } diff --git a/src/importer/run-import.ts b/src/importer/run-import.ts index f2a5851e..0b9cf238 100644 --- a/src/importer/run-import.ts +++ b/src/importer/run-import.ts @@ -2,8 +2,9 @@ import { resolve } from "node:path"; import type { CaptureSummaryProvider } from "../core/host.js"; import type { ModelPort } from "../core/profile-analysis.js"; import type { HistoryImportArgs, ImportHost } from "./import-args.js"; -import type { ImportReport } from "./importer.js"; +import type { ImportPathMap, ImportReport } from "./importer.js"; import type { UnresolvedProject } from "./opencode-project.js"; +import type { ImportSurface } from "./import-runs.js"; /** The two model roles an import uses; absent in a dry run or when the step is skipped. */ export interface HistoryImportModels { @@ -15,8 +16,18 @@ export interface HistoryImportRun { /** Directory that relative paths and the default project resolve against. */ cwd: string; models: HistoryImportModels; - onProgress?: (processed: number, total: number, promptPreview: string) => void; + /** `alreadyHandled`: units found already done so far, which need no model call. */ + onProgress?: ( + processed: number, + total: number, + promptPreview: string, + alreadyHandled?: number + ) => void; signal?: AbortSignal; + /** Record progress and take the host's lock for a run that calls models. */ + track?: { surface: ImportSurface; lockHeld?: boolean; expectedTotal?: number }; + /** Saved directory maps; defaults to the global `importPathMaps`. */ + savedPathMaps?: readonly ImportPathMap[]; /** Web selection, already re-resolved against the source; `args.source` is its real path. */ selection?: { keys: string[]; @@ -37,15 +48,103 @@ const dryRunCapture: CaptureSummaryProvider = { }, }; -/** Run one host's history import with already-parsed, validated arguments. */ +/** Another process or surface holds this host's import lock. */ +export class ImportAlreadyRunningError extends Error { + constructor(host: ImportHost) { + super(`${host === "pi" ? "A Pi" : "An OpenCode"} import is already running`); + this.name = "ImportAlreadyRunningError"; + } +} + +/** + * Run one host's history import with already-parsed, validated arguments. + * With `track`, a run that calls models takes the host's cross-process lock + * and records its progress; a dry run never does either. + */ export async function runHistoryImport( host: ImportHost, args: HistoryImportArgs, run: HistoryImportRun +): Promise { + if (!run.track || args.dryRun) return runUntracked(host, args, run); + let release: (() => Promise) | null = null; + if (!run.track.lockHeld) { + const { tryAcquireBackfillLock } = await import("./backfill-lock.js"); + release = await tryAcquireBackfillLock(host); + if (!release) throw new ImportAlreadyRunningError(host); + } + try { + const { startImportRun } = await import("./import-runs.js"); + const { workProgress } = await import("./import-progress.js"); + const controller = new AbortController(); + const onAbort = () => controller.abort(); + run.signal?.addEventListener("abort", onAbort, { once: true }); + if (run.signal?.aborted) controller.abort(); + const recorder = await startImportRun(host, run.track.surface, { + onPause: () => controller.abort(), + }); + try { + const report = await runUntracked(host, args, { + ...run, + signal: controller.signal, + onProgress: (processed, total, preview, alreadyHandled = 0) => { + // Units already in the ledger need no model call: leave them out of + // the progress so the bar and the time left describe real work. + const work = workProgress(processed, total, alreadyHandled, run.track!.expectedTotal); + recorder.progress(work.done, work.total); + run.onProgress?.(processed, total, preview, alreadyHandled); + }, + }); + const handled = report.unitsImported + report.unitsSkipped + report.unitsFailed; + await recorder.finish( + recorder.pauseRequested ? "paused" : controller.signal.aborted ? "stopped" : "done", + { + // The total stays the one progress last reported; a real run has no + // "would import" count of its own. + done: handled, + imported: report.unitsImported, + skipped: report.unitsSkipped, + failed: report.unitsFailed, + } + ); + await recordUnresolved(host, report); + return report; + } catch (error) { + await recorder + .finish( + recorder.pauseRequested ? "paused" : controller.signal.aborted ? "stopped" : "failed", + {}, + error + ) + .catch(() => {}); + throw error; + } finally { + run.signal?.removeEventListener("abort", onAbort); + } + } finally { + await release?.(); + } +} + +async function recordUnresolved(host: ImportHost, report: HistoryImportReport): Promise { + try { + const { recordUnresolvedDirectories, unresolvedDirectoriesOf } = + await import("../services/backfill-state.js"); + await recordUnresolvedDirectories(host, unresolvedDirectoriesOf(report)); + } catch { + // The list is a convenience for the Settings page; the import itself succeeded. + } +} + +async function runUntracked( + host: ImportHost, + args: HistoryImportArgs, + run: HistoryImportRun ): Promise { const at = (path: string) => resolve(run.cwd, path); const project = args.scope === "current-project" ? at(args.project ?? ".") : undefined; - const pathMaps = args.pathMaps.map((map) => ({ from: map.from, to: at(map.to) })); + const { runPathMaps } = await import("./import-path-maps.js"); + const pathMaps = await runPathMaps(args.pathMaps, run.cwd, run.savedPathMaps); const capture = run.models.capture ?? dryRunCapture; if (host === "opencode") { diff --git a/src/importer/web-import-api.ts b/src/importer/web-import-api.ts index ba4a6968..0c663e49 100644 --- a/src/importer/web-import-api.ts +++ b/src/importer/web-import-api.ts @@ -6,3 +6,6 @@ export { importReadiness } from "./import-readiness.js"; export { browseImportSources, validateImportSource } from "./import-sources.js"; export { listImportSessions, validateSessionListRequest } from "./import-sessions.js"; export { opencodeSnapshots, sweepOrphanSnapshots } from "./opencode-snapshot.js"; +export { BackfillControls } from "./backfill-controls.js"; +export { directoryMapsView } from "./map-suggestions.js"; +export { testExternalApi } from "./external-api-test.js"; diff --git a/src/importer/web-import-jobs.ts b/src/importer/web-import-jobs.ts index 02084164..d5d28fdb 100644 --- a/src/importer/web-import-jobs.ts +++ b/src/importer/web-import-jobs.ts @@ -239,6 +239,7 @@ export class SettingsImportJobs { cwd: directory, models, signal: controller.signal, + track: { surface: "web" }, selection: { keys: selection.keys, cutoff: selection.cutoff, diff --git a/src/index.ts b/src/index.ts index 8204108e..2bf6b82e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -305,16 +305,23 @@ export const OmmsPlugin: Plugin = async (ctx: PluginInput) => { } catch (error) { log("Failed to initialize opencode provider state", { error: String(error) }); } + const configModel = async () => { + const response = await ctx.client.config.get(); + return unwrapSdkData<{ model?: string }>(response)?.model ?? null; + }; + // Run now on a Settings page served by this process can use OpenCode's signed-in models. + void import("./adapters/opencode/backfill-startup.js") + .then(({ registerOpencodeBackfillModels }) => + registerOpencodeBackfillModels({ connected, directory, configModel }) + ) + .catch(() => {}); if (CONFIG.autoBackfill && process.env.OMMS_DISABLE_AUTO_BACKFILL !== "1" && isConfigured()) { void import("./adapters/opencode/backfill-startup.js") .then(({ startOpencodeBackfill }) => startOpencodeBackfill({ connected, directory, - configModel: async () => { - const response = await ctx.client.config.get(); - return unwrapSdkData<{ model?: string }>(response)?.model ?? null; - }, + configModel, notify: (message) => { void ctx.client.tui ?.showToast({ diff --git a/src/services/ai/live-model-choice.ts b/src/services/ai/live-model-choice.ts index 0b26a0f4..3a2a94ab 100644 --- a/src/services/ai/live-model-choice.ts +++ b/src/services/ai/live-model-choice.ts @@ -17,7 +17,8 @@ export interface AutoCaptureProviderRuntimeConfig { /** * Where live capture and profile learning send model calls, the same on both hosts: * - `opencode`: the configured host model (`opencodeModel`/`piModel`; "inherit" = session model) - * - `manual`: no host model, so the external API (`memoryModel`/`memoryApiUrl`/`memoryApiKey`) + * - `manual`: the external API (`memoryModel`/`memoryApiUrl`/`memoryApiKey`), because the + * host model is "external" or no host model is set * - `session`: neither is set, so the session's own model through the host's sign-in */ export type AutoCaptureProviderStatus = @@ -60,6 +61,14 @@ function getManualProviderStatus(config: AutoCaptureProviderRuntimeConfig): { }; } +/** The host chose the external API: it is the primary call, so there is no fallback. */ +function externalChoiceStatus(config: AutoCaptureProviderRuntimeConfig): AutoCaptureProviderStatus { + const manual = getManualProviderStatus(config); + return manual.ready + ? { ready: true, mode: "manual", issues: [] } + : { ready: false, issues: manual.issues }; +} + export function getAutoCaptureProviderStatus( config: AutoCaptureProviderRuntimeConfig ): AutoCaptureProviderStatus { @@ -68,6 +77,7 @@ export function getAutoCaptureProviderStatus( if (config.opencodeModel?.trim() === "inherit") { return { ready: true, mode: "session", issues: [] }; } + if (config.opencodeModel?.trim() === "external") return externalChoiceStatus(config); if (hasOpencodeProvider && hasOpencodeModel) { return { ready: true, mode: "opencode", issues: [] }; } @@ -107,6 +117,10 @@ export function resolvePiLiveModel( config: AutoCaptureProviderRuntimeConfig & { piProvider?: string; piModel?: string } ): PiLiveModelChoice { if (config.piModel?.trim() === "inherit") return { kind: "session" }; + if (config.piModel?.trim() === "external") { + const status = externalChoiceStatus(config); + return status.ready ? { kind: "manual" } : { kind: "unready", issues: status.issues }; + } if (hasValue(config.piProvider) && hasValue(config.piModel)) { return { kind: "pi", provider: config.piProvider!.trim(), model: config.piModel!.trim() }; } @@ -120,3 +134,9 @@ export function resolvePiLiveModel( export function isExternalModelReady(config: AutoCaptureProviderRuntimeConfig): boolean { return getManualProviderStatus(config).ready; } + +/** Why the external API cannot be used as a host's chosen model, or an empty list. */ +export function externalModelIssues(config: AutoCaptureProviderRuntimeConfig): string[] { + const manual = getManualProviderStatus(config); + return manual.ready ? [] : manual.issues; +} diff --git a/src/services/backfill-state.ts b/src/services/backfill-state.ts index 740a96b7..9ba989b1 100644 --- a/src/services/backfill-state.ts +++ b/src/services/backfill-state.ts @@ -23,6 +23,10 @@ export type BackfillStatus = { updatedAt: number; }; +export type UnresolvedDirectory = { directory: string; sessions: number }; + +export const MAX_UNRESOLVED_DIRECTORIES = 200; + const emptyCounts = () => ({ imported: 0, skipped: 0, failed: 0, pending: 0, unresolved: 0 }); async function table() { @@ -73,6 +77,72 @@ export async function readBackfillStatus(host: BackfillHost): Promise +): UnresolvedDirectory[] { + const byDirectory = new Map(); + for (const item of sessions) { + if (!item.directory) continue; + byDirectory.set(item.directory, (byDirectory.get(item.directory) ?? 0) + (item.sessions ?? 1)); + } + return [...byDirectory] + .map(([directory, count]) => ({ directory, sessions: count })) + .sort((a, b) => b.sessions - a.sessions || a.directory.localeCompare(b.directory)) + .slice(0, MAX_UNRESOLVED_DIRECTORIES); +} + +/** Unresolved directories of an import report with session counts. */ +export function unresolvedDirectoriesOf(report: { + unresolvedProjects?: ReadonlyArray<{ directory: string; sessions: number }>; + unresolvableSessions: ReadonlyArray<{ cwd: string | null }>; +}): UnresolvedDirectory[] { + return summarizeUnresolvedDirectories([ + ...(report.unresolvedProjects ?? []), + ...report.unresolvableSessions.map((session) => ({ directory: session.cwd })), + ]); +} + +/** + * Save the unresolved directories of a host's latest listing or run. They + * live in their own table, so recording them never sets a backfill cutoff. + */ +export async function recordUnresolvedDirectories( + host: BackfillHost, + directories: UnresolvedDirectory[], + now = Date.now() +): Promise { + const db = await tursoConnectionManager.getConnection(importLedgerDbPath()); + await db.run(`CREATE TABLE IF NOT EXISTS unresolved_directories ( + host TEXT PRIMARY KEY, directories TEXT NOT NULL, updated_at INTEGER NOT NULL + )`); + await db.run( + `INSERT INTO unresolved_directories (host, directories, updated_at) VALUES (?, ?, ?) + ON CONFLICT(host) DO UPDATE SET directories = excluded.directories, updated_at = excluded.updated_at`, + [host, JSON.stringify(directories.slice(0, MAX_UNRESOLVED_DIRECTORIES)), now] + ); +} + +/** A host's latest unresolved directories, without creating a store or table. */ +export async function readUnresolvedDirectories( + host: BackfillHost +): Promise { + if (!existsSync(importLedgerDbPath())) return []; + const db = await tursoConnectionManager.getConnection(importLedgerDbPath()); + const exists = await db.get( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'unresolved_directories'" + ); + if (!exists) return []; + const row = await db.get<{ directories: string }>( + "SELECT directories FROM unresolved_directories WHERE host = ?", + [host] + ); + return row ? (JSON.parse(String(row.directories)) as UnresolvedDirectory[]) : []; +} + /** Save only numeric counts and a redacted error, never report units or prompts. */ export async function updateBackfillStatus( host: BackfillHost, diff --git a/src/services/capture-diagnostics.ts b/src/services/capture-diagnostics.ts index bc97a801..bc724d42 100644 --- a/src/services/capture-diagnostics.ts +++ b/src/services/capture-diagnostics.ts @@ -1,13 +1,4 @@ -import { execFileSync } from "node:child_process"; -import { - appendFileSync, - chmodSync, - existsSync, - lstatSync, - mkdirSync, - readdirSync, - unlinkSync, -} from "fs"; +import { appendFileSync, existsSync, lstatSync, mkdirSync, readdirSync, unlinkSync } from "fs"; import { join } from "path"; import type { CaptureAttemptDiagnostics, @@ -18,6 +9,7 @@ import type { import type { MemoryHost, MemorySourceType } from "../types/index.js"; import { getLogDirPath } from "./log-path.js"; import { log } from "./logger.js"; +import { restrictToCurrentUser } from "./private-path.js"; import { stripPrivateContent } from "./privacy.js"; /** @@ -185,66 +177,8 @@ export function pruneTraces(config: CaptureDiagnosticsConfig, now: Date = new Da let lastTraceDate: string | null = null; -const WINDOWS_TRACE_ACL = `$ErrorActionPreference = 'Stop' -$path = $env:OMMS_TRACE_ACL_PATH -$isDirectory = [System.IO.Directory]::Exists($path) -$acl = if ($isDirectory) { - [System.IO.Directory]::GetAccessControl($path) -} else { - [System.IO.File]::GetAccessControl($path) -} -$currentUser = [System.Security.Principal.WindowsIdentity]::GetCurrent().User -$owner = $acl.GetOwner([System.Security.Principal.SecurityIdentifier]) -if ($owner.Value -ne $currentUser.Value) { - throw 'Trace path has a different owner' -} -$acl.SetAccessRuleProtection($true, $false) -foreach ($rule in @($acl.Access)) { - [void]$acl.RemoveAccessRuleSpecific($rule) -} -$inheritance = if ($isDirectory) { - [System.Security.AccessControl.InheritanceFlags]'ContainerInherit, ObjectInherit' -} else { - [System.Security.AccessControl.InheritanceFlags]::None -} -$rule = [System.Security.AccessControl.FileSystemAccessRule]::new( - $currentUser, - [System.Security.AccessControl.FileSystemRights]::FullControl, - $inheritance, - [System.Security.AccessControl.PropagationFlags]::None, - [System.Security.AccessControl.AccessControlType]::Allow -) -$acl.AddAccessRule($rule) -if ($isDirectory) { - [System.IO.Directory]::SetAccessControl($path, $acl) -} else { - [System.IO.File]::SetAccessControl($path, $acl) -}`; - -type WindowsAclRunner = ( - command: string, - args: string[], - options: { env: NodeJS.ProcessEnv; timeout: number; windowsHide: boolean; stdio: "ignore" } -) => void; - /** Restrict a trace path to the current user on both Windows and POSIX. */ -export function protectTracePath( - path: string, - mode: number, - platform = process.platform, - run: WindowsAclRunner = execFileSync -): void { - if (platform !== "win32") { - chmodSync(path, mode); - return; - } - run("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", WINDOWS_TRACE_ACL], { - env: { ...process.env, OMMS_TRACE_ACL_PATH: path }, - timeout: 10_000, - windowsHide: true, - stdio: "ignore", - }); -} +export const protectTracePath = restrictToCurrentUser; /** * Append one trace entry to the day's JSON Lines file. A single append per diff --git a/src/services/global-config-writer.ts b/src/services/global-config-writer.ts index 136791f7..fb26ed4f 100644 --- a/src/services/global-config-writer.ts +++ b/src/services/global-config-writer.ts @@ -21,8 +21,41 @@ const keys = new Set([ "opencodeBackfillModel", "piBackfillModel", "webServerAutoStart", + "memoryProvider", + "memoryApiUrl", + "memoryModel", + "memoryApiKey", + "importPathMaps", ]); +const MEMORY_PROVIDERS = [ + "openai-chat", + "openai-responses", + "anthropic", + "minimax", + "orcarouter", + "google-gemini", +]; + +/** The page may point `memoryApiKey` at a source, never store the key itself. */ +export function isSecretReference(value: unknown): value is string { + return ( + typeof value === "string" && /^(env:\/\/[A-Za-z_][A-Za-z0-9_]*|file:\/\/.+)$/.test(value.trim()) + ); +} + +function isValidEdit(key: string, value: unknown): boolean { + if (key === "captureTrace" || key === "autoBackfill" || key === "webServerAutoStart") { + return typeof value === "boolean"; + } + if (key.endsWith("RetentionDays")) return Number.isSafeInteger(value) && (value as number) >= 1; + if (key === "memoryApiKey") return isSecretReference(value); + if (key === "memoryProvider") return MEMORY_PROVIDERS.includes(value as string); + // Entries are checked by the startup validation below. + if (key === "importPathMaps") return Array.isArray(value); + return typeof value === "string" && value.trim().length > 0; +} + export class ConfigConflictError extends Error { readonly status = 409; constructor() { @@ -61,14 +94,13 @@ export async function writeGlobalConfigKeys( if (!edits || !Object.keys(edits).length) throw new Error("No settings to save"); for (const [key, value] of Object.entries(edits)) { if (!keys.has(key)) throw new Error(`Setting ${key} cannot be edited here`); - if ( - key === "captureTrace" || key === "autoBackfill" || key === "webServerAutoStart" - ? typeof value !== "boolean" - : key.endsWith("RetentionDays") - ? !Number.isSafeInteger(value) || (value as number) < 1 - : typeof value !== "string" || !value.trim() - ) { - throw new Error(`Invalid ${key} setting`); + if (!isValidEdit(key, value)) { + // Never echo the value: a rejected memoryApiKey may be a literal key. + throw new Error( + key === "memoryApiKey" + ? "memoryApiKey must be an env:// or file:// reference" + : `Invalid ${key} setting` + ); } } const wasQueued = pending > 0; diff --git a/src/services/global-version.ts b/src/services/global-version.ts new file mode 100644 index 00000000..562dc7ec --- /dev/null +++ b/src/services/global-version.ts @@ -0,0 +1,114 @@ +import { execFile } from "node:child_process"; +import { accessSync, constants } from "node:fs"; +import { delimiter, isAbsolute, join } from "node:path"; + +export interface GlobalCommandVersion { + /** The version the global `om-memory-system --version` printed, or null when not installed. */ + version: string | null; + path: string | null; + error?: "timeout" | "failed"; +} + +type Runner = ( + file: string, + args: string[], + options: { timeout: number; windowsHide: boolean; shell: false } +) => Promise; + +const TIMEOUT_MS = 3_000; +const CACHE_MS = 10 * 60_000; + +const run: Runner = (file, args, options) => + new Promise((resolve, reject) => { + execFile(file, args, { ...options, encoding: "utf8" }, (error, stdout) => + error ? reject(error) : resolve(String(stdout)) + ); + }); + +function executable(path: string): boolean { + try { + accessSync(path, constants.X_OK); + return true; + } catch { + return false; + } +} + +/** The first `om-memory-system` on PATH; on Windows npm installs a `.cmd` wrapper. */ +export function findGlobalCommand( + env: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, + exists: (path: string) => boolean = executable +): string | null { + const names = + platform === "win32" ? ["om-memory-system.exe", "om-memory-system.cmd"] : ["om-memory-system"]; + const pathValue = env.PATH ?? env.Path ?? ""; + for (const raw of pathValue.split(platform === "win32" ? ";" : delimiter)) { + const entry = raw.trim().replace(/^"(.*)"$/, "$1"); + if (!entry || !(platform === "win32" ? /^[A-Za-z]:[\\/]|^\\\\/.test(entry) : isAbsolute(entry))) + continue; + for (const name of names) { + const candidate = + platform === "win32" ? `${entry.replace(/[\\/]+$/, "")}\\${name}` : join(entry, name); + if (exists(candidate)) return candidate; + } + } + return null; +} + +/** + * The command and arguments that print the version without a user-controlled + * shell. A Windows `.cmd` wrapper can only run through cmd.exe, so it runs + * through the system's own cmd.exe with fixed arguments. + */ +export function versionInvocation( + path: string, + platform: NodeJS.Platform = process.platform, + env: NodeJS.ProcessEnv = process.env +): { file: string; args: string[] } { + if (platform === "win32" && path.toLowerCase().endsWith(".cmd")) { + const cmd = env.SystemRoot ? `${env.SystemRoot}\\System32\\cmd.exe` : "cmd.exe"; + return { file: cmd, args: ["/d", "/s", "/c", `"${path}" --version`] }; + } + return { file: path, args: ["--version"] }; +} + +let cached: { at: number; value: GlobalCommandVersion } | null = null; + +/** Look up the global command's version, at most once every 10 minutes. */ +export async function globalCommandVersion( + options: { + now?: () => number; + runner?: Runner; + find?: () => string | null; + platform?: NodeJS.Platform; + } = {} +): Promise { + const now = (options.now ?? Date.now)(); + if (cached && now - cached.at < CACHE_MS) return cached.value; + const path = (options.find ?? (() => findGlobalCommand()))(); + let value: GlobalCommandVersion; + if (!path) { + value = { version: null, path: null }; + } else { + const { file, args } = versionInvocation(path, options.platform); + try { + const output = await (options.runner ?? run)(file, args, { + timeout: TIMEOUT_MS, + windowsHide: true, + shell: false, + }); + const version = /\d+\.\d+\.\d+(?:[-+][\w.]+)?/.exec(output)?.[0] ?? null; + value = version ? { version, path } : { version: null, path, error: "failed" }; + } catch (error) { + const killed = (error as { killed?: boolean; signal?: string }).killed; + value = { version: null, path, error: killed ? "timeout" : "failed" }; + } + } + cached = { at: now, value }; + return value; +} + +export function resetGlobalVersionCache(): void { + cached = null; +} diff --git a/src/services/memory-key-source.ts b/src/services/memory-key-source.ts new file mode 100644 index 00000000..93a94a46 --- /dev/null +++ b/src/services/memory-key-source.ts @@ -0,0 +1,122 @@ +import { randomUUID } from "node:crypto"; +import { promises as fs, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { isAbsolute, join } from "node:path"; +import { restrictToCurrentUser } from "./private-path.js"; + +/** How the External API card supplies `memoryApiKey`; a pasted value never leaves this module. */ +export type MemoryKeySourceRequest = + | { source: "env"; name: string } + | { source: "file"; path: string } + | { source: "paste"; name: string; value: string; replace?: boolean }; + +export class MemoryKeySourceError extends Error { + constructor( + message: string, + readonly status = 400 + ) { + super(message); + this.name = "MemoryKeySourceError"; + } +} + +export const secretsDirectory = () => join(homedir(), ".config", "omms", "secrets"); + +const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +const KEY_FILE_NAME = /^[A-Za-z0-9_-]{1,64}$/; + +function expandHome(path: string): string { + if (path === "~") return homedir(); + if (path.startsWith("~/") || path.startsWith("~\\")) return join(homedir(), path.slice(2)); + return path; +} + +/** Check a request's shape; error messages never include the pasted value. */ +export function parseMemoryKeySource(body: unknown): MemoryKeySourceRequest { + const request = (body ?? {}) as Record; + if (request.source === "env") { + if (typeof request.name !== "string" || !ENV_NAME.test(request.name)) { + throw new MemoryKeySourceError("Enter a variable name such as ZAI_API_KEY"); + } + return { source: "env", name: request.name }; + } + if (request.source === "file") { + if (typeof request.path !== "string" || !request.path.trim()) { + throw new MemoryKeySourceError("Enter the path of a key file"); + } + return { source: "file", path: request.path.trim() }; + } + if (request.source === "paste") { + if (typeof request.name !== "string" || !KEY_FILE_NAME.test(request.name)) { + throw new MemoryKeySourceError( + "Name the key file with letters, digits, dashes, or underscores" + ); + } + if (typeof request.value !== "string" || !request.value.trim()) { + throw new MemoryKeySourceError("Paste the key"); + } + return { + source: "paste", + name: request.name, + value: request.value, + replace: request.replace === true, + }; + } + throw new MemoryKeySourceError("Choose a key source"); +} + +/** + * Turn a key source into an `env://` or `file://` reference. A pasted key is + * written to `~/.config/omms/secrets/.key`, readable only by the user, + * and replaces an existing file only when the request confirms it. + */ +export async function storeMemoryKeySource( + request: MemoryKeySourceRequest, + options: { directory?: string; platform?: NodeJS.Platform } = {} +): Promise<{ reference: string; source: "env" | "file"; path?: string }> { + if (request.source === "env") return { reference: `env://${request.name}`, source: "env" }; + if (request.source === "file") { + const path = expandHome(request.path); + let isFile: boolean; + try { + isFile = isAbsolute(path) && statSync(path).isFile(); + } catch { + isFile = false; + } + if (!isFile) throw new MemoryKeySourceError("The key file does not exist"); + return { reference: `file://${path}`, source: "file", path }; + } + + const directory = options.directory ?? secretsDirectory(); + const platform = options.platform ?? process.platform; + await fs.mkdir(directory, { recursive: true, mode: 0o700 }); + restrictToCurrentUser(directory, 0o700, platform); + const path = join(directory, `${request.name}.key`); + const content = `${request.value.trim()}\n`; + if (!request.replace) { + try { + await fs.writeFile(path, content, { flag: "wx", mode: 0o600 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + throw new MemoryKeySourceError( + `A key file named ${request.name}.key exists. Confirm to replace it.`, + 409 + ); + } + throw new MemoryKeySourceError("The key file could not be written", 500); + } + } else { + const temp = join(directory, `.${randomUUID()}.tmp`); + try { + await fs.writeFile(temp, content, { flag: "wx", mode: 0o600 }); + restrictToCurrentUser(temp, 0o600, platform); + await fs.rename(temp, path); + } catch { + throw new MemoryKeySourceError("The key file could not be written", 500); + } finally { + await fs.rm(temp, { force: true }); + } + } + restrictToCurrentUser(path, 0o600, platform); + return { reference: `file://${path}`, source: "file", path }; +} diff --git a/src/services/package-version.ts b/src/services/package-version.ts new file mode 100644 index 00000000..409e6bde --- /dev/null +++ b/src/services/package-version.ts @@ -0,0 +1,22 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** This package's version, found by walking up to its `package.json` from source or `dist/`. */ +export function packageVersion(from = import.meta.url): string { + let dir = dirname(fileURLToPath(from)); + for (;;) { + try { + const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")) as { + name?: string; + version?: string; + }; + if (pkg.name === "om-memory-system" && pkg.version) return pkg.version; + } catch { + /* Try the parent. */ + } + const parent = dirname(dir); + if (parent === dir) return "unknown"; + dir = parent; + } +} diff --git a/src/services/private-path.ts b/src/services/private-path.ts new file mode 100644 index 00000000..71def691 --- /dev/null +++ b/src/services/private-path.ts @@ -0,0 +1,67 @@ +import { execFileSync } from "node:child_process"; +import { chmodSync } from "node:fs"; + +const WINDOWS_USER_ONLY_ACL = `$ErrorActionPreference = 'Stop' +$path = $env:OMMS_PRIVATE_PATH +$isDirectory = [System.IO.Directory]::Exists($path) +$acl = if ($isDirectory) { + [System.IO.Directory]::GetAccessControl($path) +} else { + [System.IO.File]::GetAccessControl($path) +} +$currentUser = [System.Security.Principal.WindowsIdentity]::GetCurrent().User +$owner = $acl.GetOwner([System.Security.Principal.SecurityIdentifier]) +if ($owner.Value -ne $currentUser.Value) { + throw 'Path has a different owner' +} +$acl.SetAccessRuleProtection($true, $false) +foreach ($rule in @($acl.Access)) { + [void]$acl.RemoveAccessRuleSpecific($rule) +} +$inheritance = if ($isDirectory) { + [System.Security.AccessControl.InheritanceFlags]'ContainerInherit, ObjectInherit' +} else { + [System.Security.AccessControl.InheritanceFlags]::None +} +$rule = [System.Security.AccessControl.FileSystemAccessRule]::new( + $currentUser, + [System.Security.AccessControl.FileSystemRights]::FullControl, + $inheritance, + [System.Security.AccessControl.PropagationFlags]::None, + [System.Security.AccessControl.AccessControlType]::Allow +) +$acl.AddAccessRule($rule) +if ($isDirectory) { + [System.IO.Directory]::SetAccessControl($path, $acl) +} else { + [System.IO.File]::SetAccessControl($path, $acl) +}`; + +type WindowsAclRunner = ( + command: string, + args: string[], + options: { env: NodeJS.ProcessEnv; timeout: number; windowsHide: boolean; stdio: "ignore" } +) => void; + +/** + * Restrict a path to the current user: `chmod` on macOS and Linux, and on + * Windows an access list with inheritance removed and one full-control rule + * for the owner, who must be the current user. + */ +export function restrictToCurrentUser( + path: string, + mode: number, + platform = process.platform, + run: WindowsAclRunner = execFileSync +): void { + if (platform !== "win32") { + chmodSync(path, mode); + return; + } + run("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", WINDOWS_USER_ONLY_ACL], { + env: { ...process.env, OMMS_PRIVATE_PATH: path }, + timeout: 10_000, + windowsHide: true, + stdio: "ignore", + }); +} diff --git a/src/services/settings-snapshot.ts b/src/services/settings-snapshot.ts index 438f360a..b7b5fcb1 100644 --- a/src/services/settings-snapshot.ts +++ b/src/services/settings-snapshot.ts @@ -21,8 +21,23 @@ const editable = [ "opencodeBackfillModel", "piBackfillModel", "webServerAutoStart", + "memoryProvider", + "memoryApiUrl", + "memoryModel", + "importPathMaps", ] as const; +/** Keys a project config cannot override, so the page always shows the global value. */ +const globalOnly = [ + "autoBackfill", + "opencodeBackfillModel", + "piBackfillModel", + "webServerAutoStart", + "memoryProvider", + "memoryApiUrl", + "importPathMaps", +]; + function readSettingsFile(path: string | undefined): Record { if (!path) return {}; const errors: ParseError[] = []; @@ -47,13 +62,7 @@ export function getSettingsSnapshot(directory: string) { Object.hasOwn(project, key) && (key === "captureTrace" ? project[key] === false - : !key.endsWith("RetentionDays") && - ![ - "autoBackfill", - "opencodeBackfillModel", - "piBackfillModel", - "webServerAutoStart", - ].includes(key)); + : !key.endsWith("RetentionDays") && !globalOnly.includes(key)); const source = projectOverrides ? "project" : Object.hasOwn(global, key) @@ -66,18 +75,21 @@ export function getSettingsSnapshot(directory: string) { const secrets = Object.fromEntries( ["memoryApiKey", "embeddingApiKey", "webServerApiToken", "webServerAuthPassword"].map((key) => { const raw = global[key]; + const source = + typeof raw === "string" + ? raw.startsWith("env://") + ? "env" + : raw.startsWith("file://") + ? "file" + : "literal" + : null; return [ key, { set: typeof raw === "string" && raw.length > 0, - source: - typeof raw === "string" - ? raw.startsWith("env://") - ? "env" - : raw.startsWith("file://") - ? "file" - : "literal" - : null, + source, + // The variable name or file path, never a literal value. + reference: memoryKeyStatus(raw).reference, }, ]; }) @@ -87,9 +99,39 @@ export function getSettingsSnapshot(directory: string) { settings, secrets, fallback: { model: CONFIG.memoryModel ?? null, configured: isExternalModelReady(CONFIG) }, + externalKey: memoryKeyStatus(global.memoryApiKey), effective: { opencode: getAutoCaptureProviderStatus(CONFIG), pi: resolvePiLiveModel(CONFIG), }, }; } + +/** + * Where `memoryApiKey` comes from and whether it resolves in this web + * server's own process. The key value itself is never returned. + */ +export function memoryKeyStatus(raw: unknown, resolved = CONFIG.memoryApiKey) { + const source = + typeof raw !== "string" || !raw + ? null + : raw.startsWith("env://") + ? "env" + : raw.startsWith("file://") + ? "file" + : "literal"; + return { + source, + reference: + source === "env" + ? (raw as string).slice("env://".length) + : source === "file" + ? (raw as string).slice("file://".length) + : null, + resolvesInWebApp: Boolean(resolved), + warning: + source === "env" + ? "A login web app does not see variables set only in a shell profile; a key file works everywhere." + : null, + }; +} diff --git a/src/services/web-server.ts b/src/services/web-server.ts index 979fb119..5daa3990 100644 --- a/src/services/web-server.ts +++ b/src/services/web-server.ts @@ -228,6 +228,7 @@ export class WebServer { private takeoverFailures: number = 0; private readonly maxFallbackPort: number; private settingsImportJobs?: import("../importer/web-import-jobs.js").SettingsImportJobs; + private backfillControlsInstance?: import("../importer/web-import-api.js").BackfillControls; /** Run an import-page action; errors carry their own status and never file contents. */ private async importResponse(action: () => unknown, status = 200): Promise { @@ -242,6 +243,11 @@ export class WebServer { } } + private async backfillControls() { + const { BackfillControls } = await import("../importer/web-import-api.js"); + return (this.backfillControlsInstance ??= new BackfillControls()); + } + private async importJobs() { const { SettingsImportJobs } = await import("../importer/web-import-jobs.js"); return (this.settingsImportJobs ??= new SettingsImportJobs()); @@ -555,6 +561,41 @@ export class WebServer { }); } + if (path === "/api/settings/backfill/runs" && method === "GET") { + return this.jsonResponse(await (await this.backfillControls()).status()); + } + + const backfillAction = /^\/api\/settings\/backfill\/(pi|opencode)\/(run|pause|resume)$/.exec( + path + ); + if (backfillAction && method === "POST") { + const host = backfillAction[1] as "pi" | "opencode"; + const controls = await this.backfillControls(); + const cwd = this.config.directory ?? process.cwd(); + return this.importResponse(() => + backfillAction[2] === "pause" + ? controls.pause(host) + : backfillAction[2] === "resume" + ? controls.resume(host, cwd) + : controls.runNow(host, cwd) + ); + } + + if (path === "/api/settings/version" && method === "GET") { + const [{ packageVersion }, { globalCommandVersion }] = await Promise.all([ + import("./package-version.js"), + import("./global-version.js"), + ]); + const running = packageVersion(); + const global = await globalCommandVersion(); + return this.jsonResponse({ + running, + global: global.version, + globalPath: global.path, + mismatch: global.version !== null && global.version !== running, + }); + } + if (path === "/api/settings/web-autostart" && method === "GET") { const { webAutostartStatus } = await import("./web-autostart.js"); return this.jsonResponse(webAutostartStatus()); @@ -601,6 +642,60 @@ export class WebServer { return this.jsonResponse(result); } + if (path === "/api/settings/external-api/key" && method === "POST") { + // The body can hold a pasted key: never log it, and never echo it back. + const body = (await req.json()) as Record | null; + const { MemoryKeySourceError, parseMemoryKeySource, storeMemoryKeySource } = + await import("./memory-key-source.js"); + try { + const request = parseMemoryKeySource(body); + if ( + request.source === "paste" && + !isLoopbackHost(this.config.host) && + !auth?.isEnabled() + ) { + return this.jsonResponse( + { error: "Saving a key requires Basic Auth on network hosts" }, + 403 + ); + } + if (typeof body?.revision !== "string") { + return this.jsonResponse({ error: "Revision required" }, 400); + } + const stored = await storeMemoryKeySource(request); + const { writeGlobalConfigKeys } = await import("./global-config-writer.js"); + const result = await writeGlobalConfigKeys( + { memoryApiKey: stored.reference }, + body.revision + ); + const { refreshConfigIfChanged } = await import("../config.js"); + refreshConfigIfChanged(this.config.directory ?? process.cwd()); + const { memoryKeyStatus } = await import("./settings-snapshot.js"); + return this.jsonResponse({ ...result, key: memoryKeyStatus(stored.reference) }); + } catch (error) { + const status = + error instanceof MemoryKeySourceError + ? error.status + : ((error as { status?: number }).status ?? 400); + const message = error instanceof Error ? error.message : "Key source not saved"; + const secret = typeof body?.value === "string" ? body.value.trim() : ""; + return this.jsonResponse( + { error: secret ? message.replaceAll(secret, "[redacted]") : message }, + status + ); + } + } + + if (path === "/api/settings/external-api/test" && method === "POST") { + const { testExternalApi } = await import("../importer/web-import-api.js"); + return this.jsonResponse(await testExternalApi()); + } + + if (path === "/api/settings/import-maps" && method === "GET") { + const { directoryMapsView } = await import("../importer/web-import-api.js"); + return this.jsonResponse(await directoryMapsView()); + } + if (path === "/api/settings/models" && method === "GET") { const { listOpencodeSettingsModels, listPiSettingsModels } = await import("./settings-models.js"); @@ -743,7 +838,8 @@ export class WebServer { const page = parseInt(url.searchParams.get("page") || "1"); const pageSize = parseInt(url.searchParams.get("pageSize") || "20"); const includePrompts = url.searchParams.get("includePrompts") !== "false"; - const result = await handleListMemories(tag, page, pageSize, includePrompts); + const keyword = url.searchParams.get("keyword") || undefined; + const result = await handleListMemories(tag, page, pageSize, includePrompts, keyword); return this.jsonResponse(result); } diff --git a/tests/auto-backfill.test.ts b/tests/auto-backfill.test.ts index f64b192a..a63f2240 100644 --- a/tests/auto-backfill.test.ts +++ b/tests/auto-backfill.test.ts @@ -370,3 +370,54 @@ it("stops after five failed summaries and retries on the next start", async () = expect(attempts).toBe(11); expect((await readBackfillStatus("pi"))?.state).toBe("done"); }); + +it("counts only units that need work as pending and passes the dry-run total", async () => { + directory = mkdtempSync(join(tmpdir(), "omms-auto-backfill-")); + CONFIG.storagePath = directory; + let track: { expectedTotal?: number } | undefined; + const pending: number[] = []; + let clock = 0; + await scheduleAutoBackfill({ + host: "pi", + cwd: directory, + wait: async () => {}, + enabled: () => true, + now: () => (clock += 10_000), + notify: () => {}, + resolveModels: async () => ({ model: "external/glm", models: {} }), + run: (async ( + _host: string, + args: { dryRun: boolean }, + context: { + track?: { expectedTotal?: number }; + onProgress?: (p: number, t: number, s: string, handled?: number) => void; + } + ) => { + if (args.dryRun) { + return { + unitsWouldImport: 7, + unitsImported: 0, + unitsSkipped: 0, + unitsFailed: 0, + unresolvableSessions: [], + unresolvedProjects: [], + }; + } + track = context.track; + // 729 of 736 units are already in the ledger; 2 of the 7 are done. + context.onProgress?.(732, 736, "private", 729); + pending.push((await readBackfillStatus("pi"))!.counts.pending); + return { + unitsWouldImport: 0, + unitsImported: 2, + unitsSkipped: 0, + unitsFailed: 0, + unresolvableSessions: [], + unresolvedProjects: [], + }; + }) as never, + }); + expect(track?.expectedTotal).toBe(7); + const status = await readBackfillStatus("pi"); + expect(status?.counts.pending).toBe(5); +}); diff --git a/tests/backfill-model.test.ts b/tests/backfill-model.test.ts index b454edef..3ac72da1 100644 --- a/tests/backfill-model.test.ts +++ b/tests/backfill-model.test.ts @@ -22,3 +22,27 @@ describe("backfill model selection", () => { expect(() => parseBackfillModel({ piBackfillModel: "provider/" }, "pi")).toThrow(); }); }); + +describe("the external backfill model", () => { + it("parses external as its own variant", () => { + expect(parseBackfillModel({ opencodeBackfillModel: "external" }, "opencode")).toBe("external"); + }); + + it("uses the external API only when it is configured", async () => { + const { resolveExternalBackfillModels } = + await import("../src/importer/external-backfill-models.js"); + const select = () => ({ + provider: "openai-chat", + modelId: "glm", + capture: { summarize: async () => null }, + profile: { provider: "openai-chat", modelId: "glm", complete: async () => "" }, + }); + const ready = { memoryModel: "glm", memoryApiUrl: "https://x.invalid", memoryApiKey: "k" }; + expect((await resolveExternalBackfillModels("pi", ready, select)).model).toBe( + "openai-chat/glm" + ); + await expect( + resolveExternalBackfillModels("opencode", { ...ready, memoryApiUrl: undefined }, select) + ).rejects.toThrow("OpenCode backfill: memoryApiUrl is not configured"); + }); +}); diff --git a/tests/backfill-state.test.ts b/tests/backfill-state.test.ts index 237bf0b7..32b2653a 100644 --- a/tests/backfill-state.test.ts +++ b/tests/backfill-state.test.ts @@ -7,6 +7,9 @@ import { importLedgerDbPath } from "../src/importer/ledger.js"; import { getBackfillCutoff, readBackfillStatus, + readUnresolvedDirectories, + recordUnresolvedDirectories, + summarizeUnresolvedDirectories, updateBackfillStatus, } from "../src/services/backfill-state.js"; import { tursoConnectionManager } from "../src/services/turso/connection-manager.js"; @@ -47,3 +50,31 @@ it("stores each host's cutoff once and persists only bounded status metadata", a expect(row?.counts).not.toContain("prompt"); expect(row?.error).not.toContain("secret-token"); }); + +it("stores unresolved directories with session counts, capped at 200, paths only", async () => { + directory = mkdtempSync(join(tmpdir(), "omms-backfill-state-")); + CONFIG.storagePath = directory; + const sessions = [ + { directory: "/gone/a" }, + { directory: "/gone/a" }, + { directory: null }, + { directory: "/gone/b", sessions: 3 }, + ...Array.from({ length: 250 }, (_, i) => ({ directory: `/tmp/t${i}` })), + ]; + const summary = summarizeUnresolvedDirectories(sessions); + expect(summary).toHaveLength(200); + expect(summary.slice(0, 2)).toEqual([ + { directory: "/gone/b", sessions: 3 }, + { directory: "/gone/a", sessions: 2 }, + ]); + expect(await readUnresolvedDirectories("pi")).toEqual([]); + await recordUnresolvedDirectories("pi", summary, 100); + expect(await readUnresolvedDirectories("pi")).toEqual(summary); + // Recording a listing must not start a backfill or fix its cutoff. + expect(await readBackfillStatus("pi")).toBeNull(); + const db = await tursoConnectionManager.getConnection(importLedgerDbPath()); + const row = await db.get<{ directories: string }>( + "SELECT directories FROM unresolved_directories WHERE host = 'pi'" + ); + expect(Object.keys(JSON.parse(row!.directories)[0]).sort()).toEqual(["directory", "sessions"]); +}); diff --git a/tests/capture-diagnostics.test.ts b/tests/capture-diagnostics.test.ts index bf37ba37..3c13ae8b 100644 --- a/tests/capture-diagnostics.test.ts +++ b/tests/capture-diagnostics.test.ts @@ -193,7 +193,7 @@ describe("trace files", () => { it("uses a protected Windows ACL instead of Unix mode bits", () => { const calls: Array<{ command: string; args: string[]; path: string | undefined }> = []; protectTracePath("C:\\trace[1].jsonl", 0o600, "win32", (command, args, options) => { - calls.push({ command, args, path: options.env.OMMS_TRACE_ACL_PATH }); + calls.push({ command, args, path: options.env.OMMS_PRIVATE_PATH }); }); expect(calls).toHaveLength(1); expect(calls[0]?.command).toBe("powershell.exe"); diff --git a/tests/external-api-test.test.ts b/tests/external-api-test.test.ts new file mode 100644 index 00000000..a1e98e6c --- /dev/null +++ b/tests/external-api-test.test.ts @@ -0,0 +1,35 @@ +import { expect, it } from "bun:test"; +import { CONFIG } from "../src/config.js"; +import { testExternalApi } from "../src/importer/external-api-test.js"; + +const config = { + ...CONFIG, + memoryProvider: "openai-chat" as const, + memoryModel: "glm-5-turbo", + memoryApiUrl: "https://api.example.invalid/v1", + memoryApiKey: "sk-rejected-private-key", +}; + +it("reports a rejected key with the key redacted and a small output limit", async () => { + const seen: Array> = []; + const result = await testExternalApi(config, (_provider, providerConfig) => { + seen.push(providerConfig); + return { + executeToolCall: async () => ({ + success: false, + error: "401 invalid api key sk-rejected-private-key (Bearer sk-rejected-private-key)", + }), + }; + }); + expect(result.ok).toBe(false); + expect(JSON.stringify(result)).not.toContain("sk-rejected-private-key"); + expect(JSON.stringify(result)).toContain("[redacted]"); + expect(seen[0]).toMatchObject({ maxTokens: 64, extraParams: { max_tokens: 64 } }); +}); + +it("reports success for an accepted call", async () => { + const result = await testExternalApi(config, () => ({ + executeToolCall: async () => ({ success: true }), + })); + expect(result).toEqual({ ok: true, model: "openai-chat/glm-5-turbo" }); +}); diff --git a/tests/global-config-writer.test.ts b/tests/global-config-writer.test.ts index 893c4ecb..e8e71820 100644 --- a/tests/global-config-writer.test.ts +++ b/tests/global-config-writer.test.ts @@ -98,6 +98,39 @@ describe("global config writer", () => { expect(result.text).toBe(seed); }); + it("saves external API settings and maps, and refuses a literal key without writing", async () => { + const seed = '{\n // keep me\n "piModel": "a"\n}\n'; + const result = await scenario( + ` + const saved = await writeGlobalConfigKeys({ + memoryProvider: "openai-chat", + memoryApiUrl: "https://api.example.invalid/v1", + memoryModel: "glm-5-turbo", + memoryApiKey: "env://ZAI_API_KEY", + importPathMaps: [{ from: "/old", to: "/new" }], + }, readGlobalConfigRevision()); + const afterSave = readFileSync(target, "utf8"); + let literal = ""; + try { + await writeGlobalConfigKeys({ memoryApiKey: "sk-literal-secret" }, saved.revision); + } catch (error) { literal = error.message; } + let provider = ""; + try { + await writeGlobalConfigKeys({ memoryProvider: "nope" }, saved.revision); + } catch (error) { provider = error.message; } + return { afterSave, unchanged: readFileSync(target, "utf8") === afterSave, literal, provider }; + `, + seed + ); + expect(result.afterSave).toContain("// keep me"); + expect(result.afterSave).toContain('"memoryApiKey": "env://ZAI_API_KEY"'); + expect(result.afterSave).toContain('"from": "/old"'); + expect(result.unchanged).toBe(true); + expect(result.literal).toBe("memoryApiKey must be an env:// or file:// reference"); + expect(result.literal).not.toContain("sk-literal-secret"); + expect(result.provider).toBe("Invalid memoryProvider setting"); + }); + it("creates a missing config from the template", async () => { const result = await scenario(` await writeGlobalConfigKeys({ piModel: "test" }, readGlobalConfigRevision()); diff --git a/tests/global-version.test.ts b/tests/global-version.test.ts new file mode 100644 index 00000000..30cda7cd --- /dev/null +++ b/tests/global-version.test.ts @@ -0,0 +1,93 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + findGlobalCommand, + globalCommandVersion, + resetGlobalVersionCache, + versionInvocation, +} from "../src/services/global-version.js"; +import { packageVersion } from "../src/services/package-version.js"; + +afterEach(() => resetGlobalVersionCache()); + +describe("om-memory-system --version", () => { + it("prints the package.json version and exits 0", () => { + const expected = ( + JSON.parse(readFileSync(join(import.meta.dir, "../package.json"), "utf8")) as { + version: string; + } + ).version; + expect(packageVersion()).toBe(expected); + for (const flag of ["--version", "-v"]) { + const result = spawnSync("bun", ["run", join(import.meta.dir, "../src/cli/index.ts"), flag], { + encoding: "utf8", + }); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe(expected); + } + }); +}); + +describe("global command version", () => { + it("finds the command on PATH, including a Windows .cmd wrapper", () => { + const seen: string[] = []; + const exists = (path: string) => { + seen.push(path); + return path.startsWith("C:\\Users") && path.endsWith("om-memory-system.cmd"); + }; + expect( + findGlobalCommand( + { PATH: "C:\\Tools;C:\\Users\\me\\AppData\\Roaming\\npm\\" }, + "win32", + exists + ) + ).toBe("C:\\Users\\me\\AppData\\Roaming\\npm\\om-memory-system.cmd"); + expect(seen[0]).toBe("C:\\Tools\\om-memory-system.exe"); + expect(findGlobalCommand({ PATH: "relative:/usr/bin" }, "linux", () => false)).toBeNull(); + expect( + versionInvocation("C:\\npm\\om-memory-system.cmd", "win32", { SystemRoot: "C:\\Windows" }) + ).toEqual({ + file: "C:\\Windows\\System32\\cmd.exe", + args: ["/d", "/s", "/c", '"C:\\npm\\om-memory-system.cmd" --version'], + }); + }); + + it("reports a found version and caches it for 10 minutes", async () => { + let calls = 0; + let now = 0; + const options = { + now: () => now, + find: () => "/usr/local/bin/om-memory-system", + runner: async (_file: string, args: string[], opts: { timeout: number; shell: false }) => { + calls++; + expect(args).toEqual(["--version"]); + expect(opts).toMatchObject({ timeout: 3_000, shell: false }); + return "3.3.1\n"; + }, + }; + expect(await globalCommandVersion(options)).toEqual({ + version: "3.3.1", + path: "/usr/local/bin/om-memory-system", + }); + now = 9 * 60_000; + await globalCommandVersion(options); + expect(calls).toBe(1); + now = 11 * 60_000; + await globalCommandVersion(options); + expect(calls).toBe(2); + }); + + it("reports a missing command and a timeout", async () => { + expect(await globalCommandVersion({ find: () => null })).toEqual({ version: null, path: null }); + resetGlobalVersionCache(); + const timedOut = await globalCommandVersion({ + find: () => "/bin/om-memory-system", + runner: async () => { + throw Object.assign(new Error("timeout"), { killed: true, signal: "SIGTERM" }); + }, + }); + expect(timedOut).toEqual({ version: null, path: "/bin/om-memory-system", error: "timeout" }); + }); +}); diff --git a/tests/import-path-maps.test.ts b/tests/import-path-maps.test.ts new file mode 100644 index 00000000..b4420d12 --- /dev/null +++ b/tests/import-path-maps.test.ts @@ -0,0 +1,53 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { mergeImportPathMaps, runPathMaps } from "../src/importer/import-path-maps.js"; +import { resolveImportProject } from "../src/importer/import-project.js"; + +const dirs: string[] = []; +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe("saved directory maps", () => { + it("lets a run map win over a saved map with the same source", () => { + const merged = mergeImportPathMaps( + [ + { from: "/old", to: "/a" }, + { from: "/keep", to: "/k" }, + ], + [{ from: "/old", to: "/b" }] + ); + expect(merged).toEqual([ + { from: "/old", to: "/b" }, + { from: "/keep", to: "/k" }, + ]); + }); + + it("resolves run targets against the working directory and keeps saved maps", async () => { + const saved = [{ from: "/saved", to: "/s" }]; + const maps = await runPathMaps([{ from: "/old", to: "rel" }], "/work", saved); + expect(maps).toEqual([ + { from: "/saved", to: "/s" }, + { from: "/old", to: resolve("/work", "rel") }, + ]); + expect(saved).toEqual([{ from: "/saved", to: "/s" }]); + }); + + it("imports into an existing target and leaves a missing target unresolved", async () => { + const root = mkdtempSync(join(tmpdir(), "omms-maps-")); + dirs.push(root); + const app = join(root, "app"); + mkdirSync(app); + const maps = await runPathMaps([], root, [ + { from: join(root, "app-feat-x"), to: app }, + { from: join(root, "gone"), to: join(root, "missing") }, + ]); + expect(resolveImportProject(join(root, "app-feat-x"), maps)).toEqual({ + directory: app, + via: "mapped", + }); + expect(resolveImportProject(join(root, "gone"), maps).via).toBe("unresolved"); + }); +}); diff --git a/tests/import-progress.test.ts b/tests/import-progress.test.ts new file mode 100644 index 00000000..53841746 --- /dev/null +++ b/tests/import-progress.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "bun:test"; +import { + addProgressSample, + estimateProgress, + MAX_PROGRESS_SAMPLES, + workProgress, + type ProgressSample, +} from "../src/importer/import-progress.js"; + +describe("import progress estimate", () => { + it("shows 40% and about 120 minutes left at 400 of 1,000 and 5 a minute", () => { + const samples: ProgressSample[] = [0, 1, 2, 3, 4].map((minute) => ({ + at: minute * 60_000, + done: 380 + minute * 5, + })); + expect(estimateProgress(1000, 400, samples)).toEqual({ + percent: 40, + minutesLeft: 120, + ratePerMinute: 5, + }); + }); + + it("is unknown until 5 samples over 60 seconds", () => { + const four = [0, 1, 2, 3].map((i) => ({ at: i * 60_000, done: i * 10 })); + expect(estimateProgress(100, 30, four).minutesLeft).toBeNull(); + const quick = [0, 1, 2, 3, 4].map((i) => ({ at: i * 1_000, done: i * 10 })); + expect(estimateProgress(100, 40, quick)).toEqual({ + percent: 40, + minutesLeft: null, + ratePerMinute: null, + }); + }); + + it("uses only the most recent samples", () => { + let samples: ProgressSample[] = []; + for (let i = 0; i < 30; i++) samples = addProgressSample(samples, { at: i, done: i }); + expect(samples).toHaveLength(MAX_PROGRESS_SAMPLES); + expect(samples[0]).toEqual({ at: 10, done: 10 }); + }); +}); + +describe("finished work units", () => { + it("leaves ledger hits out and counts a unit only once it has finished", () => { + // P1 starts; nothing finished yet. + expect(workProgress(1, 10, 0)).toEqual({ done: 0, total: 10 }); + // A ledger hit starts after P1 finished. + expect(workProgress(2, 10, 0)).toEqual({ done: 1, total: 10 }); + // P2 starts; the hit is now counted as already handled. + expect(workProgress(3, 10, 1)).toEqual({ done: 1, total: 9 }); + // 729 of 736 in the ledger, the backfill's dry run said 7. + expect(workProgress(736, 736, 728, 7)).toEqual({ done: 7, total: 7 }); + expect(workProgress(740, 736, 728, 7).done).toBe(7); + }); +}); diff --git a/tests/import-runs.test.ts b/tests/import-runs.test.ts new file mode 100644 index 00000000..b34afc1d --- /dev/null +++ b/tests/import-runs.test.ts @@ -0,0 +1,205 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { CONFIG } from "../src/config.js"; +import { scheduleAutoBackfill } from "../src/importer/auto-backfill.js"; +import { tryAcquireBackfillLock } from "../src/importer/backfill-lock.js"; +import { readImportRun, setBackfillPaused, startImportRun } from "../src/importer/import-runs.js"; +import { runHistoryImport } from "../src/importer/run-import.js"; +import { tursoConnectionManager } from "../src/services/turso/connection-manager.js"; +import { cleanupTursoTestDirectory } from "./turso-test-utils.js"; + +const previousStorage = CONFIG.storagePath; +const directories: string[] = []; +afterEach(async () => { + for (const directory of directories.splice(0)) await cleanupTursoTestDirectory(directory); + CONFIG.storagePath = previousStorage; +}); + +function store() { + const directory = mkdtempSync(join(tmpdir(), "omms-import-runs-")); + directories.push(directory); + CONFIG.storagePath = directory; + return directory; +} + +const importArgs = { + help: false, + dryRun: false, + force: false, + skipMemories: false, + skipProfile: true, + scope: "all-projects" as const, + pathMaps: [], + errors: [], +}; + +describe("import run records", () => { + it("stores only numbers, throttles writes, and never stores the prompt preview", async () => { + const directory = store(); + let now = 1_000; + const recorder = await startImportRun("pi", "cli", { now: () => now, intervalMs: 1_000 }); + const db = await tursoConnectionManager.getConnection(join(directory, "import-ledger.db")); + const writes = async () => + (await db.get<{ done: number }>("SELECT done FROM import_runs WHERE host = 'pi'"))!.done; + + recorder.progress(1, 10); + now += 100; + recorder.progress(2, 10); + now += 100; + recorder.progress(3, 10); + await recorder.finish("done", { total: 10, done: 3, imported: 3 }); + const row = await db.get>( + "SELECT * FROM import_runs WHERE host = 'pi'" + ); + expect(JSON.parse(String(row!.samples))).toEqual([{ at: 1_000, done: 1 }]); + expect(await writes()).toBe(3); + expect(JSON.stringify(row)).not.toContain("private turn"); + expect(await readImportRun("pi")).toMatchObject({ + surface: "cli", + state: "done", + total: 10, + done: 3, + imported: 3, + percent: 30, + }); + }); + + it("shows a running row whose process is gone as stopped", async () => { + const directory = store(); + await startImportRun("opencode", "cli"); + const db = await tursoConnectionManager.getConnection(join(directory, "import-ledger.db")); + await db.run("UPDATE import_runs SET pid = 2147483647 WHERE host = 'opencode'"); + expect((await readImportRun("opencode"))?.state).toBe("stopped"); + }); + + it("refuses a second model-calling run for the same host held by another process", async () => { + const home = store(); + const script = join(home, "holder.mjs"); + const url = (path: string) => pathToFileURL(join(import.meta.dir, path)).href; + writeFileSync( + script, + `const { CONFIG } = await import(${JSON.stringify(url("../src/config.ts"))}); + CONFIG.storagePath = ${JSON.stringify(home)}; + const { tryAcquireBackfillLock } = await import(${JSON.stringify(url("../src/importer/backfill-lock.ts"))}); + const release = await tryAcquireBackfillLock("pi", ${JSON.stringify(home)}); + console.log(release ? "held" : "refused"); + await new Promise((resolve) => process.stdin.once("data", resolve)); + await release?.();` + ); + const child = Bun.spawn(["bun", "run", script], { + cwd: home, + stdin: "pipe", + stdout: "pipe", + env: { ...process.env, HOME: home, USERPROFILE: home }, + }); + try { + const reader = child.stdout.getReader(); + let output = ""; + while (!/held|refused/.test(output)) { + const chunk = await reader.read(); + if (chunk.done) break; + output += new TextDecoder().decode(chunk.value); + } + expect(output).toContain("held"); + await expect( + runHistoryImport("pi", importArgs, { cwd: home, models: {}, track: { surface: "cli" } }) + ).rejects.toThrow("A Pi import is already running"); + child.stdin.write("\n"); + child.stdin.end(); + expect(await child.exited).toBe(0); + } finally { + child.kill(); + } + const release = await tryAcquireBackfillLock("pi", home); + expect(release).not.toBeNull(); + await release!(); + }); +}); + +describe("progress counts real work", () => { + it("leaves already-imported units out of done and total, and keeps the total when paused", async () => { + store(); + let now = 0; + const recorder = await startImportRun("pi", "cli", { now: () => now, intervalMs: 0 }); + // What runHistoryImport feeds the recorder for 731 processed, 729 already done. + recorder.progress(731 - 729, 736 - 729); + now = 1; + await recorder.finish("paused", { done: 2, imported: 2 }); + expect(await readImportRun("pi")).toMatchObject({ total: 7, done: 2, state: "paused" }); + }); + + it("samples at most every 15 seconds so the rate window spans minutes", async () => { + const directory = store(); + let now = 0; + const recorder = await startImportRun("pi", "cli", { now: () => now, intervalMs: 1_000 }); + for (let second = 0; second <= 60; second++) { + now = second * 1_000; + recorder.progress(second, 1_000); + } + await recorder.finish("done", {}); + const db = await tursoConnectionManager.getConnection(join(directory, "import-ledger.db")); + const row = await db.get<{ samples: string }>( + "SELECT samples FROM import_runs WHERE host = 'pi'" + ); + expect(JSON.parse(row!.samples).map((s: { at: number }) => s.at)).toEqual([ + 0, 15_000, 30_000, 45_000, 60_000, + ]); + }); +}); + +describe("pausing a backfill", () => { + it("aborts an in-process run once a progress write sees the flag", async () => { + store(); + let now = 0; + let paused = false; + const recorder = await startImportRun("pi", "web", { + now: () => now, + intervalMs: 1, + onPause: () => (paused = true), + }); + await setBackfillPaused("pi", true); + now = 10; + recorder.progress(1, 5); + await recorder.finish("paused", {}); + expect(paused).toBe(true); + expect(recorder.pauseRequested).toBe(true); + }); + + for (const host of ["pi", "opencode"] as const) { + it(`keeps ${host}'s backfill paused across a restart until resume`, async () => { + const directory = store(); + await setBackfillPaused(host, true); + // A fresh connection stands in for the next host start. + await tursoConnectionManager.closeAll(); + const calls: boolean[] = []; + const options = { + host, + cwd: directory, + wait: async () => {}, + enabled: () => true, + notify: () => {}, + resolveModels: async () => ({ model: "external/m", models: {} }), + run: async (_host: string, args: { dryRun: boolean }) => { + calls.push(args.dryRun); + return { + unitsWouldImport: 0, + unitsImported: 0, + unitsSkipped: 0, + unitsFailed: 0, + unresolvableSessions: [], + unresolvedProjects: [], + }; + }, + }; + await scheduleAutoBackfill(options as never); + expect(calls).toEqual([]); + expect((await readImportRun(host))?.paused).toBe(true); + await setBackfillPaused(host, false); + await scheduleAutoBackfill(options as never); + expect(calls).toEqual([true]); + }); + } +}); diff --git a/tests/live-model-choice.test.ts b/tests/live-model-choice.test.ts index 8f880b35..90b2ac64 100644 --- a/tests/live-model-choice.test.ts +++ b/tests/live-model-choice.test.ts @@ -99,6 +99,38 @@ describe("one live-model rule for both hosts", () => { }); }); +describe("the external host model value", () => { + it("sends both hosts to the external API when it is ready", () => { + const configured = { ...none, ...external, opencodeProvider: "openai", piProvider: "zai" }; + const opencode = getAutoCaptureProviderStatus({ ...configured, opencodeModel: "external" }); + const pi = resolvePiLiveModel({ ...configured, piModel: "external" }); + expect(opencode).toEqual({ ready: true, mode: "manual", issues: [] }); + expect(resolveOpencodeHostModel({ ...configured, opencodeModel: "external" })).toBeNull(); + expect(pi).toEqual({ kind: "manual" }); + }); + + it("disables both hosts with the same missing settings when the API is half set", () => { + const half = { ...none, memoryModel: "cheap", memoryApiKey: "real-key" }; + const opencode = getAutoCaptureProviderStatus({ ...half, opencodeModel: "external" }); + const pi = resolvePiLiveModel({ ...half, piModel: "external" }); + expect(opencode).toEqual({ ready: false, issues: ["memoryApiUrl is not configured"] }); + expect(pi).toEqual({ kind: "unready", issues: ["memoryApiUrl is not configured"] }); + expect(resolveOpencodeHostModel({ ...half, opencodeModel: "external" })).toBeNull(); + }); + + it("leaves the other host on its own rule", () => { + const both = { + ...none, + ...external, + opencodeProvider: "openai", + opencodeModel: "gpt-5.6-luna", + piModel: "external", + }; + expect(getAutoCaptureProviderStatus(both)).toMatchObject({ mode: "opencode" }); + expect(resolvePiLiveModel(both)).toEqual({ kind: "manual" }); + }); +}); + describe("Pi live models", () => { const saved = { piProvider: CONFIG.piProvider, @@ -175,6 +207,25 @@ describe("Pi live models", () => { ); }); + it("calls only the external API when piModel is external", async () => { + Object.assign(CONFIG, { piProvider: "openai-codex", piModel: "external", ...external }); + const live = ctx(); + const models = createPiLiveModels(live.ctx); + expect((await models.capture.summarize(request))?.summary).toBe("from external"); + await models.profile()!.complete("system", "prompt"); + expect(live.called).toEqual([]); + expect(externalCalls).toEqual(["capture", "profile"]); + }); + + it("does not fall back when piModel is external and the API is not configured", async () => { + Object.assign(CONFIG, { piModel: "external", memoryModel: undefined, memoryApiKey: "" }); + const live = ctx(); + await expect(createPiLiveModels(live.ctx).capture.summarize(request)).rejects.toThrow( + "memoryModel is not configured" + ); + expect(live.called).toEqual([]); + }); + it("surfaces the Pi failure when no external API is configured", async () => { Object.assign(CONFIG, { piProvider: "openai-codex", piModel: "missing" }); Object.assign(CONFIG, { memoryModel: undefined, memoryApiUrl: undefined, memoryApiKey: "" }); diff --git a/tests/map-suggestions.test.ts b/tests/map-suggestions.test.ts new file mode 100644 index 00000000..927156c2 --- /dev/null +++ b/tests/map-suggestions.test.ts @@ -0,0 +1,78 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + readOpencodeWorktrees, + suggestMapTarget, + suggestMapTargets, +} from "../src/importer/map-suggestions.js"; + +const dirs: string[] = []; +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function root() { + const dir = mkdtempSync(join(tmpdir(), "omms-suggest-")); + dirs.push(dir); + return dir; +} + +function repo(path: string) { + mkdirSync(join(path, ".git"), { recursive: true }); + return path; +} + +describe("directory map suggestions", () => { + it("suggests the main repository of a deleted sibling worktree", () => { + const code = join(root(), "code"); + const app = repo(join(code, "app")); + repo(join(code, "ap")); + mkdirSync(join(code, "apple")); + expect(suggestMapTarget(join(code, "app-feat-x"))).toBe(app); + }); + + it("suggests a known project for a nested workspace path", () => { + const base = root(); + const app = repo(join(base, "code", "app")); + const other = repo(join(base, "code", "other")); + mkdirSync(join(base, "orca", "workspaces"), { recursive: true }); + const missing = join(base, "orca", "workspaces", "app", "feat-x"); + expect(suggestMapTarget(missing, { knownProjects: [other] })).toBe(app); + }); + + it("gives no suggestion for a temporary directory with no candidate", () => { + const base = root(); + expect(suggestMapTargets([{ directory: join(base, "tmp-xyz"), sessions: 2 }])).toEqual([ + { directory: join(base, "tmp-xyz"), sessions: 2, suggestion: null }, + ]); + }); + + it("falls back to OpenCode's recorded worktree, read without writing", async () => { + const base = root(); + const worktree = mkdirSync(join(base, "project"), { recursive: true })!; + const dbPath = join(base, "opencode.db"); + const db = new DatabaseSync(dbPath); + db.exec(`CREATE TABLE project (id TEXT, worktree TEXT); + CREATE TABLE session (id TEXT, project_id TEXT, parent_id TEXT, directory TEXT, time_created INTEGER); + CREATE TABLE message (id TEXT, session_id TEXT, time_created INTEGER, data TEXT); + CREATE TABLE part (id TEXT, message_id TEXT, session_id TEXT, time_created INTEGER, data TEXT);`); + db.prepare("INSERT INTO project VALUES (?, ?)").run("p", worktree); + const missing = join(base, "zz-deleted", "sub"); + db.prepare("INSERT INTO session VALUES (?, ?, ?, ?, ?)").run("s", "p", null, missing, 1); + db.close(); + const hash = () => createHash("sha256").update(readFileSync(dbPath)).digest("hex"); + const before = hash(); + const entries = readdirSync(base).sort(); + + const worktrees = await readOpencodeWorktrees(dbPath); + expect(suggestMapTarget(missing, { opencodeWorktree: (dir) => worktrees.get(dir) })).toBe( + worktree + ); + expect(hash()).toBe(before); + expect(readdirSync(base).sort()).toEqual(entries); + }); +}); diff --git a/tests/memory-timeline-orphan.test.ts b/tests/memory-timeline-orphan.test.ts index 5422bbe9..e2547c8c 100644 --- a/tests/memory-timeline-orphan.test.ts +++ b/tests/memory-timeline-orphan.test.ts @@ -106,4 +106,63 @@ describe("memory timeline listing", () => { expect(ids).toContain(promptId); expect(ids.indexOf("mem_linked")).toBeLessThan(ids.indexOf(promptId)); }); + + it("filters the timeline by memory keyword and keeps only linked prompts", async () => { + baseDir = mkdtempSync(join(tmpdir(), "timeline-keyword-")); + + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + + const { ensureTursoReady } = await import("../src/services/turso/ready.js"); + await ensureTursoReady(); + + const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const { tursoShardManager } = await import("../src/services/turso/shard-manager.js"); + const { tursoVectorSearch } = await import("../src/services/turso/vector-search.js"); + const { handleListMemories } = await import("../src/services/api-handlers.js"); + const { userPromptManager } = + await import("../src/services/user-prompt/user-prompt-manager.js"); + + const vector = new Float32Array(CONFIG.embeddingDimensions); + vector[0] = 1; + const scopeHash = "a1b2c3d4e5f67890"; + const containerTag = `opencode_project_${scopeHash}`; + const now = Date.now(); + const shard = await tursoShardManager.createShard("project", scopeHash, 0); + const db = await tursoConnectionManager.getConnection(shard.dbPath); + + const promptId = await userPromptManager.savePrompt("s", "m1", "C:/proj", "linked prompt"); + await userPromptManager.markAsCaptured(promptId); + const otherPrompt = await userPromptManager.savePrompt("s", "m2", "C:/proj", "other prompt"); + await userPromptManager.markAsCaptured(otherPrompt); + + await tursoVectorSearch.insertVector(db, { + id: "mem_embed", + content: "Embedding memory", + vector, + containerTag, + tags: "Embeddings, validation", + metadata: JSON.stringify({ promptId }), + createdAt: now, + updatedAt: now, + }); + await userPromptManager.linkMemoryToPrompt(promptId, "mem_embed"); + await tursoVectorSearch.insertVector(db, { + id: "mem_other", + content: "Unrelated memory", + vector, + containerTag, + tags: "ingestion", + createdAt: now, + updatedAt: now, + }); + + const result = await handleListMemories(undefined, 1, 20, true, "embeddings"); + expect(result.success).toBe(true); + const ids = (result.data as { items: Array<{ id: string }> }).items.map((i) => i.id); + expect(ids).toContain("mem_embed"); + expect(ids).toContain(promptId); + expect(ids).not.toContain("mem_other"); + expect(ids).not.toContain(otherPrompt); + }); }); diff --git a/tests/omms-config.test.ts b/tests/omms-config.test.ts index 00424f4e..dc064689 100644 --- a/tests/omms-config.test.ts +++ b/tests/omms-config.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it } from "bun:test"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; const tempDirs: string[] = []; @@ -387,6 +387,57 @@ describe("global automatic backfill settings", () => { }); }); +describe("external model values and saved directory maps", () => { + it("accepts external and expands saved maps, ignoring a project's maps", async () => { + const home = mkdtempSync(join(tmpdir(), "omms-maps-home-")); + const project = mkdtempSync(join(tmpdir(), "omms-maps-project-")); + tempDirs.push(home, project); + mkdirSync(join(home, ".config", "omms"), { recursive: true }); + mkdirSync(join(project, ".opencode"), { recursive: true }); + writeFileSync( + join(home, ".config", "omms", "omms.jsonc"), + JSON.stringify({ + piModel: "external", + opencodeModel: "external", + piBackfillModel: "external", + opencodeBackfillModel: "external", + importPathMaps: [{ from: "~/code/app-feat-x", to: "~/code/app" }], + }) + ); + writeFileSync( + join(project, ".opencode", "omms.jsonc"), + JSON.stringify({ importPathMaps: [{ from: "/x", to: "/y" }] }) + ); + const result = await runConfigScenario( + home, + ` + initConfig(${JSON.stringify(project)}); + return { maps: cfg.CONFIG.importPathMaps, pi: cfg.CONFIG.piBackfillModel }; + ` + ); + expect(result).toMatchObject({ + ok: true, + pi: "external", + maps: [{ from: join(home, "code", "app-feat-x"), to: join(home, "code", "app") }], + }); + }); + + it("rejects invalid map entries", async () => { + const { parseImportPathMaps } = await import("../src/importer/import-path-maps.js"); + for (const importPathMaps of [ + "not-a-list", + [{ from: "/a" }], + [{ from: "relative", to: "/b" }], + [null], + ]) { + expect(() => parseImportPathMaps(importPathMaps)).toThrow("importPathMaps"); + } + expect(parseImportPathMaps([{ from: "/a/", to: "/b" }])).toEqual([ + { from: resolve("/a"), to: resolve("/b") }, + ]); + }); +}); + describe("omms project config file", () => { async function projectMaxMemories(files: Record): Promise { const home = mkdtempSync(join(tmpdir(), "omms-project-config-home-")); diff --git a/tests/opencode-backfill-model.test.ts b/tests/opencode-backfill-model.test.ts index 9003caaa..8524bc24 100644 --- a/tests/opencode-backfill-model.test.ts +++ b/tests/opencode-backfill-model.test.ts @@ -59,3 +59,30 @@ it("inherits the external live model when it is configured", async () => { ); expect(result.model).toBe("openai-chat/small"); }); + +it("sends an external OpenCode backfill to the external API, or names what is missing", async () => { + const context = { + connected: [], + directory: "/project", + configModel: async () => null, + makeModels, + externalModels: () => ({ + provider: "openai-chat", + modelId: "small", + capture: { summarize: async () => null }, + profile: { provider: "openai-chat", modelId: "small", complete: async () => "" }, + }), + }; + const ready = { + opencodeBackfillModel: "external", + opencodeProvider: "anthropic", + opencodeModel: "claude", + memoryModel: "small", + memoryApiUrl: "https://example.invalid", + memoryApiKey: "test-key", + }; + expect((await resolveOpencodeBackfillModels(ready, context)).model).toBe("openai-chat/small"); + await expect( + resolveOpencodeBackfillModels({ ...ready, memoryModel: undefined }, context) + ).rejects.toThrow("memoryModel is not configured"); +}); diff --git a/tests/opencode-capture-diagnostics.test.ts b/tests/opencode-capture-diagnostics.test.ts index 39645b32..ab61956d 100644 --- a/tests/opencode-capture-diagnostics.test.ts +++ b/tests/opencode-capture-diagnostics.test.ts @@ -109,6 +109,19 @@ describe("OpenCode capture diagnostics", () => { expect(diagnostics.failureReason).toBe("schema-mismatch"); }); + it("calls only the external API when opencodeModel is external", async () => { + externalConfig(); + Object.assign(config, { opencodeProvider: "anthropic", opencodeModel: "external" }); + let hostCalls = 0; + structuredBehaviour = async () => { + hostCalls++; + return {}; + }; + const { diagnostics } = await run(); + expect(hostCalls).toBe(0); + expect(diagnostics).toMatchObject({ path: "external-api", model: "glm-5.3" }); + }); + it("reports a failed external API call as call-error", async () => { externalConfig(); toolCallResult = { success: false, error: "API error: 500", stopReason: "stop" }; diff --git a/tests/pi-backfill-model.test.ts b/tests/pi-backfill-model.test.ts index 55c0bf2c..19347d46 100644 --- a/tests/pi-backfill-model.test.ts +++ b/tests/pi-backfill-model.test.ts @@ -39,3 +39,14 @@ it("inherits the live Pi model rule", async () => { CONFIG.piModel = previous.piModel; } }); + +it("stops an external Pi backfill with the missing setting named", async () => { + const ctx = { model: undefined, modelRegistry: { find: () => undefined } } as never; + await expect( + resolvePiBackfillModels(ctx, { + piBackfillModel: "external", + memoryApiUrl: "https://example.invalid/v1", + memoryApiKey: "key", + }) + ).rejects.toThrow("memoryModel is not configured"); +}); diff --git a/tests/web-external-settings.test.ts b/tests/web-external-settings.test.ts new file mode 100644 index 00000000..a49b8cd4 --- /dev/null +++ b/tests/web-external-settings.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from "bun:test"; +import { + externalMissing, + hostModelEdit, + keySourceBody, + mapsToSave, + versionNotice, +} from "../web/src/lib/external-api-settings.js"; +import { + backfillActions, + backfillModelEdit, + progressView, +} from "../web/src/lib/auto-import-settings.js"; + +describe("External API card", () => { + it("builds each key source request and validates it before sending", () => { + expect(keySourceBody("env", { name: "ZAI_API_KEY" })).toEqual({ + source: "env", + name: "ZAI_API_KEY", + }); + expect(keySourceBody("file", { path: " ~/k.key " })).toEqual({ + source: "file", + path: "~/k.key", + }); + expect(keySourceBody("paste", { name: "zai", value: "sk-x", replace: true })).toEqual({ + source: "paste", + name: "zai", + value: "sk-x", + replace: true, + }); + expect(() => keySourceBody("env", { name: "1BAD" })).toThrow("variable name"); + expect(() => keySourceBody("paste", { name: "zai", value: " " })).toThrow("Paste the key"); + expect(() => keySourceBody("paste", { name: "../x", value: "k" })).toThrow("key file"); + }); + + it("names the missing settings that keep External API disabled", () => { + expect(externalMissing({ keySet: false })).toEqual([ + "memoryModel", + "memoryApiUrl", + "memoryApiKey", + ]); + expect(externalMissing({ memoryProvider: "orcarouter", keySet: true })).toEqual([]); + expect(externalMissing({ memoryModel: "m", memoryApiUrl: "https://x", keySet: true })).toEqual( + [] + ); + }); + + it("saves external for a host's capture and backfill model", () => { + expect(hostModelEdit("pi", "external")).toEqual({ piModel: "external" }); + expect(hostModelEdit("opencode", "zai/glm")).toEqual({ + opencodeProvider: "zai", + opencodeModel: "glm", + }); + expect(backfillModelEdit("opencode", "external")).toEqual({ + opencodeBackfillModel: "external", + }); + }); +}); + +describe("Web app version notice", () => { + it("covers equal, different, and missing global versions", () => { + expect(versionNotice({ running: "3.4.0", global: "3.4.0" })).toEqual({ + kind: "same", + command: null, + }); + expect(versionNotice({ running: "3.4.0", global: "3.3.1" })).toEqual({ + kind: "different", + command: "npm i -g om-memory-system@latest", + }); + expect(versionNotice({ running: "3.4.0", global: null })).toEqual({ + kind: "missing", + command: "npm i -g om-memory-system", + }); + }); +}); + +describe("Directory maps", () => { + it("saves kept maps and accepted rows, and leaves out removed or rejected ones", () => { + const saved = [ + { from: "/a", to: "/x" }, + { from: "/b", to: "/y" }, + ]; + expect( + mapsToSave(saved, new Set(["/b"]), [ + { directory: "/code/app-feat-x", target: "/code/app", accepted: true }, + { directory: "/tmp/gone", target: "/tmp", accepted: false }, + { directory: "/empty", target: " ", accepted: true }, + ]) + ).toEqual([ + { from: "/a", to: "/x" }, + { from: "/code/app-feat-x", to: "/code/app" }, + ]); + }); +}); + +describe("Automatic import progress", () => { + const run = { + surface: "cli" as const, + state: "running", + total: 1000, + done: 400, + percent: 40, + minutesLeft: 120, + paused: false, + }; + + it("shows percentage, done of total, and minutes left or unknown", () => { + expect(progressView(run)).toEqual({ + percent: 40, + done: "400 / 1,000", + minutesLeft: "about 120", + }); + expect(progressView({ ...run, minutesLeft: null })?.minutesLeft).toBe("unknown"); + expect(progressView(null)).toBeNull(); + }); + + it("offers Pause while running, Resume while paused, and Run now otherwise", () => { + expect(backfillActions(run, null)).toEqual({ runNow: false, pause: true, resume: false }); + expect(backfillActions({ ...run, state: "stopped", paused: true }, null)).toEqual({ + runNow: false, + pause: false, + resume: true, + }); + expect(backfillActions({ ...run, state: "done" }, null).runNow).toBe(true); + expect(backfillActions(null, "Open Pi")).toEqual({ + runNow: false, + pause: false, + resume: false, + }); + }); +}); diff --git a/tests/web-settings-api.test.ts b/tests/web-settings-api.test.ts index cbb9443a..0d833908 100644 --- a/tests/web-settings-api.test.ts +++ b/tests/web-settings-api.test.ts @@ -108,8 +108,13 @@ describe("settings API", () => { ); expect(result.status).toBe(200); expect(result.body.settings.piModel).toMatchObject({ value: "project", source: "project" }); - expect(result.body.secrets.memoryApiKey).toMatchObject({ set: true, source: "env" }); - expect(JSON.stringify(result.body)).not.toContain("OMMS_SECRET_TEST"); + expect(result.body.secrets.memoryApiKey).toMatchObject({ + set: true, + source: "env", + reference: "OMMS_SECRET_TEST", + }); + // The variable name is shown; its value never is. + expect(JSON.stringify(result.body)).not.toContain("private-test-value"); }); it("saves all automatic import and web app settings without changing live model keys", async () => { @@ -213,6 +218,123 @@ describe("settings API", () => { expect(result.foreign).toBe(403); }); + it("guards Run now, Pause, and Resume and needs the external API without a host", async () => { + const result = await scenario( + ` + const { join } = await import("node:path"); + const { CONFIG } = await import(${JSON.stringify(pathToFileURL(join(import.meta.dir, "../src/config.ts")).href)}); + CONFIG.storagePath = join(process.env.HOME, "isolated-store"); + const json = { "content-type": "application/json" }; + const form = await send("/api/settings/backfill/pi/run", "POST", {}, { "content-type": "text/plain" }); + const foreign = await send("/api/settings/backfill/pi/run", "POST", {}, { ...json, origin: "https://example.com" }); + const network = new WebServer({ enabled: true, host: "0.0.0.0", port: 4747, apiToken: "network-test-token" }); + const noToken = await network.handleRequest(new Request("http://127.0.0.1:4747/api/settings/backfill/pi/pause", { + method: "POST", headers: { ...json, "x-omms-token": token }, body: "{}" })); + CONFIG.piBackfillModel = "zai/glm-5-turbo"; + const hostModel = await send("/api/settings/backfill/pi/run", "POST", {}, json); + CONFIG.opencodeBackfillModel = "external"; + CONFIG.memoryModel = undefined; + const unconfigured = await send("/api/settings/backfill/opencode/run", "POST", {}, json); + const pause = await send("/api/settings/backfill/opencode/pause", "POST", {}, json); + const runs = await (await send("/api/settings/backfill/runs")).json(); + const { startImportRun } = await import(${JSON.stringify(pathToFileURL(join(import.meta.dir, "../src/importer/import-runs.ts")).href)}); + CONFIG.piBackfillModel = "external"; + CONFIG.memoryModel = "m"; CONFIG.memoryApiUrl = "https://x.invalid"; CONFIG.memoryApiKey = "k"; + await startImportRun("pi", "cli"); + const busy = await send("/api/settings/backfill/pi/run", "POST", {}, json); + return { form: form.status, foreign: foreign.status, noToken: noToken.status, + hostModel: [hostModel.status, (await hostModel.json()).error], + unconfigured: [unconfigured.status, (await unconfigured.json()).error], + pause: pause.status, paused: runs.opencode.run.paused, + piUnavailable: runs.pi.runNowUnavailable, + busy: [busy.status, (await busy.json()).error] }; + ` + ); + expect(result.form).toBe(415); + expect(result.foreign).toBe(403); + expect(result.noToken).toBe(401); + expect(result.hostModel).toEqual([ + 409, + "Open Pi, or choose the external API for Pi's backfill", + ]); + expect(result.unconfigured[0]).toBe(409); + expect(result.unconfigured[1]).toContain("memoryModel is not configured"); + expect(result.pause).toBe(200); + expect(result.paused).toBe(true); + expect(result.piUnavailable).toContain("Open Pi"); + expect(result.busy).toEqual([409, "A Pi import is already running"]); + }); + + it("saves a pasted key to a user-only file and never returns or logs it", async () => { + const result = await scenario(` + const { readFileSync, statSync, existsSync } = await import("node:fs"); + const { join } = await import("node:path"); + const json = { "content-type": "application/json" }; + const settings = async () => (await send("/api/settings")).json(); + const key = "sk-pasted-private-value"; + const saved = await send("/api/settings/external-api/key", "POST", + { source: "paste", name: "zai", value: key, revision: (await settings()).revision }, json); + const savedText = await saved.text(); + const again = await send("/api/settings/external-api/key", "POST", + { source: "paste", name: "zai", value: key, revision: (await settings()).revision }, json); + const againText = await again.text(); + const network = new WebServer({ enabled: true, host: "0.0.0.0", port: 4747, apiToken: "network-test-token" }); + const remote = await network.handleRequest(new Request("http://127.0.0.1:4747/api/settings/external-api/key", { + method: "POST", headers: { ...json, authorization: "Bearer network-test-token" }, + body: JSON.stringify({ source: "paste", name: "remote", value: key, revision: "x" }) })); + const dir = join(process.env.HOME, ".config", "omms", "secrets"); + const file = join(dir, "zai.key"); + const config = readFileSync(configPath, "utf8"); + const log = existsSync(process.env.OMMS_LOG_FILE) ? readFileSync(process.env.OMMS_LOG_FILE, "utf8") : ""; + const snapshot = JSON.stringify(await settings()); + return { status: saved.status, savedText, again: again.status, againText, + remote: remote.status, remoteFile: existsSync(join(dir, "remote.key")), + fileMode: (statSync(file).mode & 0o777).toString(8), dirMode: (statSync(dir).mode & 0o777).toString(8), + fileContent: readFileSync(file, "utf8").trim(), config, log, snapshot, file }; + `); + const key = "sk-pasted-private-value"; + expect(result.status).toBe(200); + expect(result.again).toBe(409); + expect(result.remote).toBe(403); + expect(result.remoteFile).toBe(false); + expect(result.fileContent).toBe(key); + if (process.platform !== "win32") { + expect(result.fileMode).toBe("600"); + expect(result.dirMode).toBe("700"); + } + expect(result.config).toContain(`"memoryApiKey": "file://${result.file}"`); + for (const text of [ + result.savedText, + result.againText, + result.config, + result.log, + result.snapshot, + ]) { + expect(text).not.toContain(key); + } + }); + + it("saves an env reference and reports it does not resolve in the web app", async () => { + const result = await scenario(` + const json = { "content-type": "application/json" }; + const before = await (await send("/api/settings")).json(); + const saved = await send("/api/settings/external-api/key", "POST", + { source: "env", name: "OMMS_MISSING_TEST_KEY", revision: before.revision }, json); + const missingFile = await send("/api/settings/external-api/key", "POST", + { source: "file", path: "/no/such/omms.key", revision: before.revision }, json); + const after = await (await send("/api/settings")).json(); + return { saved: saved.status, missingFile: missingFile.status, key: after.externalKey }; + `); + expect(result.saved).toBe(200); + expect(result.missingFile).toBe(400); + expect(result.key).toMatchObject({ + source: "env", + reference: "OMMS_MISSING_TEST_KEY", + resolvesInWebApp: false, + }); + expect(result.key.warning).toContain("shell profile"); + }); + it("lists import sessions over guarded JSON POSTs with metadata only", async () => { const result = await scenario(` const { mkdirSync, writeFileSync } = await import("node:fs"); @@ -220,11 +342,15 @@ describe("settings API", () => { mkdirSync(root, { recursive: true }); writeFileSync(root + "/s.jsonl", JSON.stringify({ type: "session", version: 3, id: "sess-1", cwd: process.env.HOME }) + "\\n" + JSON.stringify({ type: "message", id: "m1", parentId: null, timestamp: "2026-01-01T00:00:00Z", message: { role: "user", content: "secret prompt text" } }) + "\\n"); + mkdirSync(process.env.HOME + "/code/app/.git", { recursive: true }); + writeFileSync(root + "/gone.jsonl", JSON.stringify({ type: "session", version: 3, id: "sess-2", cwd: process.env.HOME + "/code/app-feat-x" }) + "\\n" + + JSON.stringify({ type: "message", id: "m2", parentId: null, timestamp: "2026-01-02T00:00:00Z", message: { role: "user", content: "other prompt" } }) + "\\n"); const json = { "content-type": "application/json" }; const validate = await send("/api/settings/imports/sources/validate", "POST", { host: "pi", path: root }, json); const source = await validate.json(); const list = await send("/api/settings/imports/sessions", "POST", { host: "pi", source: source.sourceToken, scope: "all-projects", refresh: true }, json); const listed = await list.json(); + const maps = await (await send("/api/settings/import-maps")).json(); const relative = await send("/api/settings/imports/sources/validate", "POST", { host: "pi", path: "pi-sessions" }, json); const getList = await send("/api/settings/imports/sessions"); const readiness = await send("/api/settings/imports/readiness"); @@ -232,7 +358,7 @@ describe("settings API", () => { return { validate: validate.status, kind: source.kind, list: list.status, keys: listed.rows.map((row) => row.key), text: JSON.stringify(listed), - relative: relative.status, getList: getList.status, + relative: relative.status, getList: getList.status, maps, readiness: readiness.status, readinessKeys: Object.keys(await readiness.json()).sort(), browse: browse.status, entries: (await browse.json()).entries.map((entry) => entry.name), }; @@ -240,7 +366,16 @@ describe("settings API", () => { expect(result.validate).toBe(200); expect(result.kind).toBe("pi-folder"); expect(result.list).toBe(200); - expect(result.keys).toEqual(["s.jsonl"]); + expect(result.keys).toEqual(["s.jsonl", "gone.jsonl"]); + // The listing feeds the Directory maps list, with a suggestion and no content. + expect(result.maps.pi).toEqual([ + { + directory: expect.stringContaining("code/app-feat-x"), + sessions: 1, + suggestion: expect.stringMatching(/code[\\/]app$/), + }, + ]); + expect(JSON.stringify(result.maps)).not.toContain("other prompt"); expect(result.text).not.toContain("secret prompt text"); expect(result.relative).toBe(400); expect(result.getList).not.toBe(200); diff --git a/web/src/App.tsx b/web/src/App.tsx index b92ef27e..df5d32dc 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,10 +1,12 @@ import { useEffect, useState, type MouseEvent } from "react"; +import { Select } from "$lib/components/ui/select"; import { Loader, Menu, Plus, RefreshCw, Search, Trash, TriangleAlert, X } from "lucide-react"; import { useMemoriesExplorer } from "@/hooks/useMemoriesExplorer"; import { useUserProfile } from "@/hooks/useUserProfile"; import { AiCleanupDialog } from "$lib/components/explorer/AiCleanupDialog"; import { AppSidebar } from "$lib/components/explorer/AppSidebar"; import { EditMemoryDialog } from "$lib/components/explorer/EditMemoryDialog"; +import { KeywordBadge } from "$lib/components/explorer/KeywordBadge"; import { MemoryList } from "$lib/components/explorer/MemoryList"; import { ProfileView } from "$lib/components/explorer/ProfileView"; import { TagMigrationDialog } from "$lib/components/explorer/TagMigrationDialog"; @@ -15,6 +17,7 @@ import { Checkbox } from "$lib/components/ui/checkbox"; import { Input } from "$lib/components/ui/input"; import { Label } from "$lib/components/ui/label"; import { Toaster } from "$lib/components/ui/sonner"; +import { Tooltip } from "$lib/components/ui/tooltip"; import { Textarea } from "$lib/components/ui/textarea"; import { setLanguage, useI18n } from "$lib/i18n"; import type { Lang } from "$lib/i18n/translations"; @@ -106,11 +109,18 @@ export default function App() { brand={t("brand")} projectLabel={t("tab-project")} profileLabel={t("tab-profile")} + profileSections={[ + { id: "profile-preferences", label: t("profile-preferences") }, + { id: "profile-patterns", label: t("profile-patterns") }, + { id: "profile-workflows", label: t("profile-workflows") }, + ]} langLabel={language.toUpperCase()} languageLabel={t("nav-language")} themeLabel={t("nav-theme")} settingsLabel={t("nav-settings")} closeLabel={t("nav-close")} + collapseLabel={t("nav-collapse")} + expandLabel={t("nav-expand")} onLanguageSelect={onLanguageSelect} /> @@ -168,11 +178,11 @@ export default function App() { {currentView === "project" ? ( <>
-
+
- +
+ {explorer.selectedKeyword ? ( +
+ {t("label-keyword")} +
+ + +
+
+ ) : null} +
- - + + + + + +
{explorer.selectedIds.size > 0 ? ( @@ -292,6 +324,14 @@ export default function App() { onEdit={explorer.openEdit} onDeleteMemory={explorer.deleteMemory} onDeletePrompt={explorer.deletePrompt} + activeKeyword={explorer.selectedKeyword} + onKeywordClick={(keyword) => + explorer.onKeywordFilterChange( + explorer.selectedKeyword.toLowerCase() === keyword.toLowerCase() + ? "" + : keyword + ) + } />
@@ -300,7 +340,7 @@ export default function App() {
- +
- +
diff --git a/web/src/app.css b/web/src/app.css index acdd0b4d..999901fd 100644 --- a/web/src/app.css +++ b/web/src/app.css @@ -170,3 +170,14 @@ display: grid; place-items: center; } + +/* Settings buttons: pointer and an eased shift to the primary green on hover, like the sidebar icons. */ +.settings-view button:not(:disabled) { + cursor: pointer; +} +.settings-view button:not([class*="bg-primary"]) { + transition: color 200ms ease-in; +} +.settings-view button:not(:disabled):not([class*="bg-primary"]):hover { + color: var(--primary); +} diff --git a/web/src/hooks/useMemoriesExplorer.ts b/web/src/hooks/useMemoriesExplorer.ts index 25891376..46c25a8c 100644 --- a/web/src/hooks/useMemoriesExplorer.ts +++ b/web/src/hooks/useMemoriesExplorer.ts @@ -9,6 +9,7 @@ export function useMemoriesExplorer() { const [memories, setMemories] = useState([]); const [selectedIds, setSelectedIds] = useState>(new Set()); const [selectedTag, setSelectedTag] = useState(""); + const [selectedKeyword, setSelectedKeyword] = useState(""); const [searchQuery, setSearchQuery] = useState(""); const [searchInput, setSearchInput] = useState(""); const [isSearching, setIsSearching] = useState(false); @@ -38,8 +39,9 @@ export function useMemoriesExplorer() { isSearching, searchQuery, selectedTag, + selectedKeyword, }); - stateRef.current = { currentPage, isSearching, searchQuery, selectedTag }; + stateRef.current = { currentPage, isSearching, searchQuery, selectedTag, selectedKeyword }; const setSelected = useCallback((next: Set) => { setSelectedIds(new Set(next)); @@ -65,12 +67,14 @@ export function useMemoriesExplorer() { isSearching?: boolean; searchQuery?: string; selectedTag?: string; + selectedKeyword?: string; }) => { const { currentPage: page, isSearching: searching, searchQuery: query, selectedTag: tag, + selectedKeyword: keyword, } = { ...stateRef.current, ...overrides, @@ -83,8 +87,9 @@ export function useMemoriesExplorer() { if (searching) { endpoint = `/api/search?q=${encodeURIComponent(query)}&page=${page}&pageSize=20`; if (tag) endpoint += `&tag=${encodeURIComponent(tag)}`; - } else if (tag) { - endpoint += `&tag=${encodeURIComponent(tag)}`; + } else { + if (tag) endpoint += `&tag=${encodeURIComponent(tag)}`; + if (keyword) endpoint += `&keyword=${encodeURIComponent(keyword)}`; } const result = await fetchAPI<{ @@ -204,6 +209,20 @@ export function useMemoriesExplorer() { }); } + function onKeywordFilterChange(value: string) { + setSelectedKeyword(value); + setCurrentPage(1); + setIsSearching(false); + setSearchQuery(""); + setSearchInput(""); + void loadMemories({ + page: 1, + isSearching: false, + searchQuery: "", + selectedKeyword: value, + }); + } + function onSelect(id: string, selected: boolean) { const next = new Set(selectedIds); if (selected) next.add(id); @@ -407,6 +426,8 @@ export function useMemoriesExplorer() { memories, selectedIds, selectedTag, + selectedKeyword, + onKeywordFilterChange, searchInput, setSearchInput, isSearching, diff --git a/web/src/lib/auto-import-settings.ts b/web/src/lib/auto-import-settings.ts index 025b1b82..146d0637 100644 --- a/web/src/lib/auto-import-settings.ts +++ b/web/src/lib/auto-import-settings.ts @@ -5,13 +5,52 @@ export function shouldPollBackfill(rows: Record): b return rows.pi?.state === "running" || rows.opencode?.state === "running"; } +/** One host's run record, as `/api/settings/backfill/runs` reports it. */ +export type ImportRunView = { + surface: "auto" | "web" | "cli" | "slash" | null; + state: string | null; + total: number; + done: number; + percent: number; + minutesLeft: number | null; + paused: boolean; +} | null; + +/** Progress text for a run: percentage, done of total, and minutes left or unknown. */ +export function progressView(run: ImportRunView): { + percent: number; + done: string; + minutesLeft: string; +} | null { + if (!run || run.total <= 0) return null; + return { + percent: run.percent, + done: `${run.done.toLocaleString("en-US")} / ${run.total.toLocaleString("en-US")}`, + minutesLeft: run.minutesLeft === null ? "unknown" : `about ${run.minutesLeft}`, + }; +} + +/** Which of Run now, Pause, and Resume apply to a host right now. */ +export function backfillActions( + run: ImportRunView, + unavailable: string | null +): { runNow: boolean; pause: boolean; resume: boolean } { + const running = run?.state === "running"; + const paused = run?.paused === true; + return { + runNow: !running && !paused && !unavailable, + pause: running && !paused, + resume: paused && !unavailable, + }; +} + export function manualModelFieldVisible(typedMode: boolean | undefined, known: boolean): boolean { return typedMode ?? !known; } export function backfillModelEdit(host: BackfillHost, value: string): Record { const slash = value.indexOf("/"); - if (value !== "inherit" && (slash < 1 || slash === value.length - 1)) { + if (value !== "inherit" && value !== "external" && (slash < 1 || slash === value.length - 1)) { throw new Error("Enter a model as provider/model."); } return { [`${host}BackfillModel`]: value }; diff --git a/web/src/lib/components/explorer/AppSidebar.tsx b/web/src/lib/components/explorer/AppSidebar.tsx index 4670402f..82571dcd 100644 --- a/web/src/lib/components/explorer/AppSidebar.tsx +++ b/web/src/lib/components/explorer/AppSidebar.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef, useState, type KeyboardEvent, type MouseEvent } from "react"; -import { Folder, Moon, Settings, Sun, User, X } from "lucide-react"; +import { Folder, Moon, PanelLeftClose, PanelLeftOpen, Settings, Sun, User, X } from "lucide-react"; import type { Lang } from "$lib/i18n/translations"; import { GithubIcon } from "$lib/components/icons/GithubIcon"; import { Button } from "$lib/components/ui/button"; @@ -14,11 +14,14 @@ type Props = { brand: string; projectLabel: string; profileLabel: string; + profileSections: { id: string; label: string }[]; langLabel: string; languageLabel: string; themeLabel: string; settingsLabel: string; closeLabel: string; + collapseLabel: string; + expandLabel: string; onOpenChange?: (open: boolean) => void; onLanguageSelect?: (language: Lang) => void; }; @@ -29,17 +32,30 @@ const LANGUAGE_OPTIONS: { code: Lang; label: string }[] = [ { code: "ar", label: "العربية (AR)" }, ]; +const COLLAPSED_KEY = "omms-sidebar-collapsed"; + +function readCollapsed(): boolean { + try { + return localStorage.getItem(COLLAPSED_KEY) === "1"; + } catch { + return false; + } +} + export function AppSidebar({ open = false, currentView, brand, projectLabel, profileLabel, + profileSections, langLabel, languageLabel, themeLabel, settingsLabel, closeLabel, + collapseLabel, + expandLabel, onOpenChange, onLanguageSelect, }: Props) { @@ -48,6 +64,19 @@ export function AppSidebar({ const [languageMenuOpen, setLanguageMenuOpen] = useState(false); const languageContainer = useRef(null); const languageTrigger = useRef(null); + // Collapsing only applies on desktop; the mobile drawer always shows the full sidebar. + const [collapsed, setCollapsed] = useState(readCollapsed); + const onDesktop = (classes: string) => (collapsed ? classes : ""); + + function toggleCollapsed() { + const next = !collapsed; + setCollapsed(next); + try { + localStorage.setItem(COLLAPSED_KEY, next ? "1" : "0"); + } catch { + // Storage can be blocked; the toggle still works for this page view. + } + } useEffect(() => { if (!languageMenuOpen) return; @@ -102,9 +131,32 @@ export function AppSidebar({ setOpen(false); } + function onSectionClick(event: MouseEvent, id: string) { + if (event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) { + return; + } + event.preventDefault(); + const scroll = () => + document.getElementById(id)?.scrollIntoView({ behavior: "smooth", block: "start" }); + if (currentView === "profile") { + scroll(); + } else { + navigate(ROUTES.profile); + // The profile view renders after navigation; wait for its sections to appear. + let tries = 0; + const wait = () => { + if (document.getElementById(id)) scroll(); + else if (tries++ < 50) requestAnimationFrame(wait); + }; + requestAnimationFrame(wait); + } + setOpen(false); + } + function navClass(active: boolean) { return cn( "flex w-full items-center gap-2.5 rounded-xl px-3 py-2 text-sm transition-colors", + onDesktop("md:justify-center md:px-0"), active ? "bg-sidebar-accent text-sidebar-accent-foreground" : "text-sidebar-foreground/80 hover:bg-sidebar-accent/70 hover:text-sidebar-accent-foreground" @@ -124,12 +176,13 @@ export function AppSidebar({
- + {isLinked ? ( + + + + ) : ( + + )}

{item.content}

{isLinked ? ( @@ -259,22 +285,40 @@ export function MemoryCard({ - + {isLinked ? ( + + + + ) : ( + + )}
{item.tags?.length ? (
{item.tags.map((tag) => ( - - {tag} - + ))}
) : null} diff --git a/web/src/lib/components/explorer/MemoryList.tsx b/web/src/lib/components/explorer/MemoryList.tsx index f82e3ed3..bb303f43 100644 --- a/web/src/lib/components/explorer/MemoryList.tsx +++ b/web/src/lib/components/explorer/MemoryList.tsx @@ -22,6 +22,8 @@ type Props = { onEdit: (id: string) => void; onDeleteMemory: (id: string, isLinked: boolean) => void; onDeletePrompt: (id: string, isLinked: boolean) => void; + activeKeyword?: string; + onKeywordClick?: (keyword: string) => void; }; export function MemoryList({ @@ -38,6 +40,8 @@ export function MemoryList({ onEdit, onDeleteMemory, onDeletePrompt, + activeKeyword, + onKeywordClick, }: Props) { const { t } = useI18n(); const groups = useMemo(() => groupMemories(memories), [memories]); @@ -59,7 +63,7 @@ export function MemoryList({ return (
-
+
{loading && memories.length === 0 ? (
{t("loading-init")}
) : error ? ( @@ -83,6 +87,8 @@ export function MemoryList({ onEdit={onEdit} onDeleteMemory={onDeleteMemory} onDeletePrompt={onDeletePrompt} + activeKeyword={activeKeyword} + onKeywordClick={onKeywordClick} /> ) : group.type === "prompt" ? ( ) : ( ) ) diff --git a/web/src/lib/components/explorer/ProfileView.tsx b/web/src/lib/components/explorer/ProfileView.tsx index a4d800e7..7c898e73 100644 --- a/web/src/lib/components/explorer/ProfileView.tsx +++ b/web/src/lib/components/explorer/ProfileView.tsx @@ -262,8 +262,8 @@ export function ProfileView({ profile, loading = false, onRefresh, onCleanup }:
-
-
+
+

{t("profile-preferences")} @@ -273,7 +273,7 @@ export function ProfileView({ profile, loading = false, onRefresh, onCleanup }:

{t("empty-preferences")}

) : ( <> -
+
{prefPage.items.map((item) => ( -
+

{t("profile-patterns")} @@ -298,7 +298,7 @@ export function ProfileView({ profile, loading = false, onRefresh, onCleanup }:

{t("empty-patterns")}

) : ( <> -
+
{patPage.items.map((item) => ( -
+

{t("profile-workflows")} diff --git a/web/src/lib/components/settings/AutoImportSection.tsx b/web/src/lib/components/settings/AutoImportSection.tsx index 59eb4cb3..7070a333 100644 --- a/web/src/lib/components/settings/AutoImportSection.tsx +++ b/web/src/lib/components/settings/AutoImportSection.tsx @@ -1,14 +1,24 @@ import { useEffect, useState } from "react"; +import { Select } from "$lib/components/ui/select"; import { + backfillActions, backfillModelEdit, manualModelFieldVisible, + progressView, shouldPollBackfill, type BackfillHost, + type ImportRunView, } from "$lib/auto-import-settings"; +import { externalMissing } from "$lib/external-api-settings"; import { onSettingsSnapshot, reloadSettingsSnapshot, settingsRequest } from "$lib/settings-api"; import { useSettingsText } from "$lib/i18n/settings"; -type Snapshot = { revision: string; settings: Record }; +type Snapshot = { + revision: string; + settings: Record; + secrets?: Record; +}; +type Runs = Record; type ModelList = { available: boolean; models?: Array<{ provider: string; model: string; name: string }>; @@ -32,12 +42,20 @@ export function AutoImportSection() { const s = useSettingsText(); const [snapshot, setSnapshot] = useState(); const [rows, setRows] = useState({ pi: null, opencode: null }); + const [runs, setRuns] = useState(); const [lists, setLists] = useState>({}); const [choices, setChoices] = useState>>({}); const [typedModes, setTypedModes] = useState>>({}); const [message, setMessage] = useState(""); const [busy, setBusy] = useState(false); - const polling = shouldPollBackfill(rows); + const polling = + shouldPollBackfill(rows) || + runs?.pi.run?.state === "running" || + runs?.opencode.run?.state === "running"; + const loadRuns = () => + settingsRequest("/api/settings/backfill/runs") + .then(setRuns) + .catch((error: Error) => setMessage(error.message)); useEffect(() => { let active = true; void settingsRequest("/api/settings") @@ -54,6 +72,7 @@ export function AutoImportSection() { .catch((error: Error) => { if (active) setMessage(error.message); }); + void loadRuns(); for (const host of ["pi", "opencode"] as const) { void settingsRequest(`/api/settings/models?host=${host}`) .then((value) => { @@ -83,6 +102,7 @@ export function AutoImportSection() { .catch((error: Error) => { if (active) setMessage(error.message); }); + if (active) void loadRuns(); }, 3000); return () => { active = false; @@ -98,13 +118,29 @@ export function AutoImportSection() { method: "PATCH", body: JSON.stringify({ edits, revision: snapshot.revision }), }); - setMessage(s("Saved. Changes apply at the next host start.")); + setMessage(s("Saved. Changes apply at the next run.")); } catch (error) { setMessage((error as Error).message); } await reloadSettingsSnapshot(); setBusy(false); } + async function control(host: BackfillHost, action: "run" | "pause" | "resume") { + setBusy(true); + try { + await settingsRequest(`/api/settings/backfill/${host}/${action}`, { + method: "POST", + body: "{}", + }); + setMessage( + s(action === "pause" ? "Pausing after the current exchange." : "Backfill started.") + ); + } catch (error) { + setMessage((error as Error).message); + } + await loadRuns(); + setBusy(false); + } function saveModel(host: BackfillHost, choice: string) { try { void save(backfillModelEdit(host, choice.trim())); @@ -121,7 +157,7 @@ export function AutoImportSection() {

{s("Automatic import")}

{s( - "Automatic import makes model calls. Changes apply at the next host start. Turning it off stops a running import after the current exchange." + "Automatic import makes model calls. A model change applies at the next run. Turning it off stops a running import after the current exchange." )}