diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 89fdd00..50e73be 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -7,8 +7,8 @@ { "name": "claude-code-substrate-memory", "source": "./", - "description": "Substrate organizational memory for Claude Code.", - "version": "0.1.0" + "description": "Hermes-authoritative hosted Substrate memory for Claude Code.", + "version": "0.2.0" } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 5447a8f..fdfb9a1 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "claude-code-substrate-memory", - "version": "0.1.0", - "description": "Substrate organizational memory for Claude Code with MCP tools and fail-open session capture.", + "version": "0.2.0", + "description": "Hermes-authoritative hosted Substrate memory for Claude Code with consent-gated capture.", "author": { "name": "Sightline Technologies Inc" }, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7da347b..2022cf1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,22 +13,64 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - # Runs on the pristine checkout, before any dependency is installed. - run: python scripts/verify_public_plugin_candidate.py --root . - test: - runs-on: ubuntu-latest + host-contract: strategy: fail-fast: false matrix: + os: [ubuntu-latest, macos-latest, windows-latest] python: ["3.11", "3.12"] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python }} - - run: python -m pip install --upgrade pip + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "22" - run: python -m pip install -e ".[dev]" - run: ruff check . + - run: ruff format --check . - run: python -m compileall -q src scripts - run: python -m pytest -q + - run: node scripts/plugin_runtime.cjs status + env: + SUBSTRATE_STATE_HOME: ${{ runner.temp }}/substrate-state + + full-continuation: + name: full-continuation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "22" + - run: python -m pip install -e ".[dev]" + - name: Run required 50 MiB real-MCP SIGKILL/restart continuation gate + env: + SUBSTRATE_FULL_CONTINUATION: "1" + run: | + set -euo pipefail + python -m pytest -q tests/test_continuation_process.py::test_full_posix_mcp_sigkill_restart_combined_corpus | tee full-continuation.log + grep -F "1 passed" full-continuation.log + + protected-publication-gate: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: [publication-scan, host-contract, full-continuation] + runs-on: ubuntu-latest + steps: + - name: Require exact protected publication check + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + required="$(gh api "repos/$GITHUB_REPOSITORY/branches/main/protection/required_status_checks" \ + --jq '.contexts[]?, .checks[]?.context')" + printf '%s\n' "$required" | grep -Fx 'full-continuation' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8bf209e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,225 @@ +name: release + +on: + workflow_dispatch: + inputs: + source_sha: + description: Exact independently reviewed protected-main SHA + required: true + type: string + +concurrency: + group: release-v0.2.0 + cancel-in-progress: false + +permissions: + contents: read + +jobs: + platform-gates: + if: github.ref == 'refs/heads/main' && github.sha == inputs.source_sha + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ["3.11", "3.12"] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ inputs.source_sha }} + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ matrix.python }} + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "22" + - run: python -m pip install -e ".[dev]" + - run: ruff check . + - run: ruff format --check . + - run: python -m compileall -q src scripts + - run: python -m pytest -q + - run: node scripts/plugin_runtime.cjs status + env: + SUBSTRATE_STATE_HOME: ${{ runner.temp }}/substrate-state + + full-continuation: + name: full-continuation + if: github.ref == 'refs/heads/main' && github.sha == inputs.source_sha + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ inputs.source_sha }} + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "22" + - run: python -m pip install -e ".[dev]" + - name: Run required 50 MiB real-MCP SIGKILL/restart continuation gate + env: + SUBSTRATE_FULL_CONTINUATION: "1" + run: | + set -euo pipefail + python -m pytest -q tests/test_continuation_process.py::test_full_posix_mcp_sigkill_restart_combined_corpus | tee full-continuation.log + grep -F "1 passed" full-continuation.log + + verify: + needs: [platform-gates, full-continuation] + if: github.ref == 'refs/heads/main' && github.sha == inputs.source_sha && needs.platform-gates.result == 'success' && needs.full-continuation.result == 'success' + runs-on: ubuntu-latest + outputs: + artifact_digest: ${{ steps.upload.outputs.artifact-digest }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ inputs.source_sha }} + fetch-depth: 0 + persist-credentials: false + - name: Bind exact source to protected main + env: + SOURCE_SHA: ${{ inputs.source_sha }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] + test "$GITHUB_REF" = refs/heads/main + test "$GITHUB_SHA" = "$SOURCE_SHA" + test "$(git rev-parse HEAD)" = "$SOURCE_SHA" + git fetch --no-tags origin main + test "$(git rev-parse origin/main)" = "$SOURCE_SHA" + test "$(gh api "repos/$GITHUB_REPOSITORY/branches/main" --jq .protected)" = true + test "$(gh api "repos/$GITHUB_REPOSITORY/branches/main" --jq .commit.sha)" = "$SOURCE_SHA" + required="$(gh api "repos/$GITHUB_REPOSITORY/branches/main/protection/required_status_checks" --jq '.contexts[]?, .checks[]?.context')" + printf '%s\n' "$required" | grep -Fx 'full-continuation' + if git ls-remote --exit-code --tags origin refs/tags/v0.2.0 >/dev/null 2>&1; then + echo 'immutable release tag already exists' >&2 + exit 1 + fi + - name: Scan pristine candidate + run: python scripts/verify_public_plugin_candidate.py --root . + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "22" + - run: python -m pip install -e ".[dev]" + - name: Run complete release checks + run: | + set -euo pipefail + ruff check . + ruff format --check . + python -m compileall -q src scripts + python -m pytest -q + git diff --exit-code + - name: Build exact release bytes twice + env: + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + set -euo pipefail + python scripts/build_release.py --source-commit "$SOURCE_SHA" --output "$RUNNER_TEMP/a" + python scripts/build_release.py --source-commit "$SOURCE_SHA" --output "$RUNNER_TEMP/b" + test "$(find "$RUNNER_TEMP/a" -maxdepth 1 -type f -printf '%f\n' | sort | tr '\n' ' ')" = \ + 'SHA256SUMS claude_code_substrate_memory.zip install_claude_plugin.py ' + diff -r "$RUNNER_TEMP/a" "$RUNNER_TEMP/b" + (cd "$RUNNER_TEMP/a" && sha256sum -c SHA256SUMS) + mkdir dist + cp "$RUNNER_TEMP/a"/* dist/ + - id: upload + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: release-${{ inputs.source_sha }} + path: dist/* + if-no-files-found: error + compression-level: 0 + retention-days: 1 + + publish: + needs: [verify, platform-gates, full-continuation] + if: github.ref == 'refs/heads/main' && github.sha == inputs.source_sha && needs.verify.result == 'success' && needs.platform-gates.result == 'success' && needs.full-continuation.result == 'success' + runs-on: ubuntu-latest + environment: public-release + permissions: + actions: read + contents: write + id-token: write + attestations: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: release-${{ inputs.source_sha }} + path: dist + - name: Recheck protected current main after environment approval + env: + GH_TOKEN: ${{ github.token }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + set -euo pipefail + test "$GITHUB_REF" = refs/heads/main + test "$GITHUB_SHA" = "$SOURCE_SHA" + test "$(gh api "repos/$GITHUB_REPOSITORY/branches/main" --jq .protected)" = true + test "$(gh api "repos/$GITHUB_REPOSITORY/branches/main" --jq .commit.sha)" = "$SOURCE_SHA" + required="$(gh api "repos/$GITHUB_REPOSITORY/branches/main/protection/required_status_checks" --jq '.contexts[]?, .checks[]?.context')" + printf '%s\n' "$required" | grep -Fx 'full-continuation' + checks="$(gh api --paginate "repos/$GITHUB_REPOSITORY/commits/$SOURCE_SHA/check-runs" --jq '.check_runs[] | [.name,.status,.conclusion] | @tsv')" + test "$(printf '%s\n' "$checks" | awk -F '\t' '$1 == "full-continuation" && $2 == "completed" && $3 == "success" { count += 1 } END { print count + 0 }')" -ge 1 + for os in ubuntu-latest macos-latest windows-latest; do + for py in 3.11 3.12; do + printf '%s\n' "$checks" | grep -F "platform-gates ($os, $py)" | grep -F $'\tcompleted\tsuccess' >/dev/null + done + done + if git ls-remote --exit-code --tags "https://github.com/$GITHUB_REPOSITORY.git" refs/tags/v0.2.0 >/dev/null 2>&1; then + echo 'immutable release tag already exists' >&2 + exit 1 + fi + - name: Read back exact closed artifact set + run: | + set -euo pipefail + test -n '${{ needs.verify.outputs.artifact_digest }}' + test "$(find dist -maxdepth 1 -type f -printf '%f\n' | sort | tr '\n' ' ')" = \ + 'SHA256SUMS claude_code_substrate_memory.zip install_claude_plugin.py ' + (cd dist && sha256sum -c SHA256SUMS) + - uses: actions/attest-build-provenance@e3fe62ef559997059fe8380e7d2b4c909e2d65f4 # pinned + with: + subject-path: dist/* + - name: Create immutable tag and release + env: + GH_TOKEN: ${{ github.token }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + set -euo pipefail + test "$GITHUB_REF" = refs/heads/main + test "$GITHUB_SHA" = "$SOURCE_SHA" + test "$(gh api "repos/$GITHUB_REPOSITORY/branches/main" --jq .protected)" = true + test "$(gh api "repos/$GITHUB_REPOSITORY/branches/main" --jq .commit.sha)" = "$SOURCE_SHA" + required="$(gh api "repos/$GITHUB_REPOSITORY/branches/main/protection/required_status_checks" --jq '.contexts[]?, .checks[]?.context')" + printf '%s\n' "$required" | grep -Fx 'full-continuation' + if gh release view v0.2.0 --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo 'release already exists; refusing mutation' >&2 + exit 1 + fi + gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" \ + -f ref=refs/tags/v0.2.0 -f sha="$SOURCE_SHA" >/dev/null + hashes="$(cat dist/SHA256SUMS)" + gh release create v0.2.0 \ + dist/claude_code_substrate_memory.zip \ + dist/install_claude_plugin.py \ + dist/SHA256SUMS \ + --repo "$GITHUB_REPOSITORY" \ + --verify-tag \ + --draft \ + --title 'Claude Code Substrate Memory v0.2.0' \ + --notes "Immutable protected-main release. Verify both installer and archive against SHA256SUMS:\n\n$hashes" + mkdir readback + gh release download v0.2.0 --repo "$GITHUB_REPOSITORY" --dir readback + test "$(find readback -maxdepth 1 -type f -printf '%f\n' | sort | tr '\n' ' ')" = \ + 'SHA256SUMS claude_code_substrate_memory.zip install_claude_plugin.py ' + cmp dist/SHA256SUMS readback/SHA256SUMS + cmp dist/claude_code_substrate_memory.zip readback/claude_code_substrate_memory.zip + cmp dist/install_claude_plugin.py readback/install_claude_plugin.py + gh release edit v0.2.0 --repo "$GITHUB_REPOSITORY" --draft=false diff --git a/.mcp.json b/.mcp.json index 3b091a6..ae3660c 100644 --- a/.mcp.json +++ b/.mcp.json @@ -2,14 +2,11 @@ "mcpServers": { "substrate": { "type": "stdio", - "command": "python", + "command": "node", "args": [ - "-m", - "claude_code_memory.server" - ], - "env": { - "PYTHONPATH": "${CLAUDE_PLUGIN_ROOT}/src" - } + "${CLAUDE_PLUGIN_ROOT}/scripts/plugin_runtime.cjs", + "mcp" + ] } } } diff --git a/BOUNDARY.md b/BOUNDARY.md index 3a5ea73..d1aa782 100644 --- a/BOUNDARY.md +++ b/BOUNDARY.md @@ -13,7 +13,7 @@ This boundary is declared on day one and will not be moved later. Claude Code Su ## Current implementation status -Claude Code Substrate Memory contains the Claude Code plugin/client, local spool and checkpoint machinery, client-side credential redaction, host manifests, and installation metadata. It requires a configured Substrate server for remote memory and delivery. +Claude Code Substrate Memory contains the Claude Code plugin/client, local spool and checkpoint machinery, client-side credential redaction, host manifests, and installation metadata. It connects only to the fixed hosted origin after direct human device approval. Safe Claude history import, server revocation, and a no-server mode are not implemented. The local runtime, local entity model, privacy-deletion implementation, and policy compiler are on the open side of the permanent boundary but may not be implemented in a given release. The plugin does not authorize agent actions and does not provide a no-server mode unless explicitly documented. Those absences are product gaps, not held commercial features. diff --git a/CHANGELOG.md b/CHANGELOG.md index c33d98a..9214b0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,17 @@ # Changelog -## 0.1.0 — initial public release +## 0.2.0 - release candidate -- Quarantine every contributing record when a recognized credential spans same-lineage record boundaries, including zero-content gaps. -- Bound aggregate cross-record detector state with fail-closed, observable LRU eviction. -- Pair tool results with full, collision-resistant lineage and call identities. +- Adopt Hermes-authoritative fixed-origin RFC 8628 onboarding and secure profile custody. +- Suppress preapproval, sidechain, subagent, background, cron, and worker transcript capture. +- Add strict capture/entity capability gates and canonical-only automatic recall. +- Make spool admission, checkpoints, SessionEnd, retries, auth repair, and status durable/truthful. +- Add uncapped durable SessionEnd jobs, kill/restart recovery, and independent-process checkpoint transactions. +- Add a locked, journaled, content-addressed Claude registration transaction with real rollback, own uninstall, crash recovery, and same-version no-op. +- Terminate the complete POSIX hook process group after bounded grace and use waited Windows `taskkill /T /F` cleanup. +- Add combined large-transcript MCP SIGKILL/restart acceptance plus a CI-scaled cross-platform restart case. +- Retain symlink/TOCTOU rejection, deterministic release artifacts, and protected-main release tooling. +- Match all five Hermes wiki request shapes, including `/wiki/search` and ingest `source_type`. +- Declare safe Claude history import and remote revocation unsupported. + +No public release has been published yet. diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md new file mode 100644 index 0000000..88e9012 --- /dev/null +++ b/COMPATIBILITY.md @@ -0,0 +1,15 @@ +# Compatibility + +Release `0.2.0` accepts exactly Claude Code `2.1.237`, Python 3.11/3.12, and Node.js 20/22. +The generated installer refuses a different Claude Code version. CI runs launcher and unit tests +on Linux, macOS, and Windows; a real installed Claude host acceptance run is a human release gate. + +Hooks have a 15-second bound. Capture hooks use a small local admission budget and persist a resumable job; no detached worker, +OAuth, credential-helper delivery path, or network delivery consumes that host budget. MCP owns +continuation and delivery. Capture requests use at most 6 +seconds and drain uses at most 8 seconds per worker attempt. Automatic recall uses 2.5 seconds total +and 0.9 seconds per request. Device onboarding starts only from explicit `/substrate-setup`, is +resumable for at most 900 seconds, and uses 60-second OAuth/capability requests. + +Updates are manual, immutable, checksum-verified atomic swaps. The prior directory is retained for +rollback. State is not part of the code directory and is preserved. There is no auto-update. diff --git a/README.md b/README.md index f97875c..52cda1f 100644 --- a/README.md +++ b/README.md @@ -1,131 +1,167 @@ # Claude Code Substrate Memory -`claude-code-substrate-memory` connects Claude Code to Substrate organizational memory. It -provides seven MCP tools for reading and writing memory, captures new transcript turns through -Claude Code lifecycle hooks, and keeps failed deliveries in a bounded local spool. +This repository is a release candidate for a Claude Code adapter to hosted Substrate memory. +Its behavioral authority is Hermes main +`db4ddc2f093f833ffa62f4246860de96ba398713` and the released Hermes v2.0.3 bytes at +`e4ad07cfc858618edfd69e1f3be5e8345b253037`. The frozen shared capture core is useful +implementation material, but the cited Hermes revisions define behavior. + +Version `0.2.0` uses the Python standard library at runtime. It pins Claude Code +`2.1.237`, Python 3.11/3.12, and Node.js 20/22 for the portable launcher. + +## Publication and install status + +No public Claude release exists yet. Do not install from a mutable branch and do not call this +candidate published. `.github/workflows/release.yml` will release only an exact protected-main +SHA after all checks pass. It builds the same bytes twice and permits only: + +- `claude_code_substrate_memory.zip`; +- `install_claude_plugin.py`; and +- `SHA256SUMS`. + +The archive embeds `PROVENANCE.json` with its exact source SHA and a closed source-file digest +map. Release notes must publish independent SHA-256 values for both the installer and archive. +After publication, download all three assets from the immutable tag, compare the two release-note +hashes with `SHA256SUMS`, then run the verified installer. The installer independently checks the +archive hash, provenance closure, exact Claude Code version, and source SHA. It prepares one +read-only, content-addressed source below `CLAUDE_CONFIG_DIR/plugins/substrate-installer/releases`. + +A private OS lock serializes installer processes. A durable journal records the exact prior logical +plugin version, source, enabled state, and marketplace state before any Claude mutation. The +installer then uses only Claude Code 2.1.237's documented user-scope plugin commands to uninstall +the old registration, change the marketplace, register the new source, verify the exact enabled +version and MCP inventory, and commit installer state. A crash leaves the journal in place. The next +invocation resumes the verified transition. An exact same-version rerun is read-only: it does not +change Claude registration timestamps, source inode/mtime, installer state, or the journal. -Version 0.1.0 uses only the Python standard library at runtime. +```text +python3 install_claude_plugin.py claude_code_substrate_memory.zip --claude claude +claude plugin list --json +# Start a new Claude Code session, then run /substrate-setup. -## Install +# Rollback restores and verifies the actual prior Claude registration and source: +python3 install_claude_plugin.py --rollback --claude claude -Clone the repository and load it as a Claude Code plugin: +# Own removal deregisters both plugin and marketplace but preserves Substrate state: +python3 install_claude_plugin.py --uninstall --claude claude -```bash -git clone https://github.com/Substrate-memory/claude-code-substrate-memory.git -claude --plugin-dir /absolute/path/to/claude-code-substrate-memory +# Optional source purge is separate and requires the exact plugin ID as confirmation: +python3 install_claude_plugin.py --uninstall --purge \ + --confirm-purge claude-code-substrate-memory@claude-code-substrate-memory --claude claude ``` -For development or direct CLI use, install the package into a Python 3.11+ environment: - -```bash -python -m pip install -e . -substrate-claude-code status -``` +The `python3` spelling is the documented Linux/macOS path. On Windows, run the same installer with +`py -3.12` or `py -3.11`. No test requires a credential or Substrate request. Repository tests use +a stateful fake host for failure injection. Release acceptance must also repeat the complete +install/update/rollback/uninstall sequence on exact Claude Code 2.1.237; fake-host coverage alone +does not prove Anthropic's implementation. + +Branch protection and the external server changes in `docs/server-integration.md` remain human +release gates. + +## First activation and consent + +First primary `SessionStart` prints a short instruction to run `/substrate-setup`; it performs no +network or credential-helper work inside the 15-second hook. That explicit command begins or +resumes RFC 8628 authorization at the fixed origin `https://app.trysubstrate.co`. The public +client is exactly `substrate-claude-code`; requested +scopes are exactly `capture retrieve`. Redirects and origin overrides are rejected. The browser +URL includes the issued code. A bearer value is never accepted from environment, config, argv, +stdout, or a user paste. + +Until direct human device approval succeeds, hooks do not read or spool transcript content. +After approval, run `/substrate-setup` to make a separate approve/decline history decision. +Declining history still reaches authenticated `ready` and enables future capture and recall. +Claude Code has no reviewed safe history-discovery API in this release, so **history import is +unsupported**: approval records a preference but scans and uploads nothing. + +Credentials are scoped to the Claude profile. Linux Secret Service is preferred. macOS Keychain +reads/deletes but new non-interactive writes use the owner-private fallback. Windows fallback +requires a verified current-user-only ACL. Device and access secrets never enter ordinary state. + +## Hooks and lifecycle + +`hooks/hooks.json` uses the cross-platform Node launcher, not POSIX inline environment syntax. +Every command has a 15-second host bound, and the launcher gives hook Python a separate 10-second +process-tree deadline. Capture hooks perform only a small bounded local parse/admission and fsync a +private continuation job. They never perform OAuth or delivery. The long-lived MCP process drains +the spool and resumes jobs across MCP restart; there is no claim of portable out-of-process worker +supervision. API delivery uses a 6-second request timeout and an 8-second MCP drain budget. + +- `Stop` maps a completed turn to `turn`. +- `PreCompact` maps pre-compression to `pre_compress`. +- `SessionEnd` first admits any readable suffix, then admits one content-free `session_end`. +- `SessionStart` performs no recall and may print only the content-free explicit setup instruction. +- `UserPromptSubmit` supplies the actual prompt for bounded canonical-only recall. + +Claude Code exposes no reliable memory-write or session-switch hook. This adapter does not invent +one. Session identity and ancestry remain in each event scope. See `docs/lifecycle.md`. + +Capture is primary-only. Sidechain records and official subagent invocations with nonempty +`agent_id`, `agent_transcript_path`, or `SubagentStop` signals are excluded. Primary forks and +primary `--agent` sessions are not suppressed. Checkpoint cursor updates and the +one-shot SessionEnd marker happen only after every event is durably admitted. A cross-process +transaction serializes independent hook processes. Transcript byte cursors and private pending-job +records resume without a 2,000- or 8,000-record SessionEnd ceiling. Unreadable, malformed, or +oversized input never admits content, advances a cursor, or completes SessionEnd. Conservative +credential boundary quarantine remains visible only as a content-free loss count. + +## Strict remote contract + +Every capture, recall, and network tool fails closed unless capabilities include +`claude_code_memory` in `providers`, capture schema v2, exactly 262144 bytes, stream-v2 replay +flags/status, entity-wiki-v1, entity-quality-v2, canonical flags, and semantic version floors. +Automatic recall admits only canonical v2 `memory_card` values with a known entity type, valid +entity ID, and `entities//--<8hex>.md` path. It never falls back to a summary or +snippet. + +401/403 deletes invalid local auth, marks repair required, and retains queued events. Transient +failures and `Retry-After` schedule durable bounded retry with deterministic jitter. Status is +content-free and reports actual queue depth and persistent counters. -The plugin manifests provide the MCP server, lifecycle hooks, and slash commands. The MCP server -can also be run directly with `python -m claude_code_memory.server` when `src/` is on -`PYTHONPATH`. - -## Configuration +## MCP tools -Credentials are accepted from environment variables only: +The five bounded network operations are `substrate_search`, `substrate_read`, `substrate_query`, +`substrate_ingest`, and `substrate_job_status`. They retain the Hermes `/wiki/search`, read, query, +ingest, and job-status request shapes, including ingest `source_type` (`text` or `url`) and Hermes +input bounds. Prefixes avoid MCP namespace collision. Two Claude operational adapters are additional, +not network-parity claims: `substrate_sync` retries the durable queue and `substrate_status` +reports content-free local health. No `substrate_remember` tool is exposed because Claude Code +2.1.237 provides no host-originated direct-human write signal. -```text -SUBSTRATE_API_URL=https://your-substrate-server.example -SUBSTRATE_API_KEY=your-profile-scoped-bearer-key -``` +MCP stdout is strict JSON-RPC only. Capture hooks normally emit nothing. Prompt-bound recall can +emit only from the protocol-approved `UserPromptSubmit` hook. Human authorization instructions use stderr; CLI machine status +uses JSON stdout. -Set them in the environment that launches Claude Code. Never write `SUBSTRATE_API_KEY` to -`.mcp.json`, `hooks/hooks.json`, a project settings file, or another plaintext configuration file. -Run `substrate-claude-code configure` to print the required wiring without writing any secret. +## State, update, and removal -If the variables are absent or the API is temporarily unreachable, capture remains enabled and -new events are placed in the local spool beneath `~/.substrate/claude_code_memory/`. Set -`SUBSTRATE_STATE_HOME` to relocate the state root, which is particularly useful for isolated -tests. Sidechain capture is enabled by default. Set `SUBSTRATE_CAPTURE_SIDECHAINS=0` -to use the emergency exclusion kill-switch. +Profile state is below `~/.substrate/claude_code_memory/profiles//` unless an isolated +test relocates the state root. Code update and rollback preserve credentials, spool, cursors, and +consent state because they live outside the plugin directory. -## MCP tools +Use the generated installer's `--uninstall` operation shown above. It serializes with installs, +deregisters the user-scope plugin and marketplace, verifies both absent, and reports every retained +content-addressed source. It does not touch the plugin's external Substrate credentials, spool, or +checkpoints. Source deletion is the separately confirmed `--purge` form. Purge still does not touch +Substrate state and reports `server_revoked: false` because no server revocation API is implemented. -| Tool | Purpose | -|---|---| -| `substrate_search` | Search organizational memory and return cited memory cards. | -| `substrate_read` | Read one wiki page by repository-relative path. | -| `substrate_query` | Ask a cited question over Substrate memory. | -| `substrate_ingest` | Submit text for asynchronous wiki ingestion. | -| `substrate_remember` | Record a durable fact or decision requested by the user. | -| `substrate_sync` | Retry delivery of locally spooled capture events. | -| `substrate_status` | Show content-free configuration and spool counters. | - -The repository also adds `/substrate-status` and `/substrate-recall` slash commands. - -## Hook capture - -Claude Code invokes four command hooks from `hooks/hooks.json`: - -- **Stop** captures normalized user, assistant, tool-call, tool-result, and system blocks not - previously checkpointed and emits a `turn` event. -- **PreCompact** captures the same incremental transcript window as `pre_compress` before Claude - Code compacts its context. -- **SessionEnd** emits a content-free `session_end` event containing only the normalized message - count and session boundary. A persistent marker ensures it is emitted at most once. -- **SessionStart** performs a bounded Substrate search and prints a compact Markdown recall block - when relevant memory is available. - -Each hook is a fresh process, so per-session message digests and one-shot markers are persisted -under the plugin state directory. Capture events are durably spooled before network delivery. - -## Privacy boundary - -The transcript reader captures top-level and sidechain `user`, `assistant`, and `system` records. -Sidechain records carry record/block coordinates and session ancestry. Tool calls and results are -separate text-only messages paired by `tool_call_id`. A paired result receives its tool name; an -orphaned or ambiguous result receives a reason code and no source identity. - -For every captured block, full credential detection runs before the 65,536-byte UTF-8 ceiling. If -a recognized credential occurs anywhere in a block, the whole block becomes content-free. This -prevents a secret from being cut at a former truncation boundary. Binary and media bodies, hidden -reasoning, token usage, billing fields, and arbitrary provider metadata are not captured. - -The shared capture core redacts recognized secrets before persistence and transfer. Redaction is -defense in depth, not proof that arbitrary sensitive prose is absent. Visible prompts and -assistant output can themselves contain confidential material, so configure only a trusted -Substrate server and review its access and retention policy. Failed deliveries remain in a -bounded owner-private local spool. The spool reserves capacity for -boundary events and refuses newest events under pressure instead of evicting older evidence. -`substrate-claude-code status` exposes persistent `evicted`, `quarantined`, `dropped`, and -`duplicates` counters without exposing content. - -## Fail-open behavior - -Hooks always exit with status 0. API failures, malformed hook input, unreadable or truncated -transcripts, corrupt local state, and recall failures do not block or annotate a Claude Code -session. Normal capture hooks write nothing to stdout; optional diagnostics go only to stderr when -`SUBSTRATE_DEBUG` is set. Session-start recall is the sole intentional hook output. - -## Repository map - -- `src/claude_code_memory/` — Claude Code runtime, transcript reader, hooks, recall, MCP server, - and CLI. -- `src/substrate_capture/` — frozen shared capture core used by all host plugins. -- `.claude-plugin/` — plugin and self-listing marketplace metadata. -- `.mcp.json` — stdio MCP server registration. -- `hooks/hooks.json` — Claude Code lifecycle hook registration. -- `commands/` — `/substrate-status` and `/substrate-recall` command prompts. -- `tests/` — shared-core and Claude Code host contract tests with synthetic fixtures. -- `scripts/` — shared-core provenance and publication verification tooling. +`disconnect --confirm` +deletes local custody slots only and does not claim +server revocation. `purge-local-state --confirm --delete-queued-data` separately deletes local +credentials and queued state. Server revocation is not implemented. ## Development -```bash +```text python -m pip install -e '.[dev]' -python -m pytest tests/ -q ruff check . +ruff format --check . +python -m compileall -q src scripts +python -m pytest -q +python scripts/verify_public_plugin_candidate.py --root . ``` -Do not edit `src/substrate_capture/` directly. Its digests are shared across host repositories and -verified by the test suite. - -## License +Shared `src/substrate_capture/` changes are limited to reviewed security fixes; `_vendor.json` +records their exact hashes. All tests use mocks or synthetic content. No test requires a live credential or hosted call. -MIT © 2026 Sightline Technologies Inc. See [LICENSE](LICENSE). +MIT © 2026 Sightline Technologies Inc. diff --git a/SECURITY.md b/SECURITY.md index 82a8962..ca4e895 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,25 +2,19 @@ ## Supported versions -Supported versions are identified by published releases of Claude Code Substrate Memory. - -## Report a vulnerability - -Use GitHub's private **Security advisories → Report a vulnerability** flow for this repository. Do not open a public issue containing a credential, private history, server URL, spool contents, traceback with user content, or exploit details. - -Reports must use content-free diagnostics where possible: - -- plugin and Claude Code versions; -- operating system and Python version; -- affected lifecycle/tool operation; -- failure category; -- minimal synthetic reproduction; and -- expected and observed security boundary. - -## Security boundary - -The API key is read from the process environment ONLY and is never written to a config file. Host config files are world-readable plaintext. Reports go through GitHub private security advisories. Diagnostics are content-free. - -The plugin rejects unsafe redirects and unexpected response shapes, redacts before queueing, persistence, or network transfer, and uses bounded request, response, spool, and retry resources. - -Redaction cannot identify every sensitive statement. Operators remain responsible for choosing a trusted Substrate server and protecting local host state. +Only immutable published releases identify supported versions. This repository currently contains +a release candidate, not a published release. + +Report vulnerabilities through the repository's private GitHub Security Advisory flow. Use only +content-free diagnostics: plugin/Claude/Python/OS versions, lifecycle or tool, symbolic failure +category, and a synthetic reproduction. Never include a credential, private transcript, server +response body, local path, spool contents, or user prose. + +Normal setup accepts no environment/config/argv bearer key and no origin override. Hosted traffic +is pinned to `https://app.trysubstrate.co`, rejects redirects, and negotiates the complete strict +contract before capture, recall, or tools. Credentials use OS custody or a verified owner-private +fallback. Preapproval hooks do not read transcripts. + +Redaction cannot classify all confidential prose. Approved future capture is retained in an +owner-private bounded spool until delivery or separately confirmed deletion. Server revocation is +not implemented by this client. diff --git a/commands/substrate-recall.md b/commands/substrate-recall.md index e35d6e5..bb8951c 100644 --- a/commands/substrate-recall.md +++ b/commands/substrate-recall.md @@ -1,6 +1,7 @@ --- -description: Search Substrate organizational memory for relevant context +description: Search validated canonical Substrate memory on an explicit user request --- -Call `substrate_search` using `$ARGUMENTS` as the query. Present the most relevant memory cards -with their cited paths. If no query was supplied, ask what context to recall. +Use `substrate_search` with the user's current question. Cite only returned canonical entity paths +and v2 memory cards. Do not use summary or snippet fallback and do not claim automatic injection if +the strict capability gate fails. diff --git a/commands/substrate-setup.md b/commands/substrate-setup.md new file mode 100644 index 0000000..6d7b00c --- /dev/null +++ b/commands/substrate-setup.md @@ -0,0 +1,8 @@ +--- +description: Complete direct-human Substrate authorization and the separate history decision +--- + +Run the portable plugin launcher with `setup --wait`. Show stderr approval instructions directly +to the user. Do not ask for, accept, or paste a bearer key. After approval, ask the user to choose +`--history approve` or `--history decline`. Explain that this release imports no history under +either choice. Do not infer the decision. diff --git a/commands/substrate-status.md b/commands/substrate-status.md index 9db61c0..c54ed56 100644 --- a/commands/substrate-status.md +++ b/commands/substrate-status.md @@ -1,5 +1,6 @@ --- -description: Show Substrate memory configuration and local spool status +description: Show content-free Substrate onboarding and durable queue status --- -Call the `substrate_status` tool and summarize its content-free operational result. +Run the portable plugin launcher with `status`. Report only its JSON counts, booleans, phase, and +symbolic category. Never inspect or print queued event content or credential custody values. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..0f35fa4 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,24 @@ +# Architecture and safety invariants + +Hermes main `db4ddc2f093f833ffa62f4246860de96ba398713` and release source +`e4ad07cfc858618edfd69e1f3be5e8345b253037` are the behavioral authority. + +The host adapter layers fixed-origin onboarding, profile custody, strict capability validation, +primary-runtime filtering, cross-process cursor transactions, durable delivery scheduling, and +canonical recall around the narrowly security-hardened vendored `src/substrate_capture` package. + +A transcript transition is `read -> redact -> deterministic event -> fsync spool -> fsync cursor`. +An admission failure stops before cursor or SessionEnd marker update. Delivery uses a separate +inter-process lease. Auth and transient failures release, rather than quarantine, the claimed +item. Permanent event-specific failures quarantine with persistent counters. + +The tenant-local scope derives from the private Claude profile path through a one-way hash. Capture and recall share exact `user_id`, `agent_id`, and `agent_identity` values. The builder and +server independently derive the same subject from `platform + user_id`. Host session IDs provide session +lineage. No environment agent identifier can change it. + +The 15-second host process has a separate 10-second launcher timeout and performs only a strict +small local parse/admission budget. It never waits for OAuth or delivery. Ordered private jobs bind +the lifecycle event, target EOF, session, and source. The long-lived MCP process resumes them under +bounded locks and deletes each only after its final checkpoint or SessionEnd marker commits. + +Release installation is a separate private state machine below `CLAUDE_CONFIG_DIR`. Its states are `prepare -> uninstall -> marketplace -> register -> verify -> commit`. Every boundary is journaled and fsynced. The OS lock serializes processes. Recovery resumes the journal's one desired logical registration; installer state changes only after Claude reports the exact version, source, enabled state, marketplace, and MCP inventory. Rollback uses the same state machine with the prior snapshot as its desired registration. diff --git a/docs/lifecycle.md b/docs/lifecycle.md new file mode 100644 index 0000000..fa996de --- /dev/null +++ b/docs/lifecycle.md @@ -0,0 +1,25 @@ +# Claude lifecycle mapping + +| Claude signal | Capture meaning | Notes | +|---|---|---| +| `Stop` | `turn` | A short hook durably admits one bounded primary suffix window and leaves a continuation job when needed. | +| `PreCompact` | `pre_compress` | Same bounded admission before context compression. | +| `SessionEnd` | `session_end` | One host signal records the target EOF; MCP resumes until EOF, then admits one content-free boundary. | +| `SessionStart` | none | No generic recall. It can emit only a content-free `/substrate-setup` instruction. | +| `UserPromptSubmit` | recall injection | Uses only the documented current `prompt`; no transcript/project fallback. | + +Claude Code `2.1.237` exposes no verified direct-human memory-write or history-discovery hook. +`substrate_remember` is therefore not exposed and history import remains unsupported. + +Official subagents are rejected by nonempty `agent_id`, `agent_transcript_path`, `SubagentStop`, +and sidechain flags, including string booleans. Primary forks and primary `--agent` sessions remain +valid. Concurrent hooks serialize bounded read/admit/commit transactions. A byte cursor plus a +bounded preceding-byte anchor accepts safe append suffixes and detects rotation or same-inode +truncate/regrow. Unreadable, malformed, oversized, or truncated final input never advances. + +Hooks never start detached workers and never perform OAuth or delivery. The long-lived MCP process +owns spool drain, auth repair, and continuation. Jobs and checkpoints survive MCP kill/restart. A +SIGKILL after spool admission but before cursor commit retries the deterministic event. There is no +claim of portable supervision while MCP is not running. + +Hook Python is placed in a private POSIX process group. At the 10-second deadline the Node launcher sends group `SIGTERM`, waits a bounded 500 ms grace, then sends group `SIGKILL` and waits for its direct child exit before returning. A SIGTERM-ignoring grandchild is covered. Windows uses `taskkill /PID ... /T /F` and waits for the child exit; the cross-platform restart test does not describe this as Windows SIGKILL semantics. Normal MCP and setup commands have no hook deadline or detached process group. diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 0000000..43a3736 --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,25 @@ +# Deterministic protected-main release + +Authority: Hermes `db4ddc2f093f833ffa62f4246860de96ba398713` and released bytes source +`e4ad07cfc858618edfd69e1f3be5e8345b253037`. + +The release workflow takes an independently supplied 40-character source SHA. It requires that SHA +to equal checked-out protected `main`, refuses an existing tag, runs the scanner, Ruff, formatting, +compileall, complete tests, vendored-byte checks, and build provenance tests. Publication has an +explicit dependency on the Linux/macOS/Windows by Python 3.11/3.12 matrix. After any release +environment wait, it re-reads current remote `main` and protection before attestation and again +before tag creation. It builds into two fresh directories and compares every byte. Only the installer, archive, and `SHA256SUMS` may exist. +GitHub artifact attestations bind those bytes to the workflow and source SHA. + +Local reproduction after committing a clean tree: + +```text +python scripts/build_release.py --source-commit "$(git rev-parse HEAD)" --output /tmp/build-a +python scripts/build_release.py --source-commit "$(git rev-parse HEAD)" --output /tmp/build-b +cmp /tmp/build-a/claude_code_substrate_memory.zip /tmp/build-b/claude_code_substrate_memory.zip +cmp /tmp/build-a/install_claude_plugin.py /tmp/build-b/install_claude_plugin.py +cmp /tmp/build-a/SHA256SUMS /tmp/build-b/SHA256SUMS +``` + +Publishing, protection, server registration, and real-host acceptance require humans. The builder +and workflow do not weaken these gates. diff --git a/docs/server-integration.md b/docs/server-integration.md new file mode 100644 index 0000000..8b5bd87 --- /dev/null +++ b/docs/server-integration.md @@ -0,0 +1,14 @@ +# Separate server and human release gates + +This client is complete only when a separately reviewed hosted-server change: + +1. registers public RFC 8628 client `substrate-claude-code`; +2. binds both device issue and token exchange to that exact client; +3. permits scopes exactly `capture retrieve` and rejects all others; +4. advertises `claude_code_memory` in `providers` with the complete shared strict contract; +5. updates stale Hermes contract metadata to the current Hermes v2.0.3 device flow; and +6. updates Claude metadata from manual bearer setup to fixed-origin device authorization. + +Server changes do not belong in this plugin repository. Client mocks prove the expected request and +response contract but do not prove registration. Release must remain blocked until an integration +test passes against a non-production server. Protected `main` is also a repository-admin gate. diff --git a/hooks/hooks.json b/hooks/hooks.json index a9be5af..e25ad32 100644 --- a/hooks/hooks.json +++ b/hooks/hooks.json @@ -6,7 +6,7 @@ "hooks": [ { "type": "command", - "command": "PYTHONPATH=\"${CLAUDE_PLUGIN_ROOT}/src\" python -m claude_code_memory.hook stop", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/plugin_runtime.cjs\" hook stop", "timeout": 15 } ] @@ -18,7 +18,7 @@ "hooks": [ { "type": "command", - "command": "PYTHONPATH=\"${CLAUDE_PLUGIN_ROOT}/src\" python -m claude_code_memory.hook pre-compact", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/plugin_runtime.cjs\" hook pre-compact", "timeout": 15 } ] @@ -30,7 +30,7 @@ "hooks": [ { "type": "command", - "command": "PYTHONPATH=\"${CLAUDE_PLUGIN_ROOT}/src\" python -m claude_code_memory.hook session-end", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/plugin_runtime.cjs\" hook session-end", "timeout": 15 } ] @@ -42,7 +42,19 @@ "hooks": [ { "type": "command", - "command": "PYTHONPATH=\"${CLAUDE_PLUGIN_ROOT}/src\" python -m claude_code_memory.hook session-start", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/plugin_runtime.cjs\" hook session-start", + "timeout": 15 + } + ] + } + ], + "UserPromptSubmit": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/plugin_runtime.cjs\" hook user-prompt-submit", "timeout": 15 } ] diff --git a/pyproject.toml b/pyproject.toml index 8edece8..9443ddf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "claude-code-substrate-memory" -version = "0.1.0" -description = "Substrate organizational memory for Claude Code: MCP tools plus bidirectional session capture." +version = "0.2.0" +description = "Hermes-authoritative hosted Substrate memory for Claude Code." readme = "README.md" requires-python = ">=3.11" license = { file = "LICENSE" } @@ -47,3 +47,4 @@ timeout = 60 [tool.ruff] target-version = "py311" line-length = 100 +extend-exclude = ["src/substrate_capture"] diff --git a/scripts/build_release.py b/scripts/build_release.py new file mode 100755 index 0000000..a3166b5 --- /dev/null +++ b/scripts/build_release.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Build the deterministic three-file Claude plugin release candidate.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +ARCHIVE_NAME = "claude_code_substrate_memory.zip" +INSTALLER_NAME = "install_claude_plugin.py" +CHECKSUM_NAME = "SHA256SUMS" +ARCHIVE_ROOT = "claude_code_substrate_memory" +VERSION = "0.2.0" +PROVIDER_ID = "claude_code_memory" +HERMES_AUTHORITY_MAIN = "db4ddc2f093f833ffa62f4246860de96ba398713" +HERMES_AUTHORITY_RELEASE = "e4ad07cfc858618edfd69e1f3be5e8345b253037" +FILES = ( + ".claude-plugin/marketplace.json", + ".claude-plugin/plugin.json", + ".mcp.json", + "BOUNDARY.md", + "CHANGELOG.md", + "COMPATIBILITY.md", + "LICENSE", + "README.md", + "SECURITY.md", + "commands/substrate-recall.md", + "commands/substrate-setup.md", + "commands/substrate-status.md", + "docs/architecture.md", + "docs/lifecycle.md", + "docs/releasing.md", + "docs/server-integration.md", + "hooks/hooks.json", + "scripts/plugin_runtime.cjs", +) + + +def digest_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _git(*args: str) -> str: + return subprocess.run( + ("git", "-C", str(ROOT), *args), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + +def _git_bytes(*args: str) -> bytes: + return subprocess.run( + ("git", "-C", str(ROOT), *args), + capture_output=True, + check=True, + ).stdout + + +def _source_files(source_commit: str) -> list[str]: + tracked = set(_git("ls-tree", "-r", "--name-only", source_commit).splitlines()) + paths = set(FILES) + paths.update( + path + for path in tracked + if path.startswith("src/claude_code_memory/") or path.startswith("src/substrate_capture/") + ) + if not paths.issubset(tracked): + raise ValueError("release input missing from source commit") + return sorted(paths) + + +def _zip_bytes(source_commit: str) -> bytes: + files = _source_files(source_commit) + payloads = {path: _git_bytes("show", f"{source_commit}:{path}") for path in files} + provenance = { + "format": 1, + "plugin_version": VERSION, + "provider_id": PROVIDER_ID, + "source_commit": source_commit, + "hermes_authority_main": HERMES_AUTHORITY_MAIN, + "hermes_authority_release": HERMES_AUTHORITY_RELEASE, + "files": {path: digest_bytes(payloads[path]) for path in sorted(payloads)}, + } + with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as temporary: + temporary_path = Path(temporary.name) + try: + with zipfile.ZipFile(temporary_path, "w", compression=zipfile.ZIP_STORED) as archive: + for path, value in [ + *sorted(payloads.items()), + ( + "PROVENANCE.json", + ( + json.dumps(provenance, indent=2, sort_keys=True, allow_nan=False) + "\n" + ).encode("utf-8"), + ), + ]: + info = zipfile.ZipInfo(f"{ARCHIVE_ROOT}/{path}", (1980, 1, 1, 0, 0, 0)) + mode = 0o755 if path == "scripts/plugin_runtime.cjs" else 0o644 + info.external_attr = (stat.S_IFREG | mode) << 16 + info.compress_type = zipfile.ZIP_STORED + info.create_system = 3 + archive.writestr(info, value) + return temporary_path.read_bytes() + finally: + temporary_path.unlink(missing_ok=True) + + +def build(output: Path, source_commit: str) -> dict[str, str]: + if not re.fullmatch(r"[0-9a-f]{40}", source_commit): + raise ValueError("source commit must be exact lowercase SHA") + if _git("rev-parse", "HEAD") != source_commit: + raise ValueError("source commit does not match HEAD") + if _git("status", "--porcelain"): + raise ValueError("release source tree must be clean") + absolute_output = Path(os.path.abspath(os.fspath(output.expanduser()))) + for candidate in [absolute_output, *absolute_output.parents]: + if candidate.exists() and candidate.is_symlink(): + raise ValueError("release output ancestry must not contain symlinks") + absolute_output.mkdir(parents=True, exist_ok=True) + output = absolute_output + for item in output.iterdir(): + if item.is_symlink(): + raise ValueError("release output must not contain symlinks") + if item.is_file(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) + else: + raise ValueError("unsafe release output entry") + archive = _zip_bytes(source_commit) + archive_hash = digest_bytes(archive) + template = _git_bytes("show", f"{source_commit}:scripts/install_release.py").decode("utf-8") + installer = ( + template.replace("@ARCHIVE_SHA256@", archive_hash) + .replace("@SOURCE_COMMIT@", source_commit) + .replace("@PLUGIN_VERSION@", VERSION) + .encode("utf-8") + ) + installer_hash = digest_bytes(installer) + checksums = (f"{installer_hash} {INSTALLER_NAME}\n{archive_hash} {ARCHIVE_NAME}\n").encode( + "ascii" + ) + (output / ARCHIVE_NAME).write_bytes(archive) + (output / INSTALLER_NAME).write_bytes(installer) + os.chmod(output / INSTALLER_NAME, 0o755) + (output / CHECKSUM_NAME).write_bytes(checksums) + return { + INSTALLER_NAME: installer_hash, + ARCHIVE_NAME: archive_hash, + CHECKSUM_NAME: digest_bytes(checksums), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source-commit", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + hashes = build(args.output, args.source_commit) + except (OSError, ValueError, subprocess.CalledProcessError): + print("release build failed") + return 2 + print(json.dumps(hashes, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/install_release.py b/scripts/install_release.py new file mode 100755 index 0000000..21717bf --- /dev/null +++ b/scripts/install_release.py @@ -0,0 +1,890 @@ +#!/usr/bin/env python3 +"""Generated, recoverable Claude Code plugin release installer.""" + +from __future__ import annotations + +import argparse +import contextlib +import hashlib +import io +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +import time +import uuid +import zipfile +from pathlib import Path +from typing import Any, Iterator + +EXPECTED_ARCHIVE_SHA256 = "@ARCHIVE_SHA256@" +EXPECTED_SOURCE_COMMIT = "@SOURCE_COMMIT@" +PLUGIN_VERSION = "@PLUGIN_VERSION@" +SUPPORTED_CLAUDE_CODE_VERSIONS = ("2.1.237",) +ARCHIVE_ROOT = "claude_code_substrate_memory" +MARKETPLACE_NAME = "claude-code-substrate-memory" +PLUGIN_NAME = "claude-code-substrate-memory" +PLUGIN_ID = f"{PLUGIN_NAME}@{MARKETPLACE_NAME}" +JOURNAL_FORMAT = 1 +EXPECTED_MCP = { + "substrate": { + "type": "stdio", + "command": "node", + "args": ["${CLAUDE_PLUGIN_ROOT}/scripts/plugin_runtime.cjs", "mcp"], + } +} +MCP_RECORD_PREFIX = f"plugin:{PLUGIN_NAME}:substrate:" +MCP_CONNECTED_LINE = re.compile(rf"^{re.escape(MCP_RECORD_PREFIX)} .+ - ✔ Connected$") + + +def _unresolved_absolute(path: Path) -> Path: + return Path(os.path.abspath(os.fspath(path.expanduser()))) + + +def _config_dir() -> Path: + configured = os.environ.get("CLAUDE_CONFIG_DIR", "").strip() + return _unresolved_absolute(Path(configured) if configured else Path.home() / ".claude") + + +def _transaction_root() -> Path: + return _config_dir() / "plugins" / "substrate-installer" + + +def _release_source() -> Path: + return ( + _transaction_root() / "releases" / f"{PLUGIN_VERSION}-{EXPECTED_SOURCE_COMMIT}" / "source" + ) + + +def _claude_environment() -> dict[str, str]: + environment = os.environ.copy() + environment["CLAUDE_CONFIG_DIR"] = str(_config_dir()) + return environment + + +def digest(path: Path) -> str: + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def _fsync_directory(path: Path) -> None: + if os.name != "posix": + return + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _safe_directory(path: Path, label: str, *, create: bool = False) -> Path: + """Check every component without resolving it or following symlinks.""" + absolute = _unresolved_absolute(path) + current = Path(absolute.anchor) + for part in absolute.parts[1:]: + current = current / part + try: + info = current.stat(follow_symlinks=False) + except FileNotFoundError: + if not create: + raise ValueError(f"{label} is missing") from None + try: + current.mkdir(mode=0o700) + except FileExistsError: + pass + info = current.stat(follow_symlinks=False) + if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode): + raise ValueError(f"{label} must not contain a symlink") + if os.name == "posix": + getuid = getattr(os, "getuid", None) + info = absolute.stat(follow_symlinks=False) + if not callable(getuid) or info.st_uid != getuid(): + raise ValueError(f"{label} must be owned by the current user") + os.chmod(absolute, 0o700) + return absolute + + +def _safe_parent(path: Path, label: str, *, create: bool = False) -> tuple[Path, Path]: + absolute = _unresolved_absolute(path) + parent = _safe_directory(absolute.parent, label, create=create) + try: + info = absolute.stat(follow_symlinks=False) + except FileNotFoundError: + info = None + if info is not None and stat.S_ISLNK(info.st_mode): + raise ValueError(f"{label} must not be a symlink") + return absolute, parent + + +def _assert_regular(path: Path, label: str, *, maximum: int) -> os.stat_result: + absolute, _parent = _safe_parent(path, label) + info = absolute.stat(follow_symlinks=False) + if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode) or info.st_size > maximum: + raise ValueError(f"unsafe {label}") + return info + + +def _open_nofollow(absolute: Path, flags: int) -> int: + """Open through held POSIX directory descriptors to defeat ancestor swaps.""" + if os.name != "posix": + return os.open(absolute, flags) + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | os.O_NOFOLLOW + directory = os.open(absolute.anchor, directory_flags) + try: + for part in absolute.parent.parts[1:]: + following = os.open(part, directory_flags, dir_fd=directory) + os.close(directory) + directory = following + return os.open(absolute.name, flags | os.O_NOFOLLOW, dir_fd=directory) + finally: + os.close(directory) + + +def _read_regular_once(path: Path, label: str, *, maximum: int) -> bytes: + """Open a regular file once with no-follow checks and stable identity.""" + absolute, _parent = _safe_parent(path, label) + before = _assert_regular(absolute, label, maximum=maximum) + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = _open_nofollow(absolute, flags) + try: + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_size > maximum + or (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino) + ): + raise ValueError(f"unsafe {label}") + chunks: list[bytes] = [] + remaining = maximum + 1 + while remaining: + chunk = os.read(descriptor, min(1024 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + value = b"".join(chunks) + if len(value) != opened.st_size or len(value) > maximum: + raise ValueError(f"unsafe {label}") + after = absolute.stat(follow_symlinks=False) + if (after.st_dev, after.st_ino, after.st_size) != ( + opened.st_dev, + opened.st_ino, + opened.st_size, + ): + raise ValueError(f"{label} changed while being read") + return value + finally: + os.close(descriptor) + + +def _safe_members(archive: zipfile.ZipFile) -> list[zipfile.ZipInfo]: + members = archive.infolist() + if not members or len(members) > 4096: + raise ValueError("invalid archive member count") + seen: set[str] = set() + total_size = 0 + for member in members: + name = member.filename + parts = Path(name).parts + mode = member.external_attr >> 16 + if ( + not name + or name in seen + or name.startswith(("/", "\\")) + or "\\" in name + or not parts + or parts[0] != ARCHIVE_ROOT + or any(part in {"", ".", ".."} for part in parts) + or not stat.S_ISREG(mode) + or member.flag_bits & 0x1 + or member.file_size > 4 * 1024 * 1024 + ): + raise ValueError("unsafe archive member") + total_size += member.file_size + if total_size > 32 * 1024 * 1024: + raise ValueError("archive is too large") + seen.add(name) + return members + + +def _verify_provenance(root: Path) -> None: + _safe_directory(root, "release source") + path = root / "PROVENANCE.json" + value = json.loads(_read_regular_once(path, "provenance", maximum=1024 * 1024)) + if not isinstance(value, dict): + raise ValueError("invalid provenance") + if ( + value.get("plugin_version") != PLUGIN_VERSION + or value.get("provider_id") != "claude_code_memory" + or value.get("source_commit") != EXPECTED_SOURCE_COMMIT + ): + raise ValueError("provenance identity mismatch") + expected = value.get("files") + if not isinstance(expected, dict): + raise ValueError("missing provenance file closure") + actual: dict[str, str] = {} + for item in sorted(root.rglob("*")): + info = item.stat(follow_symlinks=False) + if stat.S_ISLNK(info.st_mode): + raise ValueError("release source contains a symlink") + if stat.S_ISREG(info.st_mode): + if item.name != "PROVENANCE.json": + actual[item.relative_to(root).as_posix()] = digest(item) + elif not stat.S_ISDIR(info.st_mode): + raise ValueError("release source contains an unsafe entry") + if actual != expected: + raise ValueError("provenance file closure mismatch") + + +def _source_version(root: Path) -> str: + _safe_directory(root, "marketplace source") + plugin = json.loads( + _read_regular_once( + root / ".claude-plugin" / "plugin.json", "plugin manifest", maximum=1024 * 1024 + ) + ) + market = json.loads( + _read_regular_once( + root / ".claude-plugin" / "marketplace.json", + "marketplace manifest", + maximum=1024 * 1024, + ) + ) + entries = market.get("plugins") if isinstance(market, dict) else None + versions = [ + item.get("version") + for item in entries or [] + if isinstance(item, dict) and item.get("name") == PLUGIN_NAME + ] + version = plugin.get("version") if isinstance(plugin, dict) else None + if not isinstance(version, str) or versions != [version]: + raise ValueError("marketplace source version mismatch") + return version + + +def _harden_tree(root: Path) -> None: + for path in [root, *sorted(root.rglob("*"))]: + info = path.stat(follow_symlinks=False) + if stat.S_ISLNK(info.st_mode): + raise ValueError("installed tree must not contain symlinks") + if os.name == "posix": + os.chmod(path, 0o700 if stat.S_ISDIR(info.st_mode) else 0o600) + if os.name == "posix": + os.chmod(root / "scripts" / "plugin_runtime.cjs", 0o700) + + +def _freeze_tree(root: Path) -> None: + if os.name != "posix": + return + for path in sorted(root.rglob("*"), reverse=True): + info = path.stat(follow_symlinks=False) + os.chmod(path, 0o555 if stat.S_ISDIR(info.st_mode) else 0o444) + os.chmod(root / "scripts" / "plugin_runtime.cjs", 0o555) + os.chmod(root, 0o555) + + +def _safe_rmtree(path: Path) -> None: + try: + _safe_directory(path, "removal target") + except ValueError: + if not path.exists() and not path.is_symlink(): + return + raise + if os.name == "posix": + for item in [path, *path.rglob("*")]: + if not item.is_symlink(): + os.chmod(item, 0o700 if item.is_dir() else 0o600) + shutil.rmtree(path) + + +def _atomic_json(path: Path, value: dict[str, Any]) -> None: + path, parent = _safe_parent(path, "installer state", create=True) + encoded = ( + json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n" + ).encode() + temporary = parent / f".{path.name}.{uuid.uuid4().hex}.tmp" + descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + try: + with os.fdopen(descriptor, "wb", closefd=False) as stream: + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + finally: + os.close(descriptor) + os.replace(temporary, path) + _fsync_directory(parent) + + +def _load_json(path: Path, label: str, *, maximum: int = 1024 * 1024) -> dict[str, Any] | None: + try: + value = json.loads(_read_regular_once(path, label, maximum=maximum)) + except FileNotFoundError: + return None + if not isinstance(value, dict): + raise ValueError(f"invalid {label}") + return value + + +def _unlink_state(path: Path) -> None: + path, parent = _safe_parent(path, "installer state") + if path.exists(): + info = path.stat(follow_symlinks=False) + if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode): + raise ValueError("unsafe installer state") + path.unlink() + _fsync_directory(parent) + + +@contextlib.contextmanager +def _transaction_lock(timeout: float = 10.0) -> Iterator[None]: + root = _safe_directory(_transaction_root(), "installer transaction root", create=True) + path = root / "transaction.lock" + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags, 0o600) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode): + raise ValueError("unsafe installer lock") + if os.name == "nt": + import msvcrt + + if info.st_size == 0: + os.write(descriptor, b"0") + os.fsync(descriptor) + deadline = time.monotonic() + timeout + while True: + try: + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1) + break + except OSError: + if time.monotonic() >= deadline: + raise TimeoutError("installer lock is busy") from None + time.sleep(0.05) + try: + yield + finally: + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + else: + import fcntl + + deadline = time.monotonic() + timeout + while True: + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if time.monotonic() >= deadline: + raise TimeoutError("installer lock is busy") from None + time.sleep(0.05) + try: + yield + finally: + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) + + +def _run_claude_raw( + executable: str, *arguments: str, timeout: int = 30 +) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + (executable, *arguments), + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + timeout=timeout, + check=False, + env=_claude_environment(), + ) + if result.returncode != 0: + raise ValueError("Claude Code plugin command failed") + return result + + +def _run_claude( + executable: str, *arguments: str, json_output: bool = False, timeout: int = 30 +) -> Any: + result = _run_claude_raw(executable, *arguments, timeout=timeout) + if not json_output: + return result.stdout + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + raise ValueError("Claude Code plugin discovery returned invalid JSON") from None + + +def _claude_version(executable: str) -> str: + result = _run_claude_raw(executable, "--version", timeout=10) + match = re.search(r"(? list[dict[str, Any]]: + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + if isinstance(value, dict) and isinstance(value.get("plugins"), list): + return [item for item in value["plugins"] if isinstance(item, dict)] + return [] + + +def _plugin_record(value: Any) -> dict[str, Any] | None: + matches = [] + for item in _plugin_records(value): + identity = item.get("id") or item.get("pluginId") + if identity == PLUGIN_ID: + matches.append(item) + elif item.get("name") == PLUGIN_NAME and item.get("marketplace") == MARKETPLACE_NAME: + matches.append(item) + if len(matches) > 1: + raise ValueError("multiple active plugin registrations") + return matches[0] if matches else None + + +def _known_marketplace() -> dict[str, Any] | None: + value = _load_json( + _config_dir() / "plugins" / "known_marketplaces.json", + "marketplace registry", + ) + if value is None: + return None + item = value.get(MARKETPLACE_NAME) + return item if isinstance(item, dict) else None + + +def _marketplace_path(item: dict[str, Any]) -> Path | None: + source = item.get("source") + candidate = source.get("path") if isinstance(source, dict) else None + return ( + _unresolved_absolute(Path(candidate)) if isinstance(candidate, str) and candidate else None + ) + + +def _snapshot(executable: str) -> dict[str, Any]: + record = _plugin_record(_run_claude(executable, "plugin", "list", "--json", json_output=True)) + marketplace = _known_marketplace() + source = _marketplace_path(marketplace) if marketplace is not None else None + if source is not None: + _safe_directory(source, "registered marketplace source") + return { + "present": record is not None, + "version": str(record.get("version")) if record is not None else None, + "enabled": bool(record.get("enabled")) if record is not None else False, + "marketplace_present": marketplace is not None, + "source": str(source) if source is not None else None, + "mcp": record.get("mcpServers") if record is not None else None, + } + + +def _desired_release(source: Path) -> dict[str, Any]: + return { + "present": True, + "version": PLUGIN_VERSION, + "enabled": True, + "marketplace_present": True, + "source": str(_unresolved_absolute(source)), + "mcp": EXPECTED_MCP, + } + + +def _desired_absent() -> dict[str, Any]: + return { + "present": False, + "version": None, + "enabled": False, + "marketplace_present": False, + "source": None, + "mcp": None, + } + + +def _logical_equal(actual: dict[str, Any], desired: dict[str, Any]) -> bool: + keys = ("present", "version", "enabled", "marketplace_present", "source", "mcp") + return all(actual.get(key) == desired.get(key) for key in keys) + + +def _verify(executable: str, desired: dict[str, Any], *, activation: bool = True) -> dict[str, Any]: + current = _snapshot(executable) + if not _logical_equal(current, desired): + raise ValueError("Claude Code registration does not match the transaction") + source_value = desired.get("source") + if desired.get("marketplace_present"): + if not isinstance(source_value, str): + raise ValueError("missing marketplace source") + version = _source_version(Path(source_value)) + if desired.get("present") and version != desired.get("version"): + raise ValueError("registered source version mismatch") + if desired.get("present"): + _run_claude(executable, "plugin", "details", PLUGIN_ID) + if activation and desired.get("enabled"): + output = _run_claude(executable, "mcp", "list", timeout=30) + lines = [ + line for line in str(output).splitlines() if line.startswith(MCP_RECORD_PREFIX) + ] + if len(lines) != 1 or MCP_CONNECTED_LINE.fullmatch(lines[0]) is None: + raise ValueError("Claude Code did not activate the exact MCP server") + return current + + +def _prepare_source(archive_bytes: bytes, target: Path) -> None: + releases = _safe_directory(target.parent.parent, "installer releases", create=True) + target_parent = _safe_directory(target.parent, "release directory", create=True) + try: + _verify_provenance(target) + return + except FileNotFoundError: + pass + except (OSError, ValueError, json.JSONDecodeError): + if target.exists() or target.is_symlink(): + _safe_rmtree(target) + with tempfile.TemporaryDirectory(prefix=".prepare-", dir=releases) as temporary_name: + temporary = Path(temporary_name) + with zipfile.ZipFile(io.BytesIO(archive_bytes), "r") as archive: + members = _safe_members(archive) + archive.extractall(temporary, members) + candidate = temporary / ARCHIVE_ROOT + _verify_provenance(candidate) + _harden_tree(candidate) + if target.exists() or target.is_symlink(): + _safe_rmtree(target) + os.replace(candidate, target) + _freeze_tree(target) + _fsync_directory(target_parent) + _verify_provenance(target) + + +def _phase(journal: dict[str, Any], phase: str) -> None: + journal["phase"] = phase + _atomic_json(_transaction_root() / "journal.json", journal) + if os.environ.get("SUBSTRATE_INSTALLER_TEST_CRASH_AFTER") == phase: + os._exit(86) + + +def _normalize_to(executable: str, desired: dict[str, Any], journal: dict[str, Any]) -> None: + phase = journal.get("phase") + phases = ["prepare", "uninstall", "marketplace", "register", "verify", "commit"] + if phase not in phases: + raise ValueError("invalid installer journal phase") + + if phases.index(phase) <= phases.index("prepare"): + current = _snapshot(executable) + if current["present"]: + _run_claude(executable, "plugin", "uninstall", PLUGIN_ID, "--scope", "user") + _phase(journal, "uninstall") + phase = "uninstall" + + if phases.index(phase) <= phases.index("uninstall"): + current = _snapshot(executable) + if current["marketplace_present"]: + _run_claude( + executable, + "plugin", + "marketplace", + "remove", + MARKETPLACE_NAME, + "--scope", + "user", + ) + if desired.get("marketplace_present"): + source = desired.get("source") + if not isinstance(source, str): + raise ValueError("missing desired marketplace source") + _run_claude( + executable, + "plugin", + "marketplace", + "add", + source, + "--scope", + "user", + ) + _phase(journal, "marketplace") + phase = "marketplace" + + if phases.index(phase) <= phases.index("marketplace"): + if desired.get("present"): + current = _snapshot(executable) + if not current["present"]: + _run_claude(executable, "plugin", "install", PLUGIN_ID, "--scope", "user") + current = _snapshot(executable) + if desired.get("enabled") and not current["enabled"]: + _run_claude(executable, "plugin", "enable", PLUGIN_ID, "--scope", "user") + elif not desired.get("enabled") and current["enabled"]: + _run_claude(executable, "plugin", "disable", PLUGIN_ID, "--scope", "user") + _phase(journal, "register") + phase = "register" + + if phases.index(phase) <= phases.index("register"): + _verify(executable, desired) + _phase(journal, "verify") + phase = "verify" + + if phases.index(phase) <= phases.index("verify"): + state = { + "format": JOURNAL_FORMAT, + "active": desired, + "previous": journal.get("previous"), + "operation": journal.get("operation"), + } + _atomic_json(_transaction_root() / "state.json", state) + _phase(journal, "commit") + phase = "commit" + + if phase == "commit": + _verify(executable, desired, activation=False) + _unlink_state(_transaction_root() / "journal.json") + + +def _recover(executable: str) -> str | None: + journal = _load_json(_transaction_root() / "journal.json", "installer journal") + if journal is None: + return None + if journal.get("format") != JOURNAL_FORMAT or not isinstance(journal.get("desired"), dict): + raise ValueError("invalid installer journal") + _normalize_to(executable, journal["desired"], journal) + return str(journal.get("operation") or "unknown") + + +def _start_transaction( + executable: str, + operation: str, + desired: dict[str, Any], + previous: dict[str, Any], +) -> None: + journal = { + "format": JOURNAL_FORMAT, + "operation": operation, + "phase": "prepare", + "desired": desired, + "previous": previous, + } + _phase(journal, "prepare") + _normalize_to(executable, desired, journal) + + +def install(archive_path: Path, claude: str) -> dict[str, Any]: + archive_bytes = _read_regular_once(archive_path, "release archive", maximum=64 * 1024 * 1024) + if ( + EXPECTED_ARCHIVE_SHA256.startswith("@") + or hashlib.sha256(archive_bytes).hexdigest() != EXPECTED_ARCHIVE_SHA256 + ): + raise ValueError("archive digest mismatch") + host_version = _claude_version(claude) + target = _release_source() + recovered = _recover(claude) + current = _snapshot(claude) + desired = _desired_release(target) + state = _load_json(_transaction_root() / "state.json", "installer state") + if target.exists(): + try: + _verify_provenance(target) + except (OSError, ValueError, json.JSONDecodeError): + pass + else: + if ( + _logical_equal(current, desired) + and state is not None + and state.get("active") == desired + ): + try: + _verify(claude, desired) + except (OSError, ValueError, json.JSONDecodeError, subprocess.SubprocessError): + # The registration bytes can be healthy while Claude's live MCP + # activation is dead. Fall through to the journaled repair path. + pass + else: + return { + "installed": True, + "already_installed": True, + "registered": True, + "enabled": True, + "plugin_version": PLUGIN_VERSION, + "source_commit": EXPECTED_SOURCE_COMMIT, + "claude_code_version": host_version, + "journal": "clean", + "recovered": recovered, + } + _prepare_source(archive_bytes, target) + _run_claude(claude, "plugin", "validate", str(target)) + current = _snapshot(claude) + _start_transaction(claude, "install" if not current["present"] else "update", desired, current) + _verify(claude, desired, activation=False) + return { + "installed": True, + "already_installed": False, + "registered": True, + "enabled": True, + "plugin_version": PLUGIN_VERSION, + "source_commit": EXPECTED_SOURCE_COMMIT, + "claude_code_version": host_version, + "journal": "clean", + "recovered": recovered, + "rollback_available": current["present"], + } + + +def rollback(claude: str) -> dict[str, Any]: + host_version = _claude_version(claude) + recovered = _recover(claude) + if recovered == "rollback": + verified = _snapshot(claude) + return { + "rolled_back": True, + "registered": verified["present"], + "enabled": verified["enabled"], + "plugin_version": verified["version"], + "source": verified["source"], + "claude_code_version": host_version, + "journal": "clean", + "recovered": recovered, + } + state = _load_json(_transaction_root() / "state.json", "installer state") + if state is None or not isinstance(state.get("previous"), dict): + raise ValueError("rollback unavailable") + desired = state["previous"] + if desired.get("present"): + source = desired.get("source") + if not isinstance(source, str): + raise ValueError("rollback source unavailable") + _source_version(Path(source)) + current = _snapshot(claude) + _start_transaction(claude, "rollback", desired, current) + verified = _verify(claude, desired, activation=False) + return { + "rolled_back": True, + "registered": verified["present"], + "enabled": verified["enabled"], + "plugin_version": verified["version"], + "source": verified["source"], + "claude_code_version": host_version, + "journal": "clean", + "recovered": recovered, + } + + +def _remaining_sources() -> list[str]: + releases = _transaction_root() / "releases" + try: + _safe_directory(releases, "installer releases") + except ValueError: + if not releases.exists(): + return [] + raise + return [ + str(path) + for path in sorted(releases.glob("*/source")) + if path.is_dir() and not path.is_symlink() + ] + + +def _remaining_claude_cache() -> list[str]: + root = _config_dir() / "plugins" / "cache" / MARKETPLACE_NAME / PLUGIN_NAME + try: + _safe_directory(root, "Claude plugin cache") + except ValueError: + if not root.exists() and not root.is_symlink(): + return [] + raise + remaining: list[str] = [] + for path in sorted(root.iterdir()): + info = path.stat(follow_symlinks=False) + if stat.S_ISLNK(info.st_mode): + raise ValueError("Claude plugin cache contains a symlink") + if stat.S_ISDIR(info.st_mode): + remaining.append(str(path)) + return remaining + + +def uninstall(claude: str, *, purge: bool = False, confirmation: str = "") -> dict[str, Any]: + if purge and confirmation != PLUGIN_ID: + raise ValueError("purge confirmation does not match plugin id") + host_version = _claude_version(claude) + recovered = _recover(claude) + current = _snapshot(claude) + desired = _desired_absent() + if not _logical_equal(current, desired): + _start_transaction(claude, "uninstall", desired, current) + else: + _verify(claude, desired, activation=False) + if purge: + releases = _transaction_root() / "releases" + if releases.exists(): + _safe_rmtree(releases) + _safe_directory(releases, "installer releases", create=True) + remaining = _remaining_sources() + return { + "uninstalled": True, + "registered": False, + "marketplace_registered": False, + "claude_code_version": host_version, + "journal": "clean", + "recovered": recovered, + "purged_release_sources": purge, + "remaining_release_sources": remaining, + "remaining_claude_cache_paths": _remaining_claude_cache(), + "installer_state_retained": True, + "active_registration": "absent", + "external_substrate_state": "preserved", + "server_revoked": False, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("archive", nargs="?", type=Path) + parser.add_argument("--claude", default="claude") + actions = parser.add_mutually_exclusive_group() + actions.add_argument("--rollback", action="store_true") + actions.add_argument("--uninstall", action="store_true") + parser.add_argument("--purge", action="store_true") + parser.add_argument("--confirm-purge", default="") + args = parser.parse_args(argv) + try: + with _transaction_lock(): + if args.rollback: + if args.archive is not None or args.purge: + raise ValueError("rollback does not accept an archive or purge") + result = rollback(args.claude) + elif args.uninstall: + if args.archive is not None: + raise ValueError("uninstall does not accept an archive") + result = uninstall( + args.claude, + purge=args.purge, + confirmation=args.confirm_purge, + ) + else: + if args.archive is None or args.purge: + parser.error("archive is required") + result = install(_unresolved_absolute(args.archive), args.claude) + except ( + OSError, + ValueError, + TimeoutError, + zipfile.BadZipFile, + json.JSONDecodeError, + subprocess.SubprocessError, + ): + print(json.dumps({"error": "installation_failed"}, sort_keys=True)) + return 2 + print(json.dumps(result, sort_keys=True, allow_nan=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/plugin_runtime.cjs b/scripts/plugin_runtime.cjs new file mode 100755 index 0000000..fc2d81d --- /dev/null +++ b/scripts/plugin_runtime.cjs @@ -0,0 +1,215 @@ +#!/usr/bin/env node +"use strict"; + +const { spawn, spawnSync } = require("node:child_process"); +const path = require("node:path"); + +const root = path.resolve(__dirname, ".."); +const source = path.join(root, "src"); +const args = process.argv.slice(2); +const TASKKILL_TIMEOUT_MS = 1000; +const FINAL_EXIT_TIMEOUT_MS = 2000; + +function runBoundedCommand(command, commandArgs, timeout) { + return spawnSync(command, commandArgs, { + stdio: "ignore", + windowsHide: true, + timeout, + }); +} + +if (args[0] === "self-test-windows-cleanup") { + const failed = runBoundedCommand(process.execPath, ["-e", "process.exit(7)"], 200); + const started = Date.now(); + const stalled = runBoundedCommand( + process.execPath, + ["-e", "setInterval(() => {}, 10000)"], + 200, + ); + const bounded = Date.now() - started < 1500; + const failedDetected = failed.status !== 0 || Boolean(failed.error) || Boolean(failed.signal); + const stalledDetected = stalled.status !== 0 || Boolean(stalled.error) || Boolean(stalled.signal); + process.exit(failedDetected && stalledDetected && bounded ? 0 : 2); +} + +const nodeMajor = Number.parseInt(process.versions.node.split(".")[0], 10); +if (![20, 22].includes(nodeMajor)) { + console.error("claude-code-substrate-memory: Node.js 20 or 22 is required"); + process.exit(2); +} +const override = process.env.SUBSTRATE_PYTHON; +const candidates = override + ? [{ command: override, prefix: [] }] + : process.platform === "win32" + ? [ + { command: "py", prefix: ["-3.12"] }, + { command: "py", prefix: ["-3.11"] }, + { command: "py", prefix: ["-3"] }, + { command: "python", prefix: [] }, + { command: "python3", prefix: [] }, + ] + : [ + { command: "python3", prefix: [] }, + { command: "python", prefix: [] }, + ]; + +function selectPython() { + for (const candidate of candidates) { + const probe = spawnSync( + candidate.command, + [...candidate.prefix, "-c", "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"], + { encoding: "utf8", windowsHide: true }, + ); + if (probe.status === 0 && ["3.11", "3.12"].includes((probe.stdout || "").trim())) { + return candidate; + } + } + return null; +} + +const python = selectPython(); +if (!python) { + console.error("claude-code-substrate-memory: Python 3.11 or 3.12 is required"); + process.exit(2); +} +let moduleArgs; +if (args[0] === "hook") { + moduleArgs = ["-m", "claude_code_memory.hook", args[1] || ""]; +} else if (args[0] === "mcp") { + moduleArgs = ["-m", "claude_code_memory.server"]; +} else { + moduleArgs = ["-m", "claude_code_memory.cli", ...args]; +} +const env = { + ...process.env, + PYTHONPATH: source, + PYTHONNOUSERSITE: "1", + PYTHONDONTWRITEBYTECODE: "1", +}; +for (const name of ["SUBSTRATE_API_KEY", "HERMES_API_KEY", "SUBSTRATE_API_URL", "HERMES_API_URL"]) { + delete env[name]; +} + +const isHook = args[0] === "hook"; +const mediateWindowsHookIO = isHook && process.platform === "win32"; +const child = spawn(python.command, [...python.prefix, ...moduleArgs], { + env, + stdio: mediateWindowsHookIO ? ["inherit", "pipe", "pipe"] : "inherit", + windowsHide: true, + detached: isHook && process.platform !== "win32", +}); +if (mediateWindowsHookIO) { + // Descendants must not own the launcher's inherited output handles. If the + // OS tree-kill path fails, the caller still observes our hard exit deadline. + child.stdout.pipe(process.stdout, { end: false }); + child.stderr.pipe(process.stderr, { end: false }); +} +let timedOut = false; +let childResult = null; +let cleanupStarted = false; +let cleanupFinished = false; +let cleanupTimer = null; +let finalExitTimer = null; + +function finishIfReady() { + if (!childResult || (cleanupStarted && !cleanupFinished)) return; + const { code, signal } = childResult; + process.exit(timedOut || signal ? 2 : (code === null ? 2 : code)); +} + +function signalTree(signal) { + if (!child.pid) return; + if (isHook && process.platform !== "win32") { + try { + process.kill(-child.pid, signal); + return; + } catch (error) { + if (error && error.code !== "ESRCH") { + // Fall through to the direct child for permission and other races. + } + } + } + try { child.kill(signal); } catch (_ignored) { /* raced with exit */ } +} + +function killProcessTree() { + if (!child.pid || cleanupStarted) return; + cleanupStarted = true; + // Never wait indefinitely for taskkill, the direct child, or an exit event. + finalExitTimer = setTimeout(() => { + signalTree("SIGKILL"); + process.exit(2); + }, FINAL_EXIT_TIMEOUT_MS); + if (process.platform === "win32") { + const testMode = + process.env.SUBSTRATE_LAUNCHER_SELF_TEST === "1" + ? process.env.SUBSTRATE_TASKKILL_TEST_MODE + : null; + const taskkillCommand = testMode ? process.execPath : "taskkill"; + const taskkillArgs = + testMode === "stall" + ? ["-e", "setInterval(() => {}, 10000)"] + : testMode === "failure" + ? ["-e", "process.exit(7)"] + : ["/PID", String(child.pid), "/T", "/F"]; + const result = runBoundedCommand( + taskkillCommand, + taskkillArgs, + TASKKILL_TIMEOUT_MS, + ); + const treeKillSucceeded = + result.status === 0 && !result.error && !result.signal; + if (!treeKillSucceeded) { + // A failed or unavailable OS tree-kill cannot prove descendant cleanup. + // Bound the launcher by killing the direct child as a fallback. + signalTree("SIGKILL"); + console.error( + "claude-code-substrate-memory: Windows tree cleanup unavailable; direct child fallback used", + ); + } + cleanupFinished = true; + finishIfReady(); + return; + } + // Hook Python is the leader of a private process group. Always keep Node + // alive through the grace period: the leader can exit on SIGTERM while a + // grandchild ignores it, and an unref'ed timer would orphan that process. + signalTree("SIGTERM"); + cleanupTimer = setTimeout(() => { + signalTree("SIGKILL"); + // Give the kernel/init a bounded interval to reap killed descendants and + // ensure our direct child exit event has been delivered before Node exits. + cleanupTimer = setTimeout(() => { + cleanupFinished = true; + finishIfReady(); + }, 100); + }, 500); +} + +for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) { + process.on(signal, () => { + timedOut = true; + killProcessTree(); + }); +} + +const timer = isHook + ? setTimeout(() => { + timedOut = true; + killProcessTree(); + }, 10000) + : null; +if (timer) timer.unref(); + +child.on("error", () => { + if (timer) clearTimeout(timer); + if (cleanupTimer) clearTimeout(cleanupTimer); + if (finalExitTimer) clearTimeout(finalExitTimer); + console.error("claude-code-substrate-memory: runtime launch failed"); + process.exit(2); +}); +child.on("exit", (code, signal) => { + childResult = { code, signal }; + if (timer) clearTimeout(timer); + finishIfReady(); +}); diff --git a/scripts/verify_public_plugin_candidate.py b/scripts/verify_public_plugin_candidate.py index 03b7000..1a73f36 100644 --- a/scripts/verify_public_plugin_candidate.py +++ b/scripts/verify_public_plugin_candidate.py @@ -28,7 +28,18 @@ "htmlcov", "node_modules", } -SKIP_SUFFIXES = {".ico", ".jpeg", ".jpg", ".lock", ".pdf", ".png", ".pyc", ".woff", ".woff2", ".zip"} +SKIP_SUFFIXES = { + ".ico", + ".jpeg", + ".jpg", + ".lock", + ".pdf", + ".png", + ".pyc", + ".woff", + ".woff2", + ".zip", +} # RFC 2606 / RFC 6761 reserved names. These can never resolve to a real service, # so a URL under one of them cannot be an undeclared production endpoint. @@ -44,6 +55,7 @@ "127.0.0.1", "anthropic.com", "api.substrate.example", + "app.trysubstrate.co", "code.visualstudio.com", "docs.anthropic.com", "example.com", diff --git a/src/claude_code_memory/__init__.py b/src/claude_code_memory/__init__.py index 8cbb614..dfba18d 100644 --- a/src/claude_code_memory/__init__.py +++ b/src/claude_code_memory/__init__.py @@ -1,57 +1,59 @@ -"""Claude Code host integration for Substrate organizational memory.""" +"""Claude Code host integration for hosted Substrate memory.""" from __future__ import annotations -import os from typing import TypeAlias -from substrate_capture import CaptureEventBuilder, Deliverer, DurableSpool, SubstrateClient -from substrate_capture import config -from substrate_capture.client import SubstrateAPIError +from substrate_capture import CaptureEventBuilder, DurableSpool -PROVIDER_ID = "claude_code_memory" -__version__ = "0.1.0" +from .contract import HOSTED_ORIGIN, PROVIDER_ID, VERSION +from .credentials import credential_store +from .delivery import HostedDeliverer +from .onboarding import OnboardingManager +from .profile import capture_scope, state_home +from .strict_client import StrictHostedClient + +__version__ = VERSION Runtime: TypeAlias = tuple[ - SubstrateClient | None, + StrictHostedClient | None, DurableSpool, - Deliverer, + HostedDeliverer, CaptureEventBuilder, ] def runtime() -> Runtime: - """Build the client, durable spool, deliverer, and event builder. + """Build one fixed-origin, custody-backed, profile-scoped runtime. - An absent or invalid remote configuration disables delivery without disabling - capture: events continue to enter the local spool for a later sync. + Environment bearer keys and origin overrides are intentionally ignored. + Until device approval succeeds, no client exists and hooks do not read or + spool transcript content. """ - settings = config.resolved(PROVIDER_ID) - state = config.state_home(PROVIDER_ID) - key = config.api_key() - client: SubstrateClient | None = None - if settings["enabled"] and settings["api_url"] and key: - try: - client = SubstrateClient( - str(settings["api_url"]), - key, - timeout=float(settings["timeout"]), - user_agent=f"claude-code-substrate-memory/{__version__}", - ) - except (SubstrateAPIError, TypeError, ValueError): - client = None - - spool = DurableSpool(state / "spool") - deliverer = Deliverer(spool, client) + state = state_home() + store = credential_store(state) + onboarding = OnboardingManager(state, store=store) + key = store.get() + client = ( + StrictHostedClient( + HOSTED_ORIGIN, + key, + timeout=6.0, + user_agent=f"claude-code-substrate-memory/{VERSION}", + ) + if key + else None + ) + # Continuation can cover a large transcript after the bounded hook returns. + # Keep enough durable capacity for the accepted 8,001-record plus large-record corpus. + spool = DurableSpool(state / "spool", max_items=4096, max_bytes=128 * 1024 * 1024) + deliverer = HostedDeliverer(spool, client, onboarding, state) builder = CaptureEventBuilder( - { - "platform": "claude_code", - "agent_id": os.environ.get("CLAUDE_CODE_AGENT_ID", "default"), - }, + capture_scope(), provider_id=PROVIDER_ID, secrets=(key,) if key else (), ) return client, spool, deliverer, builder -__all__ = ["PROVIDER_ID", "Runtime", "__version__", "runtime"] +__all__ = ["PROVIDER_ID", "Runtime", "VERSION", "__version__", "runtime"] diff --git a/src/claude_code_memory/cli.py b/src/claude_code_memory/cli.py index aa6713d..667a5a1 100644 --- a/src/claude_code_memory/cli.py +++ b/src/claude_code_memory/cli.py @@ -4,30 +4,24 @@ import argparse import json +import shutil +import sys from collections.abc import Sequence from . import runtime +from .onboarding import OnboardingError, OnboardingManager +from .profile import state_home def _configure_text() -> str: - return """Configure credentials in the Claude Code process environment only: - - SUBSTRATE_API_URL=https://your-substrate-server.example - SUBSTRATE_API_KEY=your-profile-scoped-bearer-key - -The repository includes the required host wiring: - - .mcp.json - python -m claude_code_memory.server - PYTHONPATH=${CLAUDE_PLUGIN_ROOT}/src - - hooks/hooks.json - Stop -> python -m claude_code_memory.hook stop - PreCompact -> python -m claude_code_memory.hook pre-compact - SessionEnd -> python -m claude_code_memory.hook session-end - SessionStart-> python -m claude_code_memory.hook session-start - -Do not put SUBSTRATE_API_KEY in .mcp.json, hooks.json, or any other config file. + return """No bearer key or origin configuration is accepted. + +Install and register the verified plugin, then start Claude Code. SessionStart +performs no onboarding network call; it directs the human to /substrate-setup. +That explicit command begins fixed https://app.trysubstrate.co RFC 8628 approval +for scopes exactly `capture retrieve`, using public client +`substrate-claude-code`, then records the separate history decision. +Credentials use OS custody when safe or an owner-private profile file. """ @@ -35,14 +29,40 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="substrate-claude-code") commands = parser.add_subparsers(dest="command", required=True) commands.add_parser("serve", help="Run the stdio MCP server") - commands.add_parser("status", help="Show content-free local delivery status") + commands.add_parser("status", help="Show content-free local status") commands.add_parser("sync", help="Drain the durable local spool") - commands.add_parser("configure", help="Print environment and host wiring instructions") + commands.add_parser("configure", help="Print fixed-origin setup instructions") + setup = commands.add_parser("setup", help="Begin or resume device authorization") + setup.add_argument("--no-browser", action="store_true") + setup.add_argument("--wait", action="store_true") + setup.add_argument("--history", choices=("approve", "decline")) + repair = commands.add_parser("repair", help="Delete invalid local auth and reconnect") + repair.add_argument("--no-browser", action="store_true") + disconnect = commands.add_parser("disconnect", help="Delete local credentials only") + disconnect.add_argument("--confirm", action="store_true") + purge = commands.add_parser( + "purge-local-state", help="Delete credentials and queued local state" + ) + purge.add_argument("--confirm", action="store_true") + purge.add_argument("--delete-queued-data", action="store_true") return parser +def _print_human_instructions(result: dict[str, object]) -> None: + if result.get("phase") == "authorization_pending": + print("Approve Substrate in your browser:", file=sys.stderr) + print(str(result.get("verification_uri_complete") or ""), file=sys.stderr) + print(f"Code: {result.get('user_code') or ''}", file=sys.stderr) + elif result.get("phase") == "awaiting_history_consent": + print( + "Choose --history approve or --history decline. History import is unsupported in " + "this release, so neither choice scans or uploads old transcripts.", + file=sys.stderr, + ) + + def main(argv: Sequence[str] | None = None) -> int: - """Run one CLI subcommand.""" + """Run one command without rendering a credential or non-symbolic server error.""" args = _parser().parse_args(argv) if args.command == "serve": from .server import main as server_main @@ -52,6 +72,51 @@ def main(argv: Sequence[str] | None = None) -> int: print(_configure_text(), end="") return 0 + manager = OnboardingManager(state_home()) + try: + if args.command == "setup": + result = manager.begin(open_browser=not args.no_browser) + if args.wait and result.get("phase") == "authorization_pending": + result = manager.poll_until_terminal() + if args.history: + result = manager.consent_history(args.history == "approve") + _print_human_instructions(result) + print(json.dumps(result, sort_keys=True)) + return 0 if result.get("phase") not in {"declined", "failed", "repair_required"} else 2 + if args.command == "repair": + result = manager.repair(open_browser=not args.no_browser) + _print_human_instructions(result) + print(json.dumps(result, sort_keys=True)) + return 0 + if args.command == "disconnect": + if not args.confirm: + print(json.dumps({"error": "confirmation_required"}, sort_keys=True)) + return 2 + result = manager.disconnect_local() + result["server_revocation_performed"] = False + print(json.dumps(result, sort_keys=True)) + return 0 + if args.command == "purge-local-state": + if not args.confirm or not args.delete_queued_data: + print(json.dumps({"error": "two_confirmations_required"}, sort_keys=True)) + return 2 + home = state_home() + manager.disconnect_local() + if home.is_symlink(): + raise OSError("state root must not be a symlink") + shutil.rmtree(home) + print( + json.dumps( + {"local_state_deleted": True, "server_revocation_performed": False}, + sort_keys=True, + ) + ) + return 0 + except (OnboardingError, OSError) as exc: + category = exc.category if isinstance(exc, OnboardingError) else "local_state_error" + print(json.dumps({"error": category}, sort_keys=True)) + return 2 + _client, _spool, deliverer, _builder = runtime() result = deliverer.status() if args.command == "status" else deliverer.drain() print(json.dumps(result, sort_keys=True)) diff --git a/src/claude_code_memory/contract.py b/src/claude_code_memory/contract.py new file mode 100644 index 0000000..81ffccc --- /dev/null +++ b/src/claude_code_memory/contract.py @@ -0,0 +1,14 @@ +"""Single source of runtime and protocol compatibility constants.""" + +from __future__ import annotations + +VERSION = "0.2.0" +PROVIDER_ID = "claude_code_memory" +HOSTED_ORIGIN = "https://app.trysubstrate.co" +OAUTH_CLIENT_ID = "substrate-claude-code" +OAUTH_SCOPES = "capture retrieve" +SUPPORTED_CLAUDE_CODE_VERSIONS = ("2.1.237",) +SUPPORTED_PYTHON = ((3, 11), (3, 12)) +HERMES_AUTHORITY_MAIN = "db4ddc2f093f833ffa62f4246860de96ba398713" +HERMES_AUTHORITY_RELEASE = "e4ad07cfc858618edfd69e1f3be5e8345b253037" +CLAUDE_BASE = "2a5051304ac7e8b960e3cdd9d4d2e96dcba76b80" diff --git a/src/claude_code_memory/credentials.py b/src/claude_code_memory/credentials.py new file mode 100644 index 0000000..d4a3c58 --- /dev/null +++ b/src/claude_code_memory/credentials.py @@ -0,0 +1,277 @@ +"""Profile-scoped credential custody for hosted Substrate onboarding in Claude Code. + +The public API never returns credential values. Native credential helpers are +preferred; an owner-private file is the deliberately small portability fallback. +""" + +from __future__ import annotations + +import hashlib +import os +import shutil +import stat +import subprocess +from pathlib import Path + +from .local_security import secure_windows_tree + +_SERVICE = "co.trysubstrate.claude-code" + + +def _profile_account(home: Path, slot: str) -> str: + digest = hashlib.sha256(os.fsencode(str(home.resolve()))).hexdigest()[:24] + return f"{digest}:{slot}" + + +class CredentialStore: + """Abstract secret slot storage.""" + + backend = "unknown" + + def get(self, slot: str = "access-token") -> str: + raise NotImplementedError + + def put(self, value: str, slot: str = "access-token") -> None: + raise NotImplementedError + + def delete(self, slot: str = "access-token") -> None: + raise NotImplementedError + + +class SecretToolStore(CredentialStore): + backend = "secret-service" + + def __init__(self, home: Path) -> None: + self.account = _profile_account(home, "profile") + + def get(self, slot: str = "access-token") -> str: + result = subprocess.run( + ("secret-tool", "lookup", "service", _SERVICE, "account", self.account, "slot", slot), + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + return result.stdout.rstrip("\n") if result.returncode == 0 else "" + + def put(self, value: str, slot: str = "access-token") -> None: + if not value or len(value) > 16384: + raise ValueError("invalid credential") + subprocess.run( + ( + "secret-tool", + "store", + "--label", + "Substrate for Claude Code", + "service", + _SERVICE, + "account", + self.account, + "slot", + slot, + ), + input=value, + text=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + check=True, + ) + + def delete(self, slot: str = "access-token") -> None: + result = subprocess.run( + ("secret-tool", "clear", "service", _SERVICE, "account", self.account, "slot", slot), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + check=False, + ) + if result.returncode != 0 and self.get(slot): + raise OSError("native credential deletion failed") + if self.get(slot): + raise OSError("native credential remains after deletion") + + +class MacOSKeychainStore(CredentialStore): + backend = "macos-keychain" + + def __init__(self, home: Path) -> None: + self.account = _profile_account(home, "profile") + + def get(self, slot: str = "access-token") -> str: + result = subprocess.run( + ( + "security", + "find-generic-password", + "-a", + self.account, + "-s", + f"{_SERVICE}.{slot}", + "-w", + ), + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + return result.stdout.rstrip("\n") if result.returncode == 0 else "" + + def put(self, value: str, slot: str = "access-token") -> None: + if not value or len(value) > 16384: + raise ValueError("invalid credential") + # Apple's security tool has no stdin form. Avoid it when process + # inspection is not private by falling back to the protected file. + raise OSError("non-interactive keychain write unavailable") + + def delete(self, slot: str = "access-token") -> None: + result = subprocess.run( + ("security", "delete-generic-password", "-a", self.account, "-s", f"{_SERVICE}.{slot}"), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + check=False, + ) + if result.returncode != 0 and self.get(slot): + raise OSError("native credential deletion failed") + if self.get(slot): + raise OSError("native credential remains after deletion") + + +class PrivateFileStore(CredentialStore): + backend = "owner-private-file" + + def __init__(self, home: Path) -> None: + root = home / "credentials" + if root.exists() and root.is_symlink(): + raise OSError("credential directory must not be a symlink") + root.mkdir(parents=True, exist_ok=True, mode=0o700) + if os.name == "posix": + os.chmod(root, 0o700) + elif os.name == "nt": + secure_windows_tree(root) + self.root = root + + def _path(self, slot: str) -> Path: + if not slot or not slot.replace("-", "").isalnum(): + raise ValueError("invalid credential slot") + return self.root / slot + + def get(self, slot: str = "access-token") -> str: + path = self._path(slot) + if path.is_symlink(): + return "" + try: + info = path.stat(follow_symlinks=False) + if not stat.S_ISREG(info.st_mode): + return "" + if os.name == "posix": + getuid = getattr(os, "getuid", None) + if ( + not callable(getuid) + or info.st_uid != getuid() + or stat.S_IMODE(info.st_mode) & 0o077 + ): + return "" + value = path.read_text(encoding="utf-8") + except (FileNotFoundError, OSError, UnicodeError): + return "" + return value if 0 < len(value) <= 16384 else "" + + def put(self, value: str, slot: str = "access-token") -> None: + if not value or len(value) > 16384 or "\x00" in value: + raise ValueError("invalid credential") + path = self._path(slot) + if path.is_symlink(): + raise OSError("credential path must not be a symlink") + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(temporary, flags, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + if os.name == "posix": + os.chmod(temporary, 0o600) + elif os.name == "nt": + secure_windows_tree(self.root) + os.replace(temporary, path) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + def delete(self, slot: str = "access-token") -> None: + path = self._path(slot) + if path.is_symlink(): + raise OSError("credential path must not be a symlink") + try: + path.unlink() + except FileNotFoundError: + pass + + +class PreferredCredentialStore(CredentialStore): + """Use a functioning native vault, otherwise the explicit private fallback.""" + + def __init__(self, home: Path) -> None: + self.fallback = PrivateFileStore(home) + self.native: CredentialStore | None = None + self._active_backend = self.fallback.backend + if shutil.which("secret-tool"): + self.native = SecretToolStore(home) + elif sys_platform() == "darwin" and shutil.which("security"): + self.native = MacOSKeychainStore(home) + + @property + def backend(self) -> str: + return self._active_backend + + def get(self, slot: str = "access-token") -> str: + if self.native is not None: + try: + value = self.native.get(slot) + if value: + self._active_backend = self.native.backend + return value + except (OSError, subprocess.SubprocessError): + pass + return self.fallback.get(slot) + + def put(self, value: str, slot: str = "access-token") -> None: + if self.native is not None: + try: + self.native.put(value, slot) + self.fallback.delete(slot) + self._active_backend = self.native.backend + return + except (OSError, subprocess.SubprocessError): + pass + self.fallback.put(value, slot) + self._active_backend = self.fallback.backend + + def delete(self, slot: str = "access-token") -> None: + if self.native is not None: + self.native.delete(slot) + if self.native.get(slot): + raise OSError("native credential remains after deletion") + self.fallback.delete(slot) + if self.fallback.get(slot): + raise OSError("fallback credential remains after deletion") + + +def sys_platform() -> str: + import sys + + return sys.platform + + +def credential_store(home: Path) -> CredentialStore: + return PreferredCredentialStore(home.resolve()) diff --git a/src/claude_code_memory/delivery.py b/src/claude_code_memory/delivery.py new file mode 100644 index 0000000..f27d53f --- /dev/null +++ b/src/claude_code_memory/delivery.py @@ -0,0 +1,290 @@ +"""Truthful, retrying hosted delivery with durable scheduling and auth repair.""" + +from __future__ import annotations + +import hashlib +import json +import time +from pathlib import Path +from typing import Any + +from substrate_capture import ENDPOINTS, DurableSpool, SubstrateAPIError, secure_atomic_json_write +from substrate_capture.spool import validate_no_symlink_ancestors + +from .onboarding import OnboardingManager +from .state import FileLock +from .strict_client import StrictHostedClient + +_RETAIN = frozenset( + { + "http_401", + "http_403", + "http_408", + "http_425", + "http_429", + "http_500", + "http_502", + "http_503", + "http_504", + "not_configured", + "server_upgrade_required", + "timeout", + "transport_error", + } +) +_AUTH = frozenset({"http_401", "http_403"}) +_MAX_ATTEMPTS = 64 +_MAX_RETRY_SECONDS = 300.0 + + +class HostedDeliverer: + """Drain a shared spool under one process lease and persist content-free health.""" + + def __init__( + self, + spool: DurableSpool, + client: StrictHostedClient | None, + onboarding: OnboardingManager, + state_root: Path, + *, + budget_seconds: float = 8.0, + clock: Any = time.monotonic, + wall_clock: Any = time.time, + ) -> None: + self.spool = spool + self.client = client + self.onboarding = onboarding + self.state_root = state_root + self.budget_seconds = min(10.0, max(0.5, float(budget_seconds))) + self._clock = clock + self._wall_clock = wall_clock + self._path = state_root / "delivery-status.json" + self._receipts = validate_no_symlink_ancestors(state_root / "delivery-receipts") + self._receipts.mkdir(parents=True, exist_ok=True, mode=0o700) + validate_no_symlink_ancestors(self._receipts) + self._drain_lock = FileLock(state_root / ".delivery.lock") + self._state = self._load_state() + + @staticmethod + def _empty_state() -> dict[str, Any]: + return { + "version": 1, + "delivered": 0, + "receipt_base": 0, + "deferred": 0, + "quarantined": 0, + "rejected": 0, + "auth_repairs": 0, + "attempt": 0, + "next_retry_at": 0.0, + "last_category": "", + "last_success_at": 0.0, + "capability_validated_at": 0.0, + } + + def _load_state(self) -> dict[str, Any]: + state = self._empty_state() + try: + if self._path.is_symlink() or self._path.stat().st_size > 64 * 1024: + return state + value = json.loads(self._path.read_text(encoding="utf-8")) + except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError): + return state + if not isinstance(value, dict) or value.get("version") != 1: + return state + for key in ( + "delivered", + "receipt_base", + "deferred", + "quarantined", + "rejected", + "auth_repairs", + "attempt", + ): + item = value.get(key) + if isinstance(item, int) and not isinstance(item, bool) and item >= 0: + state[key] = item + for key in ("next_retry_at", "last_success_at", "capability_validated_at"): + item = value.get(key) + if isinstance(item, (int, float)) and not isinstance(item, bool) and item >= 0: + state[key] = float(item) + category = value.get("last_category") + if isinstance(category, str) and len(category) <= 64: + state["last_category"] = category + if "receipt_base" not in value: + state["receipt_base"] = int(state["delivered"]) + state["delivered"] = int(state["receipt_base"]) + self._receipt_count() + return state + + def _receipt_path(self, event_id: str) -> Path: + return self._receipts / f"{hashlib.sha256(event_id.encode('utf-8')).hexdigest()}.json" + + def _has_receipt(self, event_id: str) -> bool: + path = self._receipt_path(event_id) + try: + if path.is_symlink() or path.stat(follow_symlinks=False).st_size > 4096: + return False + value = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError): + return False + return isinstance(value, dict) and value.get("event_id") == event_id + + def _record_receipt(self, event_id: str) -> None: + path = self._receipt_path(event_id) + if not self._has_receipt(event_id): + secure_atomic_json_write( + path, + {"version": 1, "event_id": event_id, "accepted": True}, + ) + + def _receipt_count(self) -> int: + try: + return sum( + 1 + for path in self._receipts.glob("*.json") + if path.is_file() and not path.is_symlink() + ) + except OSError: + return 0 + + def _save_state(self) -> None: + secure_atomic_json_write(self._path, self._state) + + def enqueue(self, event: dict[str, Any]) -> Path | None: + try: + return self.spool.append(event) + except (OSError, ValueError): + with self._drain_lock: + self._state = self._load_state() + self._state["rejected"] += 1 + self._state["last_category"] = "spool_rejected" + self._save_state() + return None + + def _attempt(self, event: dict[str, Any]) -> tuple[str, float | None]: + if self.client is None: + return "not_configured", None + kind = str(event.get("kind") or event.get("capture_kind") or "") + path = ENDPOINTS.get(kind) + event_id = event.get("event_id") + if path is None: + return "unknown_kind", None + if not isinstance(event_id, str) or not event_id: + return "invalid_event", None + try: + self.client.request("POST", path, body=event, idempotency_key=event_id) + except SubstrateAPIError as exc: + return exc.category, exc.retry_after + self._state["capability_validated_at"] = self._wall_clock() + return "", None + + @staticmethod + def _retry_delay(event_id: str, attempt: int, hint: float | None, *, auth: bool) -> float: + base = 30.0 if auth else min(_MAX_RETRY_SECONDS, 2.0 ** min(attempt, 8)) + digest = hashlib.sha256(event_id.encode("utf-8")).digest() + fraction = int.from_bytes(digest[:4], "big") / 0xFFFFFFFF + jittered = base * (0.8 + 0.4 * fraction) + hinted = hint if isinstance(hint, (int, float)) and 0 <= hint <= _MAX_RETRY_SECONDS else 0.0 + return min(_MAX_RETRY_SECONDS, max(jittered, float(hinted))) + + def drain(self) -> dict[str, Any]: + deadline = self._clock() + self.budget_seconds + with self._drain_lock: + self._state = self._load_state() + if self._wall_clock() < float(self._state["next_retry_at"]): + return self.status(_locked=True) + for _ in range(_MAX_ATTEMPTS): + if self._clock() >= deadline: + break + claimed = self.spool.claim_oldest() + if claimed is None: + self._state["attempt"] = 0 + self._state["next_retry_at"] = 0.0 + break + try: + event = self.spool.load(claimed) + except (OSError, ValueError): + self.spool.quarantine(claimed) + self._state["quarantined"] += 1 + self._state["last_category"] = "corrupt_event" + continue + event_id = str(event.get("event_id") or "") + if event_id and self._has_receipt(event_id): + self.spool.remove(claimed) + self._state["delivered"] = ( + int(self._state["receipt_base"]) + self._receipt_count() + ) + self._state["attempt"] = 0 + self._state["next_retry_at"] = 0.0 + continue + category, retry_after = self._attempt(event) + self._state["last_category"] = category + if not category: + # The receipt is the commit record. If power fails after it, + # restart removes the still-spooled event without redelivery. + self._record_receipt(event_id) + self.spool.remove(claimed) + self._state["delivered"] = ( + int(self._state["receipt_base"]) + self._receipt_count() + ) + self._state["attempt"] = 0 + self._state["next_retry_at"] = 0.0 + self._state["last_success_at"] = self._wall_clock() + continue + if category in _RETAIN: + self.spool.release(claimed) + self._state["deferred"] += 1 + self._state["attempt"] += 1 + delay = self._retry_delay( + event_id, + int(self._state["attempt"]), + retry_after, + auth=category in _AUTH, + ) + self._state["next_retry_at"] = self._wall_clock() + delay + if category in _AUTH: + self.onboarding.require_repair(category) + self.client = None + self._state["auth_repairs"] += 1 + break + self.spool.quarantine(claimed) + self._state["quarantined"] += 1 + self._save_state() + return self.status(_locked=True) + + def status(self, *, _locked: bool = False) -> dict[str, Any]: + if not _locked: + with self._drain_lock: + self._state = self._load_state() + return self.status(_locked=True) + onboarding = self.onboarding.status() + authenticated = bool(onboarding.get("authenticated")) + phase = str(onboarding.get("phase") or "unknown")[:64] + current_auth = authenticated and phase in {"awaiting_history_consent", "ready"} + jobs_root = self.state_root / "hook-jobs" + try: + capture_jobs_pending = sum( + 1 for path in jobs_root.glob("*.json") if not path.is_symlink() and path.is_file() + ) + except OSError: + capture_jobs_pending = 0 + return { + "pending": len(self.spool), + "capture_jobs_pending": capture_jobs_pending, + "credential_present": authenticated, + "onboarding_phase": phase, + "connected": current_auth and bool(self._state["last_success_at"]), + "last_category": self._state["last_category"], + "retry_scheduled": self._wall_clock() < float(self._state["next_retry_at"]), + "capability_validated": current_auth and bool(self._state["capability_validated_at"]), + "last_success_recorded": bool(self._state["last_success_at"]), + "capability_validation_recorded": bool(self._state["capability_validated_at"]), + **{ + key: self._state[key] + for key in ("delivered", "deferred", "quarantined", "rejected", "auth_repairs") + }, + **self.spool.statistics(), + } + + +__all__ = ["HostedDeliverer"] diff --git a/src/claude_code_memory/hook.py b/src/claude_code_memory/hook.py index fe88258..da143f3 100644 --- a/src/claude_code_memory/hook.py +++ b/src/claude_code_memory/hook.py @@ -1,25 +1,44 @@ -"""Fail-open Claude Code lifecycle hook entry point.""" +"""Fail-open Claude lifecycle adapter with durable local continuation jobs.""" from __future__ import annotations +import hashlib import json import os +import stat import sys +import time +import uuid from collections.abc import Callable, Sequence +from pathlib import Path from typing import Any, TextIO -from substrate_capture import Checkpoint, content_digest -from substrate_capture import config +from substrate_capture import CaptureEventBuilder, DurableSpool, secure_atomic_json_write +from substrate_capture.spool import validate_no_symlink_ancestors -from . import PROVIDER_ID, Runtime, runtime +from . import Runtime, runtime +from .contract import PROVIDER_ID +from .profile import capture_scope, state_home from .recall import recall_block -from .transcript import read_messages +from .state import FileLock, SessionTransaction +from .transcript import read_message_window _EVENT_KINDS = {"stop": "turn", "pre-compact": "pre_compress"} -_VALID_EVENTS = frozenset({*_EVENT_KINDS, "session-end", "session-start"}) +_VALID_EVENTS = frozenset({*_EVENT_KINDS, "session-end", "session-start", "user-prompt-submit"}) _MAX_HOOK_INPUT_CHARS = 1024 * 1024 +_CAPTURE_WINDOW_MESSAGES = 32 +_CAPTURE_WINDOW_BYTES = 512 * 1024 +_EVENT_BUILD_BATCH = _CAPTURE_WINDOW_MESSAGES +_MAX_JOB_BYTES = 32 * 1024 +_MAX_PENDING_JOBS = 128 +_MAX_JOB_SCAN = _MAX_PENDING_JOBS +_MAX_PARTIAL_RECORD_BYTES = 1024 * 1024 +_RETRY_BASE_SECONDS = 0.05 +_RETRY_MAX_SECONDS = 2.0 +_HOOK_LOCAL_BUDGET_SECONDS = 2.5 RuntimeFactory = Callable[[], Runtime] -RecallFunction = Callable[[int], str] +AuthorizationFunction = Callable[[], bool] +RecallFunction = Callable[[int, dict[str, Any]], str] def _debug(message: str, sink: TextIO) -> None: @@ -28,66 +47,602 @@ def _debug(message: str, sink: TextIO) -> None: sink.flush() +def _reject_constant(_value: str) -> None: + raise ValueError("non-JSON numeric constant") + + def _read_hook_data(source: TextIO) -> dict[str, Any]: raw = source.read(_MAX_HOOK_INPUT_CHARS + 1) if len(raw) > _MAX_HOOK_INPUT_CHARS: return {} - value = json.loads(raw or "{}") + value = json.loads(raw or "{}", parse_constant=_reject_constant) return value if isinstance(value, dict) else {} -def _checkpoint(session_id: str) -> Checkpoint: - return Checkpoint(config.state_home(PROVIDER_ID) / "checkpoints", session_id) +def _truthy(value: Any) -> bool: + if value is True: + return True + return isinstance(value, str) and value.strip().casefold() in {"1", "true", "yes", "on"} + + +def _nonempty(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _is_primary(data: dict[str, Any]) -> bool: + """Accept primary hooks while rejecting documented official subagents.""" + # In Claude Code 2.1.237, these fields are present only for subagent hooks. + if any(_nonempty(data.get(key)) for key in ("agent_id", "agentId", "agent_transcript_path")): + return False + if str(data.get("hook_event_name") or "").strip() == "SubagentStop": + return False + if any( + _truthy(data.get(key)) + for key in ( + "isSidechain", + "is_sidechain", + "isSubagent", + "is_subagent", + "isBackground", + "is_background", + ) + ): + return False + # parent_session_id is valid for a primary fork and agent_type is valid for + # --agent primary sessions; neither is a subagent signal by itself. + return True + + +def _authorized() -> bool: + """Read durable human approval without a vault or network wait.""" + path = state_home() / "onboarding" / "state.json" + try: + if path.is_symlink() or path.stat(follow_symlinks=False).st_size > 64 * 1024: + return False + value = json.loads(path.read_text(encoding="utf-8"), parse_constant=_reject_constant) + except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError, ValueError): + return False + return isinstance(value, dict) and value.get("phase") in { + "awaiting_history_consent", + "ready", + } + + +def _bootstrap_onboarding(errors: TextIO) -> None: + """Emit only a content-free setup instruction; setup itself is explicit.""" + errors.write("Substrate approval is required before transcript capture.\n") + errors.write("Run /substrate-setup to start or resume fixed-origin device approval.\n") + errors.flush() + + +class _AdmissionOnly: + """Local spool admission used by hooks; it never reads credentials or networks.""" + + def __init__(self, spool: DurableSpool) -> None: + self.spool = spool + + def enqueue(self, event: dict[str, Any]) -> Path | None: + try: + return self.spool.append(event) + except (OSError, ValueError): + return None + + def drain(self) -> dict[str, Any]: + return {"pending": len(self.spool)} + + +def _local_runtime() -> Runtime: + state = state_home() + spool = DurableSpool(state / "spool") + deliverer = _AdmissionOnly(spool) + builder = CaptureEventBuilder(capture_scope(), provider_id=PROVIDER_ID) + return None, spool, deliverer, builder # type: ignore[return-value] + + +def _admit_events(deliverer: Any, events: list[dict[str, Any]]) -> bool: + # Deterministic event IDs make a partially admitted batch idempotent on retry. + return all(deliverer.enqueue(event) is not None for event in events) + + +def _message_events( + builder: Any, + kind: str, + session_id: str, + messages: list[dict[str, Any]], + *, + start_index: int, + skipped_records: int, +) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + for offset in range(0, len(messages), _EVENT_BUILD_BATCH): + batch = messages[offset : offset + _EVENT_BUILD_BATCH] + events.extend( + builder.iter_message_events( + kind, + session_id, + batch, + start_index=start_index + offset, + payload={ + "source": "claude_code_transcript", + "skipped_records": skipped_records if offset == 0 else 0, + }, + capture_origin="hook", + deterministic=True, + ) + ) + return events def _capture_transcript( event: str, data: dict[str, Any], runtime_factory: RuntimeFactory, -) -> None: + *, + max_windows: int | None = None, + deadline: float | None = None, + lock_timeout: float = 2.0, + drain: bool = False, +) -> bool: + """Process bounded append-only windows and return true only after EOF/boundary.""" session_id = str(data.get("session_id") or "")[:512] transcript_path = data.get("transcript_path") - settings = config.resolved(PROVIDER_ID) - messages = ( - read_messages(transcript_path, include_sidechains=bool(settings["capture_sidechains"])) - if isinstance(transcript_path, str) - else [] - ) - client, spool, deliverer, builder = runtime_factory() - del client, spool - checkpoint = _checkpoint(session_id) - - if event == "session-end": - if not checkpoint.mark("session_end"): - deliverer.drain() - return - boundary = {"start": 0, "end": len(messages)} - capture = builder.payload_event( - "session_end", - session_id, - {"summary": {"message_count": len(messages), "boundary": "session_end"}}, - boundary=boundary, - capture_origin="hook", + if not session_id or not isinstance(transcript_path, str) or not transcript_path: + return True + _client, _spool, deliverer, builder = runtime_factory() + transactions = state_home() / "checkpoints" + windows = 0 + with SessionTransaction(transactions, session_id, lock_timeout=lock_timeout) as transaction: + state = dict(transaction.state) + if state.get("session_end"): + return True + target_cursor = data.get("target_cursor") + reached_eof = ( + isinstance(target_cursor, int) + and not isinstance(target_cursor, bool) + and int(state["cursor"]) >= target_cursor ) - deliverer.enqueue(capture) + while not reached_eof and (max_windows is None or windows < max_windows): + if deadline is not None and time.monotonic() >= deadline: + return False + previous_cursor = int(state["cursor"]) + window = read_message_window( + transcript_path, + cursor=previous_cursor, + expected_source_id=str(state["source_id"]), + expected_anchor=str(state.get("anchor", "")), + include_sidechains=False, + limit=_CAPTURE_WINDOW_MESSAGES, + max_bytes=_CAPTURE_WINDOW_BYTES, + end_cursor=( + int(data["target_cursor"]) + if isinstance(data.get("target_cursor"), int) + and not isinstance(data.get("target_cursor"), bool) + else None + ), + ) + if not window.readable: + return False + job_source_id = data.get("source_id") + if ( + isinstance(job_source_id, str) + and job_source_id + and window.source_id != job_source_id + ): + return False + if window.reset: + state.update( + cursor=0, + message_index=0, + source_id=window.source_id, + anchor="", + session_end=False, + ) + start_index = int(state["message_index"]) + messages: list[dict[str, Any]] = [] + for offset, message in enumerate(window.messages): + selected = dict(message) + selected["_capture_index"] = start_index + offset + messages.append(selected) + kind = "turn" if event == "session-end" else _EVENT_KINDS[event] + events = _message_events( + builder, + kind, + session_id, + messages, + start_index=start_index, + skipped_records=window.skipped_records, + ) + if not messages and window.skipped_records: + events.append( + builder.payload_event( + kind, + session_id, + { + "source": "claude_code_transcript", + "loss_signal": "records_quarantined", + "skipped_records": window.skipped_records, + }, + boundary={"start": start_index, "end": start_index}, + capture_origin="hook", + deterministic=True, + ) + ) + if not _admit_events(deliverer, events): + return False + state.update( + cursor=window.next_cursor, + message_index=start_index + len(messages), + source_id=window.source_id, + anchor=window.anchor, + loss_count=int(state.get("loss_count", 0)) + window.skipped_records, + ) + transaction.commit(state) + windows += 1 + if window.complete: + reached_eof = True + break + if window.next_cursor == previous_cursor and not messages: + return False + if not reached_eof: + return False + if event == "session-end": + boundary = {"start": 0, "end": int(state["message_index"])} + capture = builder.payload_event( + "session_end", + session_id, + { + "summary": { + "message_count": int(state["message_index"]), + "boundary": "session_end", + "loss_count": int(state.get("loss_count", 0)), + } + }, + boundary=boundary, + capture_origin="hook", + deterministic=True, + ) + if deliverer.enqueue(capture) is None: + return False + state["session_end"] = True + transaction.commit(state) + if drain: deliverer.drain() - return - - kind = _EVENT_KINDS[event] - digests = [content_digest(message) for message in messages] - start = checkpoint.unseen(digests) - if start < len(messages): - for capture in builder.iter_message_events( - kind, - session_id, - messages[start:], - start_index=start, - payload={"source": "claude_code_transcript"}, - capture_origin="hook", + return True + + +def _jobs_root() -> Path: + root = validate_no_symlink_ancestors(state_home() / "hook-jobs") + root.mkdir(parents=True, exist_ok=True, mode=0o700) + validate_no_symlink_ancestors(root) + if os.name == "posix": + os.chmod(root, 0o700) + return root + + +def _job_id(event: str, session_id: str, transcript_path: str) -> str: + encoded = json.dumps( + [event, session_id, os.path.abspath(os.path.expanduser(transcript_path))], + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +class _QueueFullError(OSError): + """The bounded durable queue cannot truthfully admit more work.""" + + +def _source_identity(info: os.stat_result) -> str: + return hashlib.sha256(f"{info.st_dev}:{info.st_ino}".encode("ascii")).hexdigest()[:32] + + +def _anchor_from_stream(stream: Any, position: int) -> str: + start = max(0, position - 4096) + stream.seek(start) + sample = stream.read(position - start) + digest = hashlib.sha256() + digest.update(position.to_bytes(8, "big")) + digest.update(sample) + return digest.hexdigest() + + +def _queue_job(event: str, data: dict[str, Any]) -> str | None: + session_id = data.get("session_id") + transcript_path = data.get("transcript_path") + if not isinstance(session_id, str) or not session_id or len(session_id) > 512: + return None + if not isinstance(transcript_path, str) or not transcript_path or len(transcript_path) > 8192: + return None + source = Path(transcript_path).expanduser().absolute() + if source.is_symlink(): + return None + try: + with source.open("rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode): + return None + target_cursor = info.st_size + target_anchor = _anchor_from_stream(stream, target_cursor) + partial_record = False + if target_cursor: + stream.seek(target_cursor - 1) + partial_record = stream.read(1) != b"\n" + except OSError: + return None + identifier = f"{time.time_ns():020d}-{uuid.uuid4().hex}" + root = _jobs_root() + path = root / f"{identifier}.json" + if path.is_symlink(): + raise OSError("hook job must not be a symlink") + with FileLock(root / ".admission.lock", timeout=0.2): + admitted = 0 + for candidate in root.iterdir(): + if candidate.name.endswith(".json"): + admitted += 1 + if admitted >= _MAX_PENDING_JOBS: + raise _QueueFullError("hook continuation queue is full") + secure_atomic_json_write( + path, + { + "version": 2, + "event": event, + "session_id": session_id, + "transcript_path": str(source), + "target_cursor": target_cursor, + "source_id": _source_identity(info), + "target_anchor": target_anchor, + "partial_record": partial_record, + "completion_cursor": None, + "completion_anchor": None, + "attempts": 0, + "next_attempt_ns": 0, + "rotation": 0, + }, + ) + return identifier + + +def _load_job(identifier: str) -> tuple[Path, dict[str, Any]]: + parts = identifier.split("-", 1) + if ( + len(parts) != 2 + or len(parts[0]) != 20 + or not parts[0].isdigit() + or len(parts[1]) != 32 + or any(character not in "0123456789abcdef" for character in parts[1]) + ): + raise ValueError("invalid hook job") + path = _jobs_root() / f"{identifier}.json" + if path.is_symlink(): + raise OSError("hook job must not be a symlink") + info = path.stat(follow_symlinks=False) + if not stat.S_ISREG(info.st_mode) or info.st_size > _MAX_JOB_BYTES: + raise OSError("invalid hook job") + value = json.loads(path.read_text(encoding="utf-8"), parse_constant=_reject_constant) + if not isinstance(value, dict) or value.get("version") not in {1, 2}: + raise ValueError("invalid hook job") + event = value.get("event") + session_id = value.get("session_id") + transcript_path = value.get("transcript_path") + target_cursor = value.get("target_cursor") + if ( + event not in {*_EVENT_KINDS, "session-end"} + or not isinstance(session_id, str) + or not session_id + or len(session_id) > 512 + or not isinstance(transcript_path, str) + or not transcript_path + or len(transcript_path) > 8192 + or not isinstance(target_cursor, int) + or isinstance(target_cursor, bool) + or target_cursor < 0 + ): + raise ValueError("invalid hook job") + if value.get("version") == 2: + completion = value.get("completion_cursor") + if ( + not isinstance(value.get("source_id"), str) + or len(value["source_id"]) != 32 + or not isinstance(value.get("target_anchor"), str) + or len(value["target_anchor"]) != 64 + or not isinstance(value.get("partial_record"), bool) + or ( + completion is not None + and (not isinstance(completion, int) or isinstance(completion, bool)) + ) + or ( + value.get("completion_anchor") is not None + and ( + not isinstance(value.get("completion_anchor"), str) + or len(value["completion_anchor"]) != 64 + ) + ) + or not isinstance(value.get("attempts"), int) + or isinstance(value.get("attempts"), bool) + or not isinstance(value.get("next_attempt_ns"), int) + or isinstance(value.get("next_attempt_ns"), bool) + or not isinstance(value.get("rotation"), int) + or isinstance(value.get("rotation"), bool) ): - deliverer.enqueue(capture) - deliverer.drain() - checkpoint.advance(digests) + raise ValueError("invalid hook job") + else: + value.update( + partial_record=False, + completion_cursor=None, + completion_anchor=None, + attempts=0, + next_attempt_ns=0, + rotation=0, + ) + return path, value + + +def _completion_target(data: dict[str, Any]) -> int | None: + """Freeze only the boundary completing the record partial at admission.""" + target = int(data["target_cursor"]) + if data.get("version") == 1: + return target + source = Path(str(data["transcript_path"])) + if source.is_symlink(): + return None + with source.open("rb") as stream: + info = os.fstat(stream.fileno()) + if ( + not stat.S_ISREG(info.st_mode) + or _source_identity(info) != data.get("source_id") + or info.st_size < target + or _anchor_from_stream(stream, target) != data.get("target_anchor") + ): + return None + if not data.get("partial_record"): + return target + completion = data.get("completion_cursor") + if isinstance(completion, int) and not isinstance(completion, bool): + if info.st_size < completion or data.get("completion_anchor") != _anchor_from_stream( + stream, completion + ): + return None + stream.seek(completion - 1) + return completion if stream.read(1) == b"\n" else None + context_start = max(0, target - _MAX_PARTIAL_RECORD_BYTES) + stream.seek(context_start) + prefix = stream.read(target - context_start) + separator = prefix.rfind(b"\n") + if context_start and separator < 0: + return None + line_start = context_start + separator + 1 + stream.seek(line_start) + raw = stream.readline(_MAX_PARTIAL_RECORD_BYTES + 1) + boundary = stream.tell() + if len(raw) > _MAX_PARTIAL_RECORD_BYTES or not raw.endswith(b"\n") or boundary <= target: + return None + try: + record = json.loads( + raw.decode("utf-8", errors="strict"), + parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value)), + ) + except (UnicodeDecodeError, ValueError, json.JSONDecodeError): + return None + if not isinstance(record, dict) or _anchor_from_stream(stream, target) != data.get( + "target_anchor" + ): + return None + data["completion_anchor"] = _anchor_from_stream(stream, boundary) + return boundary + + +def _schedule_retry(path: Path, data: dict[str, Any]) -> None: + attempts = max(0, int(data.get("attempts", 0))) + 1 + delay = min(_RETRY_MAX_SECONDS, _RETRY_BASE_SECONDS * (2 ** min(attempts - 1, 8))) + data.update( + attempts=attempts, + next_attempt_ns=time.time_ns() + int(delay * 1_000_000_000), + rotation=time.time_ns(), + ) + secure_atomic_json_write(path, data) + + +def _remove_job(path: Path) -> None: + if path.is_symlink(): + raise OSError("hook job must not be a symlink") + path.unlink(missing_ok=True) + + +def _resume_job( + identifier: str, + *, + runtime_factory: RuntimeFactory = runtime, + authorization_fn: AuthorizationFunction = _authorized, + max_windows: int | None = None, + deadline: float | None = None, + drain: bool = True, +) -> bool: + try: + path, data = _load_job(identifier) + with FileLock(path.with_suffix(".worker.lock"), timeout=0.2): + if not path.exists(): + return False + try: + if not authorization_fn(): + return False + target = _completion_target(data) + if target is not None: + if data.get("partial_record") and data.get("completion_cursor") is None: + data["completion_cursor"] = target + secure_atomic_json_write(path, data) + capture_data = dict(data) + capture_data["target_cursor"] = target + if _capture_transcript( + str(data["event"]), + capture_data, + runtime_factory, + max_windows=max_windows, + deadline=deadline, + lock_timeout=0.2, + drain=drain, + ): + _remove_job(path) + return True + except Exception: # noqa: BLE001 - a durable job must yield to later jobs + pass + if path.exists(): + _schedule_retry(path, data) + except (OSError, ValueError, TimeoutError, json.JSONDecodeError): + return False + return False + + +def process_pending_jobs( + *, + runtime_factory: RuntimeFactory = runtime, + authorization_fn: AuthorizationFunction = _authorized, + maximum: int = 4, + deadline: float | None = None, +) -> int: + """Resume durable capture jobs in the long-lived MCP process.""" + completed = 0 + now = time.time_ns() + candidates: list[tuple[int, int, str]] = [] + try: + scanned = 0 + for path in _jobs_root().iterdir(): + if deadline is not None and time.monotonic() >= deadline: + break + if not path.name.endswith(".json"): + continue + if scanned >= _MAX_JOB_SCAN: + break + scanned += 1 + try: + _loaded_path, data = _load_job(path.stem) + except (OSError, ValueError, json.JSONDecodeError): + continue + candidates.append( + ( + int(data.get("next_attempt_ns", 0)), + int(data.get("rotation", 0)), + path.stem, + ) + ) + except OSError: + return 0 + for next_attempt_ns, _rotation, identifier in sorted(candidates): + if completed >= max(0, maximum): + break + if deadline is not None and time.monotonic() >= deadline: + break + if next_attempt_ns > now: + continue + if _resume_job( + identifier, + runtime_factory=runtime_factory, + authorization_fn=authorization_fn, + deadline=deadline, + drain=False, + ): + completed += 1 + return completed def run( @@ -96,10 +651,11 @@ def run( stdin: TextIO | None = None, stdout: TextIO | None = None, stderr: TextIO | None = None, - runtime_factory: RuntimeFactory = runtime, + runtime_factory: RuntimeFactory = _local_runtime, recall_fn: RecallFunction = recall_block, + authorization_fn: AuthorizationFunction = _authorized, ) -> int: - """Run one hook and always return zero.""" + """Durably admit bounded local work; host operation always continues.""" source = stdin if stdin is not None else sys.stdin sink = stdout if stdout is not None else sys.stdout errors = stderr if stderr is not None else sys.stderr @@ -107,21 +663,47 @@ def run( if event not in _VALID_EVENTS: return 0 data = _read_hook_data(source) + if not _is_primary(data): + return 0 + authorized = authorization_fn() if event == "session-start": - settings = config.resolved(PROVIDER_ID) - block = recall_fn(int(settings["recall_limit"])) + if not authorized: + _bootstrap_onboarding(errors) + return 0 + if event == "user-prompt-submit": + if not authorized or not isinstance(data.get("prompt"), str): + return 0 + block = recall_fn(5, data) if block: sink.write(block) sink.flush() return 0 - _capture_transcript(event, data, runtime_factory) + if not authorized: + return 0 + identifier = _queue_job(event, data) + if identifier is None: + return 0 + deadline = time.monotonic() + _HOOK_LOCAL_BUDGET_SECONDS + if _resume_job( + identifier, + runtime_factory=runtime_factory, + authorization_fn=authorization_fn, + max_windows=1, + deadline=deadline, + drain=False, + ): + return 0 + except (_QueueFullError, TimeoutError): + errors.write( + "claude-code-substrate-memory: continuation queue unavailable; capture not admitted\n" + ) + errors.flush() except Exception as exc: # noqa: BLE001 - hooks must never block Claude Code _debug(type(exc).__name__, errors) return 0 def main(argv: Sequence[str] | None = None) -> int: - """CLI wrapper for ``python -m claude_code_memory.hook ``.""" arguments = list(sys.argv[1:] if argv is None else argv) event = arguments[0] if arguments else "" return run(event) diff --git a/src/claude_code_memory/local_security.py b/src/claude_code_memory/local_security.py new file mode 100644 index 0000000..c73743d --- /dev/null +++ b/src/claude_code_memory/local_security.py @@ -0,0 +1,121 @@ +"""Host-private storage enforcement for Claude Code plugin state.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + + +def secure_windows_tree(root: Path) -> None: + """Replace and verify the Windows DACL with a current-user-only allow list.""" + if os.name != "nt": + return + shell = shutil.which("powershell.exe") or shutil.which("pwsh.exe") or shutil.which("pwsh") + if not shell: + raise OSError("private Windows storage ACL unavailable") + script = r""" +$ErrorActionPreference = 'Stop' +$root = [System.IO.Path]::GetFullPath($env:SUBSTRATE_ACL_ROOT) +$current = [System.Security.Principal.WindowsIdentity]::GetCurrent().User +$fullControl = [System.Security.AccessControl.FileSystemRights]::FullControl +$allow = [System.Security.AccessControl.AccessControlType]::Allow +$none = [System.Security.AccessControl.PropagationFlags]::None +$icacls = Join-Path $env:SystemRoot 'System32\icacls.exe' +if (-not (Test-Path -LiteralPath $icacls -PathType Leaf)) { + throw 'icacls unavailable' +} +function Invoke-Icacls([string[]]$arguments) { + & $icacls @arguments | Out-Null + if ($LASTEXITCODE -ne 0) { + throw 'icacls failed' + } +} +function Test-Private([string]$path, [bool]$directory) { + $acl = Get-Acl -LiteralPath $path + $owner = $acl.GetOwner([System.Security.Principal.SecurityIdentifier]) + $rules = @($acl.GetAccessRules( + $true, $false, [System.Security.Principal.SecurityIdentifier])) + $inheritance = [System.Security.AccessControl.InheritanceFlags]::None + if ($directory) { + $inheritance = [System.Security.AccessControl.InheritanceFlags]'ContainerInherit, ObjectInherit' + } + return ($acl.AreAccessRulesProtected -and + $owner.Value -eq $current.Value -and + $rules.Count -eq 1 -and + $rules[0].IdentityReference.Value -eq $current.Value -and + $rules[0].AccessControlType -eq $allow -and + ($rules[0].FileSystemRights -band $fullControl) -eq $fullControl -and + $rules[0].InheritanceFlags -eq $inheritance -and + $rules[0].PropagationFlags -eq $none) +} +function Protect-One([string]$path, [bool]$directory) { + if (Test-Private $path $directory) { + return + } + $sid = '*' + $current.Value + $grant = $sid + ':F' + if ($directory) { + $grant = $sid + ':(OI)(CI)F' + } + + # Install an explicit lifeline before removing inherited access. Native + # icacls avoids Set-Acl owner/descriptor replacement failures on hosted + # Windows while still changing the real filesystem DACL. + Invoke-Icacls -arguments @($path, '/grant:r', $grant, '/Q') + Invoke-Icacls -arguments @($path, '/inheritance:r', '/Q') + + $before = Get-Acl -LiteralPath $path + $rules = @($before.GetAccessRules( + $true, $false, [System.Security.Principal.SecurityIdentifier])) + $otherSids = @($rules | ForEach-Object { + $_.IdentityReference.Value + } | Where-Object { + $_ -ne $current.Value + } | Select-Object -Unique) + foreach ($otherSid in $otherSids) { + Invoke-Icacls -arguments @($path, '/remove', ('*' + $otherSid), '/Q') + } + $currentDeny = @($rules | Where-Object { + $_.IdentityReference.Value -eq $current.Value -and + $_.AccessControlType -eq [System.Security.AccessControl.AccessControlType]::Deny + }) + if ($currentDeny.Count -ne 0) { + Invoke-Icacls -arguments @($path, '/remove:d', $sid, '/Q') + } + Invoke-Icacls -arguments @($path, '/grant:r', $grant, '/Q') + $owner = $before.GetOwner([System.Security.Principal.SecurityIdentifier]) + if ($owner.Value -ne $current.Value) { + Invoke-Icacls -arguments @($path, '/setowner', $sid, '/Q') + } + + if (-not (Test-Private $path $directory)) { + throw 'ACL verification failed' + } +} +$entries = @(Get-ChildItem -LiteralPath $root -Force -Recurse) +foreach ($entry in $entries) { + if ($entry.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { + throw 'reparse point rejected' + } +} +Protect-One $root $true +foreach ($entry in $entries) { + Protect-One $entry.FullName $entry.PSIsContainer +} +""" + try: + environment = os.environ.copy() + environment["SUBSTRATE_ACL_ROOT"] = str(root) + subprocess.run( + (shell, "-NoProfile", "-NonInteractive", "-Command", script), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + env=environment, + timeout=30, + check=True, + ) + except (OSError, subprocess.SubprocessError): + raise OSError("private Windows storage ACL unavailable") from None diff --git a/src/claude_code_memory/onboarding.py b/src/claude_code_memory/onboarding.py new file mode 100644 index 0000000..793b8d0 --- /dev/null +++ b/src/claude_code_memory/onboarding.py @@ -0,0 +1,536 @@ +"""Fixed-origin, resumable RFC 8628 onboarding for Claude Code.""" + +from __future__ import annotations + +import json +import subprocess +import os +import threading +import time +import webbrowser +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from urllib.request import HTTPRedirectHandler, OpenerDirector, Request, build_opener + +from substrate_capture import SubstrateAPIError, secure_atomic_json_write + +from .contract import HOSTED_ORIGIN, OAUTH_CLIENT_ID, OAUTH_SCOPES, VERSION +from .credentials import CredentialStore, credential_store +from .strict_client import StrictHostedClient + +CLIENT_ID = OAUTH_CLIENT_ID +SCOPES = OAUTH_SCOPES +DEVICE_GRANT = "urn:ietf:params:oauth:grant-type:device_code" +_MAX_RESPONSE_BYTES = 64 * 1024 +_STATE_VERSION = 1 +_TERMINAL_PHASES = frozenset({"ready", "declined", "failed", "repair_required"}) +_SAFE_STATE_FIELDS = frozenset( + { + "state_version", + "phase", + "hosted_origin", + "verification_uri", + "verification_uri_complete", + "user_code", + "expires_at", + "interval", + "updated_at", + "error_class", + "connected_at", + "disconnected_at", + "history_decision", + "history_decided_at", + } +) +_SAFE_OAUTH_ERRORS = frozenset( + { + "access_denied", + "authorization_pending", + "expired_token", + "invalid_client", + "invalid_grant", + "invalid_request", + "invalid_scope", + "slow_down", + "unsupported_grant_type", + } +) +_TRANSIENT_HTTP = frozenset({408, 425, 429, 500, 502, 503, 504}) + + +class OnboardingError(RuntimeError): + """A sanitized onboarding failure.""" + + def __init__(self, category: str) -> None: + self.category = category + super().__init__(category) + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request( + self, + req: Request, + fp: Any, + code: int, + msg: str, + headers: Any, + newurl: str, + ) -> None: + return None + + +def _hosted_url(value: Any) -> str: + if not isinstance(value, str) or not value or len(value) > 4096: + raise OnboardingError("invalid_response") + parsed = urlsplit(value) + if ( + f"{parsed.scheme}://{parsed.netloc}" != HOSTED_ORIGIN + or parsed.username is not None + or parsed.password is not None + or parsed.fragment + ): + raise OnboardingError("invalid_response") + return value + + +def _oauth_error(status: int, value: dict[str, Any]) -> str: + if status in _TRANSIENT_HTTP: + return f"http_{status}" + error = value.get("error") + return error if isinstance(error, str) and error in _SAFE_OAUTH_ERRORS else "invalid_response" + + +class HostedOAuthClient: + """Small no-redirect client pinned to the two hosted RFC 8628 endpoints.""" + + def __init__( + self, *, timeout: float = 60.0, opener: OpenerDirector | Any | None = None + ) -> None: + self.timeout = max(1.0, min(float(timeout), 60.0)) + self._opener = opener or build_opener(_NoRedirect()) + + def _post(self, path: str, values: dict[str, str]) -> tuple[int, dict[str, Any]]: + if path not in {"/oauth/device_authorization", "/oauth/token"}: + raise OnboardingError("invalid_request") + request = Request( + HOSTED_ORIGIN + path, + data=urlencode(values).encode("ascii"), + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": f"claude-code-substrate-memory/{VERSION}", + }, + ) + try: + response = self._opener.open(request, timeout=self.timeout) + except HTTPError as exc: + response = exc + except (URLError, OSError, TimeoutError): + raise OnboardingError("transport_error") from None + try: + status = int(response.status) + headers = response.headers + content_type = ( + headers.get_content_type() + if callable(getattr(headers, "get_content_type", None)) + else str(headers.get("Content-Type", "")).split(";", 1)[0].strip().lower() + ) + raw = response.read(_MAX_RESPONSE_BYTES + 1) + finally: + response.close() + if len(raw) > _MAX_RESPONSE_BYTES: + raise OnboardingError("response_too_large") + if content_type != "application/json": + raise OnboardingError("invalid_content_type") + try: + value = json.loads(raw.decode("utf-8", errors="strict")) + except (UnicodeError, json.JSONDecodeError): + raise OnboardingError("invalid_response") from None + if not isinstance(value, dict): + raise OnboardingError("invalid_response") + return status, value + + def begin(self) -> dict[str, Any]: + status, value = self._post( + "/oauth/device_authorization", {"client_id": CLIENT_ID, "scope": SCOPES} + ) + if status != 200: + raise OnboardingError(_oauth_error(status, value)) + device_code = value.get("device_code") + user_code = value.get("user_code") + if not isinstance(device_code, str) or not 0 < len(device_code) <= 4096: + raise OnboardingError("invalid_response") + if not isinstance(user_code, str) or not 0 < len(user_code) <= 64: + raise OnboardingError("invalid_response") + verification_uri = _hosted_url(value.get("verification_uri")) + complete_value = value.get("verification_uri_complete") + if complete_value: + complete = _hosted_url(complete_value) + query = dict(parse_qsl(urlsplit(complete).query, keep_blank_values=True)) + if query.get("user_code") != user_code: + raise OnboardingError("invalid_response") + else: + parsed = urlsplit(verification_uri) + query_items = [ + (key, item) + for key, item in parse_qsl(parsed.query, keep_blank_values=True) + if key != "user_code" + ] + query_items.append(("user_code", user_code)) + complete = urlunsplit(parsed._replace(query=urlencode(query_items))) + expires = value.get("expires_in") + interval = value.get("interval", 5) + if not isinstance(expires, int) or isinstance(expires, bool): + raise OnboardingError("invalid_response") + if not isinstance(interval, int) or isinstance(interval, bool): + raise OnboardingError("invalid_response") + return { + "device_code": device_code, + "user_code": user_code, + "verification_uri": verification_uri, + "verification_uri_complete": complete, + "expires_in": max(1, min(expires, 3600)), + "interval": max(1, min(interval, 60)), + } + + def poll(self, device_code: str) -> dict[str, Any]: + status, value = self._post( + "/oauth/token", + { + "grant_type": DEVICE_GRANT, + "device_code": device_code, + "client_id": CLIENT_ID, + }, + ) + if status == 200: + token = value.get("access_token") + scope = str(value.get("scope", SCOPES)).split() + if ( + not isinstance(token, str) + or not 0 < len(token) <= 16384 + or str(value.get("token_type", "")).casefold() != "bearer" + or set(scope) != set(SCOPES.split()) + ): + raise OnboardingError("invalid_response") + return {"status": "approved", "access_token": token} + error = _oauth_error(status, value) + if error in {"authorization_pending", "slow_down", "access_denied", "expired_token"}: + return {"status": error} + raise OnboardingError(error) + + +def validate_hosted_credential(token: str) -> None: + """Validate the exact full capture and canonical-entity contract.""" + try: + StrictHostedClient(HOSTED_ORIGIN, token, timeout=60.0).require_capabilities() + except SubstrateAPIError as exc: + raise OnboardingError(exc.category) from None + + +def _empty_state() -> dict[str, Any]: + return { + "state_version": _STATE_VERSION, + "phase": "new", + "hosted_origin": HOSTED_ORIGIN, + "updated_at": time.time(), + } + + +class OnboardingManager: + """Content-free, resumable hosted onboarding for one private Claude profile.""" + + def __init__( + self, + home: Path, + *, + api: HostedOAuthClient | None = None, + store: CredentialStore | None = None, + capability_check: Callable[[str], None] | None = None, + browser_open: Callable[[str], bool] | None = None, + ) -> None: + self.home = home.resolve() + self.root = self.home / "onboarding" + if self.root.exists() and self.root.is_symlink(): + raise OSError("onboarding directory must not be a symlink") + self.root.mkdir(parents=True, exist_ok=True, mode=0o700) + if os.name == "posix": + os.chmod(self.root, 0o700) + self.path = self.root / "state.json" + self.api = api or HostedOAuthClient() + self.store = store or credential_store(self.home) + self.capability_check = capability_check or validate_hosted_credential + self.browser_open = browser_open or webbrowser.open + self._mutex = threading.RLock() + + @contextmanager + def _lock(self) -> Iterator[None]: + with self._mutex: + lock_path = self.root / ".state.lock" + if lock_path.is_symlink(): + raise OSError("onboarding lock must not be a symlink") + handle = lock_path.open("a+b") + try: + if os.name == "posix": + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + elif os.name == "nt": + import msvcrt + + if lock_path.stat().st_size == 0: + handle.write(b"\0") + handle.flush() + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + yield + finally: + if os.name == "posix": + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + elif os.name == "nt": + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + handle.close() + + def _load(self) -> dict[str, Any]: + if self.path.is_symlink(): + raise OSError("onboarding state must not be a symlink") + try: + value = json.loads(self.path.read_text(encoding="utf-8")) + except FileNotFoundError: + return _empty_state() + except (OSError, UnicodeError, json.JSONDecodeError): + return {**_empty_state(), "phase": "repair_required", "error_class": "state_corrupt"} + if ( + not isinstance(value, dict) + or value.get("state_version") != _STATE_VERSION + or value.get("hosted_origin") != HOSTED_ORIGIN + ): + return { + **_empty_state(), + "phase": "repair_required", + "error_class": "state_incompatible", + } + return {key: value[key] for key in _SAFE_STATE_FIELDS if key in value} + + def _save(self, state: dict[str, Any]) -> None: + clean = {key: state[key] for key in _SAFE_STATE_FIELDS if key in state} + clean.update( + state_version=_STATE_VERSION, hosted_origin=HOSTED_ORIGIN, updated_at=time.time() + ) + secure_atomic_json_write(self.path, clean) + + def _status_unlocked(self) -> dict[str, Any]: + state = self._load() + result = { + key: state[key] + for key in ( + "phase", + "hosted_origin", + "verification_uri", + "verification_uri_complete", + "user_code", + "expires_at", + "interval", + "error_class", + "connected_at", + "disconnected_at", + "history_decision", + "history_decided_at", + ) + if key in state + } + phase = result.get("phase") + result["authenticated"] = phase in {"ready", "awaiting_history_consent"} and bool( + self.store.get() + ) + result["credential_backend"] = self.store.backend + result["ready"] = phase == "ready" and result["authenticated"] + result["history_import"] = "not_supported" + return result + + def status(self) -> dict[str, Any]: + with self._lock(): + return self._status_unlocked() + + def begin(self, *, open_browser: bool = True) -> dict[str, Any]: + with self._lock(): + state = self._load() + if state.get("phase") == "repair_required" and self.store.get(): + return self._status_unlocked() + if self.store.get(): + if state.get("phase") not in {"ready", "awaiting_history_consent"}: + state.update(phase="awaiting_history_consent", connected_at=time.time()) + state.pop("error_class", None) + self._save(state) + return self._status_unlocked() + if ( + state.get("phase") == "authorization_pending" + and float(state.get("expires_at", 0)) > time.time() + and self.store.get("onboarding-device") + ): + return self._status_unlocked() + grant = self.api.begin() + device_code = str(grant.pop("device_code")) + self.store.put(device_code, "onboarding-device") + expires_at = time.time() + int(grant.pop("expires_in")) + state = { + **_empty_state(), + **grant, + "phase": "authorization_pending", + "expires_at": expires_at, + } + self._save(state) + if open_browser: + try: + self.browser_open(str(state["verification_uri_complete"])) + except (OSError, webbrowser.Error): + pass + return self._status_unlocked() + + def poll(self) -> dict[str, Any]: + with self._lock(): + state = self._load() + if state.get("phase") != "authorization_pending": + return self._status_unlocked() + if float(state.get("expires_at", 0)) <= time.time(): + self.store.delete("onboarding-device") + state.update(phase="failed", error_class="authorization_expired") + self._save(state) + return self._status_unlocked() + device_code = self.store.get("onboarding-device") + if not device_code: + state.update(phase="repair_required", error_class="missing_device_credential") + self._save(state) + return self._status_unlocked() + response = self.api.poll(device_code) + poll_status = response["status"] + if poll_status in {"authorization_pending", "slow_down"}: + state.pop("error_class", None) + if poll_status == "slow_down": + state["interval"] = min(60, int(state.get("interval", 5)) + 5) + self._save(state) + return self._status_unlocked() + if poll_status in {"access_denied", "expired_token"}: + self.store.delete("onboarding-device") + state.update( + phase="declined" if poll_status == "access_denied" else "failed", + error_class=poll_status, + ) + self._save(state) + return self._status_unlocked() + token = str(response["access_token"]) + try: + self.capability_check(token) + self.store.put(token) + except (OnboardingError, SubstrateAPIError, OSError): + self.store.delete("onboarding-device") + state.update(phase="failed", error_class="capability_check_failed") + self._save(state) + return self._status_unlocked() + self.store.delete("onboarding-device") + state.update(phase="awaiting_history_consent", connected_at=time.time()) + state.pop("error_class", None) + self._save(state) + return self._status_unlocked() + + def poll_until_terminal( + self, + *, + timeout: float = 900.0, + sleep: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.monotonic, + ) -> dict[str, Any]: + deadline = clock() + max(1.0, min(float(timeout), 900.0)) + status = self.status() + while status.get("phase") == "authorization_pending" and clock() < deadline: + sleep(max(1.0, min(float(status.get("interval", 5)), 60.0))) + try: + status = self.poll() + except OnboardingError as exc: + if exc.category not in { + "transport_error", + *(f"http_{code}" for code in _TRANSIENT_HTTP), + }: + raise + return status + + def consent_history(self, approved: bool) -> dict[str, Any]: + """Persist direct-human history consent before any future discovery. + + Claude Code has no reviewed safe history discovery API, so this release + starts no import even when preference is approved. + """ + if not isinstance(approved, bool): + raise TypeError("approved must be bool") + with self._lock(): + state = self._load() + if not self.store.get() or state.get("phase") not in { + "awaiting_history_consent", + "ready", + }: + raise OnboardingError("not_authenticated") + state.update( + phase="ready", + history_decision="approved" if approved else "declined", + history_decided_at=time.time(), + ) + self._save(state) + return self._status_unlocked() + + def require_repair(self, category: str = "authentication_rejected") -> dict[str, Any]: + with self._lock(): + state = self._load() + state.update(phase="repair_required", error_class=str(category)[:64]) + self._save(state) + try: + self.store.delete() + except (OSError, subprocess.SubprocessError): + state["error_class"] = "credential_delete_failed" + self._save(state) + return self._status_unlocked() + + def repair(self, *, open_browser: bool = True) -> dict[str, Any]: + with self._lock(): + state = self._load() + state.update(phase="repair_required", error_class="repair_in_progress") + self._save(state) + try: + self.store.delete() + self.store.delete("onboarding-device") + except (OSError, subprocess.SubprocessError): + state["error_class"] = "credential_delete_failed" + self._save(state) + return self._status_unlocked() + if self.store.get() or self.store.get("onboarding-device"): + state["error_class"] = "credential_delete_failed" + self._save(state) + return self._status_unlocked() + self._save(_empty_state()) + return self.begin(open_browser=open_browser) + + def disconnect_local(self) -> dict[str, Any]: + """Reviewed extension: remove local secrets but make no revoke claim.""" + with self._lock(): + self.store.delete() + self.store.delete("onboarding-device") + state = _empty_state() + state["disconnected_at"] = time.time() + self._save(state) + return self._status_unlocked() + + +__all__ = [ + "CLIENT_ID", + "DEVICE_GRANT", + "HOSTED_ORIGIN", + "HostedOAuthClient", + "OnboardingError", + "OnboardingManager", + "SCOPES", + "validate_hosted_credential", +] diff --git a/src/claude_code_memory/profile.py b/src/claude_code_memory/profile.py new file mode 100644 index 0000000..1d2a339 --- /dev/null +++ b/src/claude_code_memory/profile.py @@ -0,0 +1,55 @@ +"""Private profile state and stable tenant-local identity.""" + +from __future__ import annotations + +import hashlib +import os +import stat +from pathlib import Path + +from substrate_capture import config +from substrate_capture.spool import validate_no_symlink_ancestors + +from .contract import PROVIDER_ID +from .local_security import secure_windows_tree + + +def _claude_profile_path() -> Path: + configured = os.environ.get("CLAUDE_CONFIG_DIR", "").strip() + return (Path(configured).expanduser() if configured else Path.home() / ".claude").absolute() + + +def profile_key() -> str: + """Return a non-reversible key for this Claude profile.""" + canonical = os.path.normcase(os.path.normpath(str(_claude_profile_path()))) + return hashlib.sha256(("claude-profile\0" + canonical).encode("utf-8")).hexdigest()[:24] + + +def state_home() -> Path: + """Return the private, provider- and Claude-profile-scoped state directory.""" + root = validate_no_symlink_ancestors( + config.state_home(PROVIDER_ID) / "profiles" / profile_key() + ) + root.mkdir(parents=True, exist_ok=True, mode=0o700) + validate_no_symlink_ancestors(root) + if os.name == "posix": + os.chmod(root, stat.S_IRWXU) + elif os.name == "nt": + secure_windows_tree(root) + return root + + +def tenant_identity() -> str: + """Return one stable tenant-local subject without exposing a local path.""" + return hashlib.sha256((PROVIDER_ID + "\0" + profile_key()).encode("ascii")).hexdigest()[:24] + + +def capture_scope() -> dict[str, str]: + identity = tenant_identity() + return { + "platform": "claude_code", + "user_id": identity, + "agent_id": identity, + "agent_identity": identity, + "profile": profile_key(), + } diff --git a/src/claude_code_memory/recall.py b/src/claude_code_memory/recall.py index c245b01..2626d2d 100644 --- a/src/claude_code_memory/recall.py +++ b/src/claude_code_memory/recall.py @@ -1,18 +1,35 @@ -"""Automatic, bounded Substrate recall for Claude Code session startup.""" +"""Bounded canonical-only automatic recall for Claude SessionStart.""" from __future__ import annotations +import re import threading import time -from pathlib import Path from typing import Any from . import runtime +from .profile import capture_scope _RECALL_BUDGET_SECONDS = 2.5 _REQUEST_TIMEOUT_SECONDS = 0.9 _MAX_BLOCK_CHARS = 6000 _MAX_CARD_CHARS = 700 +_ENTITY_TYPES = frozenset( + { + "person", + "agent", + "organization", + "project", + "product", + "place", + "event", + "other", + "system", + "service", + "automation", + } +) +_ENTITY_FILENAME = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,79})--[0-9a-f]{8}\.md") def _compact(value: Any, maximum: int) -> str: @@ -21,21 +38,57 @@ def _compact(value: Any, maximum: int) -> str: return " ".join(value.split())[:maximum] -def _recall_once(limit: int, deadline: float) -> str: +def _canonical_path(item: dict[str, Any]) -> str: + path = item.get("canonical_path") + entity_id = item.get("entity_id") + entity_type = item.get("entity_type") + card = item.get("memory_card") + if ( + item.get("page_type") != "entity" + or item.get("quality_version") != 2 + or not isinstance(card, str) + or not card.strip() + or not isinstance(path, str) + or not isinstance(entity_id, str) + or not entity_id.strip() + or len(entity_id) > 256 + or not isinstance(entity_type, str) + or entity_type not in _ENTITY_TYPES + or "\\" in path + ): + return "" + parts = path.split("/") + if ( + len(parts) != 3 + or parts[0] != "entities" + or parts[1] != entity_type + or _ENTITY_FILENAME.fullmatch(parts[2]) is None + ): + return "" + return path + + +def _query_context(data: dict[str, Any]) -> str: + """Use only the documented UserPromptSubmit prompt value.""" + return _compact(data.get("prompt"), 1200) + + +def _recall_once(limit: int, data: dict[str, Any], deadline: float) -> str: requested = limit if isinstance(limit, int) and not isinstance(limit, bool) else 5 bounded_limit = min(25, max(1, requested)) + query = _query_context(data) + if not query: + return "" client, _spool, _deliverer, _builder = runtime() if client is None or time.monotonic() >= deadline: return "" - original_timeout = client.timeout client.timeout = min(float(client.timeout), _REQUEST_TIMEOUT_SECONDS) try: - project = Path.cwd().name[:128] or "current project" response = client.memory_search( - f"Relevant context, decisions, and preferences for {project}", + query, limit=bounded_limit, - scope={"platform": "claude_code", "agent_id": "default"}, + scope=capture_scope(), ) finally: client.timeout = original_timeout @@ -44,23 +97,19 @@ def _recall_once(limit: int, deadline: float) -> str: results = response.get("results") if not isinstance(results, list) or not results: return "" - lines = ["## Recalled Substrate context"] + seen: set[str] = set() for item in results[:bounded_limit]: if time.monotonic() >= deadline or not isinstance(item, dict): break - title = _compact(item.get("title"), 160) or _compact(item.get("path"), 160) - card = ( - _compact(item.get("memory_card"), _MAX_CARD_CHARS) - or _compact(item.get("summary"), _MAX_CARD_CHARS) - or _compact(item.get("snippet"), _MAX_CARD_CHARS) - ) - if not card: + path = _canonical_path(item) + entity_id = str(item.get("entity_id") or "").strip() + card = _compact(item.get("memory_card"), _MAX_CARD_CHARS) + if not path or not card or entity_id in seen: continue - path = _compact(item.get("canonical_path") or item.get("path"), 240) - label = title or "Memory" - citation = f" (`{path}`)" if path else "" - lines.append(f"- **{label}**{citation}: {card}") + title = _compact(item.get("title"), 160) or entity_id[:160] + lines.append(f"- **{title}** (`{path}`): {card}") + seen.add(entity_id) if sum(len(line) + 1 for line in lines) >= _MAX_BLOCK_CHARS: break if len(lines) == 1: @@ -68,15 +117,15 @@ def _recall_once(limit: int, deadline: float) -> str: return "\n".join(lines)[:_MAX_BLOCK_CHARS].rstrip() + "\n" -def recall_block(limit: int) -> str: - """Return a compact Markdown memory block within a hard wall-clock budget.""" +def recall_block(limit: int, data: dict[str, Any] | None = None) -> str: + """Return canonical memory cards within a hard wall-clock budget.""" deadline = time.monotonic() + _RECALL_BUDGET_SECONDS result: list[str] = [] def worker() -> None: try: - result.append(_recall_once(limit, deadline)) - except Exception: # noqa: BLE001 - recall must never interrupt session startup + result.append(_recall_once(limit, data or {}, deadline)) + except Exception: # noqa: BLE001 - recall must never interrupt startup result.append("") thread = threading.Thread(target=worker, name="substrate-recall", daemon=True) diff --git a/src/claude_code_memory/server.py b/src/claude_code_memory/server.py index 873c377..f4704ae 100644 --- a/src/claude_code_memory/server.py +++ b/src/claude_code_memory/server.py @@ -1,23 +1,61 @@ -"""Claude Code stdio MCP server entry point.""" +"""Claude Code stdio MCP server and long-lived local repair worker.""" from __future__ import annotations -from substrate_capture import Server, build_toolset, serve +import threading +import time + +from substrate_capture import Server, serve from . import __version__, runtime +from .hook import process_pending_jobs +from .tools import build_tools + + +def _background_worker( + stop: threading.Event, + runtime_value: tuple[object, object, object, object], +) -> None: + """Drain and resume durable work only while the MCP server is alive.""" + _client, _spool, deliverer, _builder = runtime_value + while not stop.is_set(): + cycle_deadline = time.monotonic() + 2.0 + try: + deliverer.drain() # type: ignore[attr-defined] + process_pending_jobs( + runtime_factory=lambda: runtime_value, # type: ignore[arg-type,return-value] + maximum=4, + deadline=cycle_deadline, + ) + deliverer.drain() # type: ignore[attr-defined] + except Exception: # noqa: BLE001 - durable state is retried next cycle/restart + pass + stop.wait(0.5) def main() -> int: - """Run the seven-tool Substrate MCP server over stdio.""" - client, _spool, deliverer, builder = runtime() - tools = build_toolset( - client=client, - deliverer=deliverer, - builder=builder, - session_id="", - status=deliverer.status, + """Run JSON-RPC plus its bounded in-process continuation worker.""" + runtime_value = runtime() + client, _spool, deliverer, builder = runtime_value + stop = threading.Event() + worker = threading.Thread( + target=_background_worker, + args=(stop, runtime_value), + name="substrate-claude-mcp-worker", + daemon=True, ) - return serve(Server("substrate-claude-code", __version__, tools)) + worker.start() + try: + return serve( + Server( + "substrate-claude-code", + __version__, + build_tools(client=client, deliverer=deliverer, builder=builder), + ) + ) + finally: + stop.set() + worker.join(timeout=0.25) if __name__ == "__main__": diff --git a/src/claude_code_memory/state.py b/src/claude_code_memory/state.py new file mode 100644 index 0000000..df1e001 --- /dev/null +++ b/src/claude_code_memory/state.py @@ -0,0 +1,200 @@ +"""Cross-process transactions for hook cursors and durable operational state.""" + +from __future__ import annotations + +import json +import os +import stat +import threading +import time +from contextlib import AbstractContextManager +from pathlib import Path +from typing import Any + +from substrate_capture import secure_atomic_json_write, session_key + +try: + import fcntl +except ImportError: # pragma: no cover - Windows only + fcntl = None # type: ignore[assignment] + import msvcrt +else: # pragma: no cover - keeps the Windows name explicit + msvcrt = None # type: ignore[assignment] + +_MAX_STATE_BYTES = 64 * 1024 + + +def _try_lock(descriptor: int) -> bool: + if fcntl is not None: + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return False + return True + if os.fstat(descriptor).st_size == 0: + os.write(descriptor, b"\0") + os.fsync(descriptor) + os.lseek(descriptor, 0, os.SEEK_SET) + try: + msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1) + except OSError: + return False + return True + + +def _unlock(descriptor: int) -> None: + if fcntl is not None: + fcntl.flock(descriptor, fcntl.LOCK_UN) + return + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + + +class FileLock(AbstractContextManager["FileLock"]): + """A symlink-safe owner-private inter-process lock file.""" + + def __init__(self, path: Path, *, timeout: float = 2.0) -> None: + self.path = path + self.timeout = max(0.0, min(float(timeout), 30.0)) + self._descriptor: int | None = None + self._thread_lock = threading.Lock() + + def __enter__(self) -> "FileLock": + deadline = time.monotonic() + self.timeout + if not self._thread_lock.acquire(timeout=self.timeout): + raise TimeoutError("lock acquisition timed out") + descriptor: int | None = None + try: + self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if self.path.is_symlink(): + raise OSError("lock must not be a symlink") + flags = os.O_RDWR | os.O_CREAT + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(self.path, flags, 0o600) + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode): + raise OSError("lock must be a regular file") + if os.name == "posix": + os.chmod(self.path, 0o600) + while not _try_lock(descriptor): + if time.monotonic() >= deadline: + raise TimeoutError("lock acquisition timed out") + time.sleep(0.01) + self._descriptor = descriptor + return self + except Exception: + if descriptor is not None: + os.close(descriptor) + self._thread_lock.release() + raise + + def __exit__(self, *args: object) -> None: + descriptor = self._descriptor + self._descriptor = None + try: + if descriptor is not None: + _unlock(descriptor) + os.close(descriptor) + finally: + self._thread_lock.release() + + +class StateCorruptError(OSError): + """A state file was present but unsafe or unreadable.""" + + +class SessionTransaction(AbstractContextManager["SessionTransaction"]): + """Serialize one session's read/admit/cursor-commit transaction.""" + + def __init__(self, root: Path, session_id: str, *, lock_timeout: float = 2.0) -> None: + key = session_key(session_id) + self.root = root + self.path = root / f"{key}.json" + self.lock = FileLock(root / f".{key}.lock", timeout=lock_timeout) + self.state: dict[str, Any] = {} + self._entered = False + + def __enter__(self) -> "SessionTransaction": + self.lock.__enter__() + self._entered = True + self.state = self._load() + return self + + def __exit__(self, *args: object) -> None: + self._entered = False + self.lock.__exit__(*args) + + def _load(self) -> dict[str, Any]: + if self.path.is_symlink(): + raise StateCorruptError("session state symlink") + try: + info = self.path.stat(follow_symlinks=False) + except FileNotFoundError: + return { + "version": 1, + "cursor": 0, + "message_index": 0, + "source_id": "", + "anchor": "", + "session_end": False, + "loss_count": 0, + } + if not stat.S_ISREG(info.st_mode) or info.st_size > _MAX_STATE_BYTES: + raise StateCorruptError("invalid session state") + try: + value = json.loads(self.path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise StateCorruptError("unreadable session state") from exc + if not isinstance(value, dict) or value.get("version") != 1: + raise StateCorruptError("incompatible session state") + cursor = value.get("cursor") + index = value.get("message_index") + loss = value.get("loss_count", 0) + source_id = value.get("source_id", "") + anchor = value.get("anchor", "") + ended = value.get("session_end", False) + if ( + not isinstance(cursor, int) + or isinstance(cursor, bool) + or cursor < 0 + or not isinstance(index, int) + or isinstance(index, bool) + or index < 0 + or not isinstance(loss, int) + or isinstance(loss, bool) + or loss < 0 + or not isinstance(source_id, str) + or len(source_id) > 128 + or not isinstance(anchor, str) + or len(anchor) > 128 + or not isinstance(ended, bool) + ): + raise StateCorruptError("invalid session state fields") + return { + "version": 1, + "cursor": cursor, + "message_index": index, + "source_id": source_id, + "anchor": anchor, + "session_end": ended, + "loss_count": loss, + } + + def commit(self, state: dict[str, Any]) -> None: + if not self._entered: + raise RuntimeError("transaction is not active") + selected = { + "version": 1, + "cursor": int(state["cursor"]), + "message_index": int(state["message_index"]), + "source_id": str(state["source_id"])[:128], + "anchor": str(state.get("anchor", ""))[:128], + "session_end": bool(state.get("session_end", False)), + "loss_count": int(state.get("loss_count", 0)), + } + secure_atomic_json_write(self.path, selected) + self.state = selected + + +__all__ = ["FileLock", "SessionTransaction", "StateCorruptError"] diff --git a/src/claude_code_memory/strict_client.py b/src/claude_code_memory/strict_client.py new file mode 100644 index 0000000..a35991a --- /dev/null +++ b/src/claude_code_memory/strict_client.py @@ -0,0 +1,122 @@ +"""Fixed-origin client with the full Hermes-authoritative contract gate.""" + +from __future__ import annotations + +from typing import Any + +from substrate_capture import SubstrateAPIError, SubstrateClient + +from .contract import PROVIDER_ID + + +def _strict_semver(value: Any) -> tuple[int, int, int] | None: + if not isinstance(value, str): + return None + parts = value.split(".") + if ( + len(parts) != 3 + or any(not part.isascii() or not part.isdigit() for part in parts) + or any(len(part) > 1 and part.startswith("0") for part in parts) + ): + return None + parsed = tuple(int(part) for part in parts) + if any(part > 1_000_000 for part in parsed): + return None + return parsed # type: ignore[return-value] + + +def validate_capabilities(capabilities: dict[str, Any]) -> None: + """Reject every partial, foreign, legacy, or too-new shared contract.""" + providers = capabilities.get("providers") + versions = capabilities.get("capture_schema_versions") + replay = capabilities.get("history_replay") + entity = capabilities.get("entity_memory") + quality = capabilities.get("entity_quality") + replay_min = ( + _strict_semver(replay.get("min_plugin_version")) if isinstance(replay, dict) else None + ) + entity_min = ( + _strict_semver(entity.get("min_plugin_version")) if isinstance(entity, dict) else None + ) + quality_min = ( + _strict_semver(quality.get("min_plugin_version")) if isinstance(quality, dict) else None + ) + valid = ( + isinstance(providers, list) + and PROVIDER_ID in providers + and isinstance(versions, list) + and 2 in versions + and capabilities.get("max_event_bytes") == 262_144 + and isinstance(replay, dict) + and replay.get("protocol") == "stream-v2" + and replay_min is not None + and replay.get("content_free_completion") is True + and replay.get("incremental_windows") is True + and replay.get("status_version") == 2 + and isinstance(entity, dict) + and entity.get("protocol") == "entity-wiki-v1" + and entity_min is not None + and entity.get("search_endpoint") == "/api/v1/hermes/memory/search" + and entity.get("canonical_wiki_pages") is True + and entity.get("entity_page_type") == "entity" + and isinstance(quality, dict) + and quality.get("protocol") == "entity-quality-v2" + and quality_min is not None + and quality.get("memory_card") is True + and quality.get("quality_version") == 2 + and quality.get("canonical_redirects") is True + ) + if not valid: + raise SubstrateAPIError("server_upgrade_required") + + +class StrictHostedClient(SubstrateClient): + """A client that negotiates the complete contract before every capability use.""" + + def __post_init__(self) -> None: + super().__post_init__() + self._capability_profile = None + + def require_capabilities(self) -> None: + if self._capability_profile == {"strict": True}: + return + value = super().request("GET", "/api/v1/hermes/capabilities") + if not isinstance(value, dict): + raise SubstrateAPIError("invalid_response") + validate_capabilities(value) + self._capability_profile = {"strict": True} + + def request( + self, + method: str, + path: str, + *, + query: dict[str, Any] | None = None, + body: dict[str, Any] | None = None, + idempotency_key: str | None = None, + ) -> Any: + if path != "/api/v1/hermes/capabilities": + self.require_capabilities() + return super().request( + method, + path, + query=query, + body=body, + idempotency_key=idempotency_key, + ) + + def capabilities(self) -> dict[str, Any]: + value = super().request("GET", "/api/v1/hermes/capabilities") + if not isinstance(value, dict): + raise SubstrateAPIError("invalid_response") + validate_capabilities(value) + self._capability_profile = {"strict": True} + return value + + def capability_profile(self) -> dict[str, Any]: + self.require_capabilities() + return { + "memory_search": True, + "server_provider": PROVIDER_ID, + "schema_versions": [2], + } diff --git a/src/claude_code_memory/tools.py b/src/claude_code_memory/tools.py new file mode 100644 index 0000000..6af113c --- /dev/null +++ b/src/claude_code_memory/tools.py @@ -0,0 +1,166 @@ +"""Claude MCP adapter: five bounded network tools plus explicit operations.""" + +from __future__ import annotations + +from typing import Any + +from substrate_capture import CaptureEventBuilder, SubstrateAPIError, Tool + +from .delivery import HostedDeliverer +from .strict_client import StrictHostedClient + +_MAX_QUERY_CHARS = 4096 +_MAX_CONTENT_CHARS = 262_144 +_MAX_QUESTION_CHARS = 16_384 +_MAX_JOB_ID_CHARS = 512 + + +def build_tools( + *, + client: StrictHostedClient | None, + deliverer: HostedDeliverer, + builder: CaptureEventBuilder, +) -> list[Tool]: + def require_client() -> StrictHostedClient: + if client is None: + raise SubstrateAPIError("not_configured") + return client + + def guard(action: Any) -> Any: + try: + return action() + except SubstrateAPIError as exc: + return {"error": exc.category} + except (KeyError, TypeError, ValueError): + return {"error": "invalid_arguments"} + + def text(args: dict[str, Any], key: str, maximum: int = _MAX_QUERY_CHARS) -> str: + value = args.get(key) + if not isinstance(value, str) or not value.strip() or len(value) > maximum: + raise ValueError(key) + return value + + def limit(args: dict[str, Any], default: int = 8) -> int: + value = args.get("limit", default) + if not isinstance(value, int) or isinstance(value, bool): + return default + return max(1, min(25, value)) + + def source_type(args: dict[str, Any]) -> str: + value = args.get("source_type", "text") + if value not in {"text", "url"}: + raise ValueError("source_type") + return str(value) + + def search(args: dict[str, Any]) -> Any: + def action() -> dict[str, Any]: + value = require_client().search(text(args, "query"), limit=limit(args)) + if isinstance(value, dict): + return value + if isinstance(value, list): + return {"results": value} + raise ValueError("invalid search response") + + return guard(action) + + def read(args: dict[str, Any]) -> Any: + return guard(lambda: require_client().read_page(text(args, "path"))) + + def query(args: dict[str, Any]) -> Any: + def action() -> Any: + save = args.get("save_as_synthesis", False) + if not isinstance(save, bool): + raise ValueError("save_as_synthesis") + return require_client().query_wiki( + text(args, "question", _MAX_QUESTION_CHARS), + save_as_synthesis=save, + ) + + return guard(action) + + def ingest(args: dict[str, Any]) -> Any: + return guard( + lambda: require_client().ingest( + text(args, "content", _MAX_CONTENT_CHARS), + title=(text(args, "title", 512) if args.get("title") is not None else None), + source_type=source_type(args), + ) + ) + + def job_status(args: dict[str, Any]) -> Any: + return guard(lambda: require_client().job_status(text(args, "job_id", _MAX_JOB_ID_CHARS))) + + def sync(_args: dict[str, Any]) -> dict[str, Any]: + return {"ok": True, **deliverer.drain()} + + def status(_args: dict[str, Any]) -> dict[str, Any]: + return {"ok": True, **deliverer.status()} + + def obj(properties: dict[str, Any], required: list[str]) -> dict[str, Any]: + return { + "type": "object", + "properties": properties, + "required": required, + "additionalProperties": False, + } + + return [ + Tool( + "substrate_search", + "Search validated canonical Substrate memory cards.", + obj( + { + "query": {"type": "string", "maxLength": _MAX_QUERY_CHARS}, + "limit": {"type": "integer", "minimum": 1, "maximum": 25, "default": 8}, + }, + ["query"], + ), + search, + ), + Tool( + "substrate_read", + "Read one bounded wiki page by repository-relative path.", + obj({"path": {"type": "string", "maxLength": _MAX_QUERY_CHARS}}, ["path"]), + read, + ), + Tool( + "substrate_query", + "Ask one bounded question over Substrate memory.", + obj( + { + "question": {"type": "string", "maxLength": _MAX_QUESTION_CHARS}, + "save_as_synthesis": {"type": "boolean", "default": False}, + }, + ["question"], + ), + query, + ), + Tool( + "substrate_ingest", + "Submit explicit text for bounded asynchronous wiki ingestion.", + obj( + { + "content": {"type": "string", "maxLength": _MAX_CONTENT_CHARS}, + "title": {"type": "string", "maxLength": 512}, + "source_type": { + "type": "string", + "enum": ["text", "url"], + "default": "text", + }, + }, + ["content"], + ), + ingest, + ), + Tool( + "substrate_job_status", + "Read content-free status for one ingestion job.", + obj({"job_id": {"type": "string", "maxLength": _MAX_JOB_ID_CHARS}}, ["job_id"]), + job_status, + ), + Tool("substrate_sync", "Retry the durable capture spool.", obj({}, []), sync), + Tool("substrate_status", "Show content-free local status.", obj({}, []), status), + ] + + +__all__ = ["build_tools"] diff --git a/src/claude_code_memory/transcript.py b/src/claude_code_memory/transcript.py index 8f55221..fce5bac 100644 --- a/src/claude_code_memory/transcript.py +++ b/src/claude_code_memory/transcript.py @@ -4,6 +4,9 @@ import hashlib import json +import os +import stat +from dataclasses import dataclass from collections import OrderedDict from collections.abc import Iterator from pathlib import Path @@ -60,10 +63,7 @@ def set(self, key: str, state: _TailState) -> list[_TailState]: self._states[key] = state self.total_bytes += state[3] evicted: list[_TailState] = [] - while ( - self.total_bytes > self.max_bytes - or len(self._states) > self.max_lineages - ): + while self.total_bytes > self.max_bytes or len(self._states) > self.max_lineages: _evicted_key, evicted_state = self._states.popitem(last=False) self.total_bytes -= evicted_state[3] self.evicted_count += 1 @@ -220,11 +220,26 @@ def _identity_digest(*values: Any) -> str: return digest.hexdigest() +def _flag(value: Any) -> bool: + return value is True or ( + isinstance(value, str) and value.strip().casefold() in {"1", "true", "yes", "on"} + ) + + +def _record_is_subagent(record: dict[str, Any]) -> bool: + if isinstance(record.get("agentId"), str) and record["agentId"].strip(): + return True + return any( + _flag(record.get(key)) + for key in ("isSidechain", "is_sidechain", "isSubagent", "is_subagent") + ) + + def _lineage_identity(record: dict[str, Any]) -> str: return _identity_digest( record.get("sessionId"), record.get("agentId"), - record.get("isSidechain") is True, + _flag(record.get("isSidechain")), ) @@ -241,7 +256,7 @@ def _coordinates( "message_id": _redacted_bound(record.get("uuid"), secrets), "parent_message_id": _redacted_bound(record.get("parentUuid"), secrets), "agent_id": _redacted_bound(record.get("agentId"), secrets), - "is_sidechain": record.get("isSidechain") is True, + "is_sidechain": _flag(record.get("isSidechain")), }, } @@ -500,9 +515,7 @@ def _record_content(record: dict[str, Any]) -> str: if not isinstance(content, list): return "" return "".join( - piece - for piece in (_block_content(block) for block in content) - if piece is not None + piece for piece in (_block_content(block) for block in content) if piece is not None ) @@ -515,9 +528,7 @@ def _credential_crossing_spans( boundary = len(previous_tail) return [ (start, end) - for start, end in credential_redaction_spans( - previous_tail + current_content, secrets - ) + for start, end in credential_redaction_spans(previous_tail + current_content, secrets) if start < boundary < end ] @@ -550,85 +561,200 @@ def _discard_line_remainder(stream: Any, chunk: bytes) -> None: chunk = stream.readline(_MAX_LINE_BYTES + 1) -def read_messages(path: str | Path, *, include_sidechains: bool = True) -> list[dict[str, Any]]: - """Return bounded, redacted transcript blocks in transcript order. +@dataclass(frozen=True) +class TranscriptWindow: + """One bounded, resumable transcript window with content-free health.""" - Invalid records, oversized lines, truncated final JSON, and filesystem errors - are ignored. Sidechains are captured by default and can be disabled with the - explicit configuration kill-switch. This function never raises. + messages: list[dict[str, Any]] + next_cursor: int + source_id: str + readable: bool + complete: bool + skipped_records: int + reset: bool + anchor: str + + +def read_message_window( + path: str | Path, + *, + cursor: int = 0, + expected_source_id: str = "", + expected_anchor: str = "", + include_sidechains: bool = False, + limit: int = MAX_MESSAGES, + max_bytes: int = 512 * 1024, + end_cursor: int | None = None, + reject_malformed: bool = True, +) -> TranscriptWindow: + """Read after a durable byte cursor without hiding source failures. + + The caller must commit ``next_cursor`` only after every returned message is + durably admitted. A changed/truncated source resets explicitly. Malformed or + oversized records make the whole window unreadable and never advance. A bounded + hash of bytes immediately before the cursor proves append-only continuation and + detects same-inode truncate/regrow replacement without discarding a safe suffix. """ parsed: list[dict[str, Any]] = [] + skipped = 0 + reset = False + current_cursor = max(0, int(cursor)) if not isinstance(cursor, bool) else 0 + source_id = "" + next_cursor = current_cursor + next_anchor = "" try: + source = Path(path) + if source.is_symlink(): + raise OSError("transcript must not be a symlink") secrets = configured_secret_values() overlap_chars = credential_detector_overlap_chars() - # Each lineage owns a detector-sized suffix. The LRU additionally caps - # aggregate raw state; eviction quarantines referenced records before - # dropping their context, and increments only a content-free counter. stream_tails = _LineageTailBudget( max_bytes=_MAX_OVERLAP_TOTAL_BYTES, max_lineages=_MAX_OVERLAP_LINEAGES, ) - with Path(path).open("rb") as stream: - record_index = 0 - while len(parsed) < MAX_MESSAGES: + with source.open("rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode): + raise OSError("transcript must be a regular file") + source_id = hashlib.sha256(f"{info.st_dev}:{info.st_ino}".encode("ascii")).hexdigest()[ + :32 + ] + + def anchor_at(position: int) -> str: + start = max(0, position - 4096) + stream.seek(start) + sample = stream.read(position - start) + digest = hashlib.sha256() + digest.update(position.to_bytes(8, "big")) + digest.update(sample) + return digest.hexdigest() + + bounded_end = ( + info.st_size if end_cursor is None else max(0, min(int(end_cursor), info.st_size)) + ) + identity_changed = bool(expected_source_id and expected_source_id != source_id) + invalid_cursor = current_cursor > bounded_end + anchor_changed = bool( + current_cursor + and (not expected_anchor or anchor_at(current_cursor) != expected_anchor) + ) + if identity_changed or invalid_cursor or anchor_changed: + current_cursor = 0 + next_cursor = 0 + reset = True + if current_cursor: + stream.seek(current_cursor - 1) + if stream.read(1) != b"\n": + raise OSError("transcript cursor is not at a record boundary") + # Re-read the immediately preceding bounded record to restore + # redaction overlap without persisting sensitive lexical tails. + context_start = max(0, current_cursor - _MAX_LINE_BYTES - 1) + stream.seek(context_start) + context = stream.read(current_cursor - context_start) + previous_raw = context[:-1] + separator = previous_raw.rfind(b"\n") + previous_raw = previous_raw[separator + 1 :] + if previous_raw and len(previous_raw) <= _MAX_LINE_BYTES: + try: + previous_record = json.loads( + previous_raw.decode("utf-8", errors="strict"), + parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value)), + ) + except (UnicodeDecodeError, ValueError, json.JSONDecodeError): + previous_record = None + if isinstance(previous_record, dict): + previous_content = _record_content(previous_record) + previous_key = _lineage_identity(previous_record) + if previous_content and previous_key: + previous_tail = previous_content[-overlap_chars:] + placeholder = list( + _parse_record(previous_record, context_start, secrets) + ) + stream_tails.set( + previous_key, + ( + previous_tail, + [(0, len(previous_tail), placeholder)], + credential_content_has_open_continuation( + previous_content, secrets + ), + len(previous_tail.encode("utf-8")), + ), + ) + stream.seek(current_cursor) + bounded_limit = max(1, min(int(limit), MAX_MESSAGES)) + bounded_bytes = max(1, min(int(max_bytes), 1024 * 1024 * 1024)) + window_bytes = 0 + while len(parsed) < bounded_limit: + line_start = stream.tell() + if line_start >= bounded_end: + break raw = stream.readline(_MAX_LINE_BYTES + 1) + next_cursor = stream.tell() if not raw: break + if stream.tell() > bounded_end: + stream.seek(line_start) + next_cursor = line_start + break + if window_bytes and window_bytes + len(raw) > bounded_bytes: + stream.seek(line_start) + next_cursor = line_start + break + window_bytes += len(raw) if len(raw) > _MAX_LINE_BYTES: + if reject_malformed: + raise OSError("oversized transcript record") _discard_line_remainder(stream, raw) - record_index += 1 + next_cursor = stream.tell() + skipped += 1 continue try: - record = json.loads(raw.decode("utf-8", errors="strict")) + record = json.loads( + raw.decode("utf-8", errors="strict"), + parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value)), + ) except (UnicodeDecodeError, json.JSONDecodeError): - record_index += 1 + if not raw.endswith(b"\n") and next_cursor >= info.st_size: + next_cursor = line_start + break + if reject_malformed: + raise OSError("malformed transcript record") from None + skipped += 1 continue if not isinstance(record, dict): - record_index += 1 + if reject_malformed: + raise OSError("malformed transcript record") + skipped += 1 continue if record.get("type") not in {"user", "assistant", "system"}: - record_index += 1 continue - if record.get("isSidechain") is True and not include_sidechains: - record_index += 1 + if _record_is_subagent(record) and not include_sidechains: continue - current_messages: list[dict[str, Any]] = [] - for message in _parse_record(record, record_index, secrets): - if len(parsed) >= MAX_MESSAGES: - break - parsed.append(message) - current_messages.append(message) + current_messages = list(_parse_record(record, line_start, secrets)) + parsed.extend(current_messages) current_content = _record_content(record) stream_key = _lineage_identity(record) previous_state = stream_tails.get(stream_key) if not current_content: stream_tails.preserve(stream_key) - record_index += 1 continue if previous_state is None: - previous_tail, previous_segments, previous_continuation = ( - "", - [], - False, - ) + previous_tail, previous_segments, previous_continuation = "", [], False else: - previous_tail, previous_segments, previous_continuation, _ = ( - previous_state - ) + previous_tail, previous_segments, previous_continuation, _ = previous_state combined = previous_tail + current_content continuation_active = False if previous_continuation: - continuation_chars, continuation_active = ( - credential_continuation_prefix(current_content) + continuation_chars, continuation_active = credential_continuation_prefix( + current_content ) if continuation_chars: _quarantine_record(current_messages) - crossing_spans = _credential_crossing_spans( - previous_tail, current_content, secrets - ) - for start, end in crossing_spans: + crossing_spans = _credential_crossing_spans(previous_tail, current_content, secrets) + for span_start, span_end in crossing_spans: for segment_start, segment_end, segment_messages in previous_segments: - if segment_start < end and segment_end > start: + if segment_start < span_end and segment_end > span_start: _quarantine_record(segment_messages) _quarantine_record(current_messages) current_detected = any( @@ -640,9 +766,7 @@ def read_messages(path: str | Path, *, include_sidechains: bool = True) -> list[ ) and credential_content_has_open_continuation(combined, secrets): continuation_active = True boundary = len(previous_tail) - segments = previous_segments + [ - (boundary, len(combined), current_messages) - ] + segments = previous_segments + [(boundary, len(combined), current_messages)] cutoff = max(0, len(combined) - overlap_chars) tail = combined[cutoff:] tail_state: _TailState = ( @@ -665,10 +789,13 @@ def read_messages(path: str | Path, *, include_sidechains: bool = True) -> list[ identity = id(evicted_messages) if identity not in seen_records: seen_records.add(identity) - _quarantine_record( - evicted_messages, code="overlap_state_evicted" - ) - record_index += 1 + _quarantine_record(evicted_messages, code="overlap_state_evicted") + skipped += 1 + final_info = os.fstat(stream.fileno()) + if final_info.st_size < next_cursor: + raise OSError("transcript truncated while reading") + complete = next_cursor >= bounded_end + next_anchor = anchor_at(next_cursor) _pair_tool_results(parsed) normalized: list[dict[str, Any]] = [] for index, message in enumerate(parsed): @@ -677,9 +804,44 @@ def read_messages(path: str | Path, *, include_sidechains: bool = True) -> list[ item = normalize_message(message, index=index, secrets=secrets) if item is not None: normalized.append(item) - return normalized - except Exception: # noqa: BLE001 - transcript capture must always fail open - return [] + return TranscriptWindow( + normalized, + next_cursor, + source_id, + True, + complete, + skipped, + reset, + next_anchor, + ) + except Exception: # noqa: BLE001 - caller observes readable=False and preserves its cursor + return TranscriptWindow([], current_cursor, source_id, False, False, 0, False, "") + + +def read_messages(path: str | Path, *, include_sidechains: bool = True) -> list[dict[str, Any]]: + """Compatibility wrapper that follows verified windows through current EOF.""" + messages: list[dict[str, Any]] = [] + cursor = 0 + source_id = "" + anchor = "" + while True: + window = read_message_window( + path, + cursor=cursor, + expected_source_id=source_id, + expected_anchor=anchor, + include_sidechains=include_sidechains, + max_bytes=1024 * 1024 * 1024, + reject_malformed=False, + ) + if not window.readable: + return messages + messages.extend(window.messages) + if len(messages) >= MAX_MESSAGES: + return messages[:MAX_MESSAGES] + if window.complete or window.next_cursor == cursor: + return messages + cursor, source_id, anchor = window.next_cursor, window.source_id, window.anchor __all__ = [ @@ -687,5 +849,7 @@ def read_messages(path: str | Path, *, include_sidechains: bool = True) -> list[ "MAX_MESSAGES", "MAX_MESSAGE_CHARS", "SOURCE_PROTOCOL_USER_EDIT", + "TranscriptWindow", + "read_message_window", "read_messages", ] diff --git a/src/substrate_capture/_vendor.json b/src/substrate_capture/_vendor.json index a0bce37..8e6b807 100644 --- a/src/substrate_capture/_vendor.json +++ b/src/substrate_capture/_vendor.json @@ -2,14 +2,14 @@ "files": { "__init__.py": "ee06117bc4b7a94c1f2e9c5c8b9a25e5e65a7c23bf7db26b00ba9990dccb3088", "checkpoint.py": "e7e687a119a756c5e5992c1dec6e34c792839bc07d9332563003241a2ab7d030", - "client.py": "9e4df698fb3919b502766d835c9ee6e3c9995381c75772a6452942bc0f54cd4a", - "config.py": "ea34befc39ee48b11c011cf7890a53ba0eaadd57a5e4c395068489319d638d3e", + "client.py": "78b919827fb9d77db76765e1e871b2b0bf54e5be84199962ce81e7afc7146870", + "config.py": "bb80af21fc09c03704473463b09fccd5dcd9dbd34bdc47856500539bf41be8fa", "delivery.py": "6c48d989d47fedbda01fdc966915bd39ba47c8f1fefb3815b28a74b8e01dd740", "events.py": "fb1bb57e52a804f9a17a48b8657737d6c761c1db9c839abd7a2eb76e328bd6ff", - "mcp.py": "50b2f046792ed7da20644c167a1bd38da464d2025ac1a0949599f2e5ac90097e", + "mcp.py": "479ee0dfb05f4ab799aff497f11a4bd844f4359d2d287c9b90dc205545f6d4ad", "py.typed": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "redaction.py": "40b57020c3957d1dda72e0b36fa81cf0b40598a968166d8fb1c91729b91016d3", - "spool.py": "8ab56569e61da99f7fd8e1f539f4ecf3cdbf68cce5f50406212f5da4929aa977", + "spool.py": "4ebba9466b7ffb057ca2860e18c050b17b2b89ceca6887a7f348f45424ed432c", "tools.py": "1e4887131c95d10e99550de4a3d4f38953661adbceb1fbec26bd5986552f5902" }, "vendor_version": "1.0.0" diff --git a/src/substrate_capture/client.py b/src/substrate_capture/client.py index 4e271f1..a57c0ff 100644 --- a/src/substrate_capture/client.py +++ b/src/substrate_capture/client.py @@ -194,7 +194,7 @@ def request( data = None if body is not None: try: - data = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + data = json.dumps(body, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8") except (TypeError, ValueError): raise SubstrateAPIError("invalid_request") from None if len(data) > _MAX_REQUEST_BYTES: @@ -224,14 +224,14 @@ def request( return {} try: decoded = raw.decode("utf-8", errors="strict") - value = json.loads(decoded) + value = json.loads(decoded, parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value))) except (UnicodeDecodeError, json.JSONDecodeError): raise SubstrateAPIError("invalid_response") from None return self._shape_response(path, value) except SubstrateAPIError: raise except HTTPError as exc: - retry_after = _retry_after_seconds(exc.headers) if exc.code == 429 else None + retry_after = _retry_after_seconds(exc.headers) if exc.code in {429, 503} else None raise SubstrateAPIError(f"http_{exc.code}", retry_after=retry_after) from None except TimeoutError: raise SubstrateAPIError("timeout") from None diff --git a/src/substrate_capture/config.py b/src/substrate_capture/config.py index 5d75634..4842d14 100644 --- a/src/substrate_capture/config.py +++ b/src/substrate_capture/config.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import Any -from .spool import secure_atomic_json_write +from .spool import secure_atomic_json_write, validate_no_symlink_ancestors CONFIG_FILENAME = "config.json" _MAX_CONFIG_BYTES = 64 * 1024 @@ -56,10 +56,9 @@ def state_home(provider_id: str) -> Path: raise ValueError("invalid provider_id") override = os.environ.get("SUBSTRATE_STATE_HOME", "").strip() root = Path(override).expanduser() if override else Path.home() / ".substrate" - target = root / provider_id - if target.is_symlink(): - raise ValueError("state directory must not be a symlink") + target = validate_no_symlink_ancestors(root / provider_id) target.mkdir(parents=True, exist_ok=True) + validate_no_symlink_ancestors(target) _harden(target) return target diff --git a/src/substrate_capture/mcp.py b/src/substrate_capture/mcp.py index 20b15cd..5434f93 100644 --- a/src/substrate_capture/mcp.py +++ b/src/substrate_capture/mcp.py @@ -37,9 +37,9 @@ def bounded_json(value: Any) -> str: what the memory contains. """ try: - text = json.dumps(value, ensure_ascii=False, separators=(",", ":")) + text = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) except (TypeError, ValueError): - return json.dumps({"error": "invalid_result"}, separators=(",", ":")) + return json.dumps({"error": "invalid_result"}, separators=(",", ":"), allow_nan=False) encoded = text.encode("utf-8") if len(encoded) <= MAX_RESULT_BYTES: return text @@ -50,6 +50,7 @@ def bounded_json(value: Any) -> str: "original_bytes": len(encoded), }, separators=(",", ":"), + allow_nan=False, ) @@ -173,7 +174,7 @@ def serve( _write(sink, _error(None, INVALID_REQUEST, "message too large")) continue try: - message = json.loads(stripped) + message = json.loads(stripped, parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value))) except ValueError: # A malformed line must not end the session. _write(sink, _error(None, PARSE_ERROR, "parse error")) @@ -188,5 +189,5 @@ def serve( def _write(sink: TextIO, message: dict[str, Any]) -> None: - sink.write(json.dumps(message, ensure_ascii=False, separators=(",", ":")) + "\n") + sink.write(json.dumps(message, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + "\n") sink.flush() diff --git a/src/substrate_capture/spool.py b/src/substrate_capture/spool.py index 242e638..cf013d5 100644 --- a/src/substrate_capture/spool.py +++ b/src/substrate_capture/spool.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import os import stat @@ -23,15 +24,23 @@ msvcrt = None # type: ignore[assignment] -def _lock_file(descriptor: int) -> None: - if fcntl is not None: - fcntl.flock(descriptor, fcntl.LOCK_EX) - return - if os.fstat(descriptor).st_size == 0: +def _lock_file(descriptor: int, *, timeout: float = 2.0) -> None: + deadline = time.monotonic() + max(0.0, timeout) + if fcntl is None and os.fstat(descriptor).st_size == 0: os.write(descriptor, b"\0") os.fsync(descriptor) - os.lseek(descriptor, 0, os.SEEK_SET) - msvcrt.locking(descriptor, msvcrt.LK_LOCK, 1) + while True: + try: + if fcntl is not None: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + else: + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1) + return + except (BlockingIOError, OSError): + if time.monotonic() >= deadline: + raise TimeoutError("spool lock acquisition timed out") from None + time.sleep(0.01) def _unlock_file(descriptor: int) -> None: @@ -58,15 +67,30 @@ def _fsync_directory(path: Path) -> None: os.close(descriptor) + +def validate_no_symlink_ancestors(path: Path) -> Path: + """Return an absolute path after rejecting every existing symlink ancestor.""" + absolute = path.expanduser().absolute() + chain = [absolute, *absolute.parents] + for candidate in reversed(chain): + try: + info = candidate.lstat() + except FileNotFoundError: + continue + if stat.S_ISLNK(info.st_mode): + raise OSError("path ancestry must not contain symlinks") + if candidate != absolute and not stat.S_ISDIR(info.st_mode): + raise OSError("path ancestor must be a directory") + return absolute + def _safe_root(root: Path) -> Path: - absolute = root.absolute() - if absolute.exists() and absolute.is_symlink(): - raise OSError("spool root must not be a symlink") + absolute = validate_no_symlink_ancestors(root) absolute.mkdir(parents=True, exist_ok=True, mode=0o700) - if absolute.is_symlink() or not absolute.is_dir(): + validate_no_symlink_ancestors(absolute) + if not absolute.is_dir(): raise OSError("invalid spool root") _chmod_private(absolute, 0o700) - return absolute.resolve(strict=True) + return absolute def _safe_child(root: Path, path: Path) -> Path: @@ -84,7 +108,7 @@ def secure_atomic_json_write(target: Path, value: Any) -> None: """Write JSON with exclusive temp creation, fsync, replace, and directory fsync.""" root = _safe_root(target.parent) target = _safe_child(root, target) - payload = (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode( + payload = (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + "\n").encode( "utf-8" ) temporary = root / f".{target.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" @@ -137,7 +161,11 @@ def __init__( def append(self, event: dict[str, Any]) -> Path: try: payload = json.dumps( - event, ensure_ascii=False, separators=(",", ":"), sort_keys=True + event, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + allow_nan=False, ).encode("utf-8") except (TypeError, ValueError): with self._transaction(refresh_counters=True): @@ -161,12 +189,28 @@ def append(self, event: dict[str, Any]) -> Path: self._increment_locked("dropped") raise ValueError("spool capacity unavailable") self._sequence += 1 - target = self.root / ( + # A UUID suffix plus exclusive final creation prevents two instances + # with identical clocks/PIDs/sequences from overwriting each other. + prefix = ( f"{time.time_ns():020d}-{os.getpid()}-{threading.get_ident()}-" - f"{self._sequence:08d}.json" + f"{self._sequence:08d}" + ) + event_id = event.get("event_id") + marker = ( + hashlib.sha256(event_id.encode("utf-8")).hexdigest() + if isinstance(event_id, str) and event_id + else "" ) - self._write_payload_locked(target, payload) - return target + suffix = f"-{marker}" if marker else "" + for _ in range(16): + target = self.root / f"{prefix}-{uuid.uuid4().hex}{suffix}.json" + try: + self._write_payload_locked(target, payload) + except FileExistsError: + continue + return target + self._increment_locked("dropped") + raise OSError("unable to allocate collision-free spool item") def statistics(self) -> dict[str, int]: """Return persistent content-free loss and quarantine counters.""" @@ -200,7 +244,7 @@ def load(self, path: Path) -> dict[str, Any]: if len(raw) > self.max_bytes: raise ValueError("spooled event exceeds limit") try: - value = json.loads(raw.decode("utf-8", errors="strict")) + value = json.loads(raw.decode("utf-8", errors="strict"), parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value))) except (UnicodeDecodeError, json.JSONDecodeError): raise ValueError("corrupt spooled event") from None if not isinstance(value, dict): @@ -281,12 +325,19 @@ def _duplicate_locked(self, event: dict[str, Any]) -> Path | None: event_id = event.get("event_id") if not isinstance(event_id, str) or not event_id: return None + marker = hashlib.sha256(event_id.encode("utf-8")).hexdigest() for path in self._files_locked(): + candidate_marker = path.stem.rsplit("-", 1)[-1] + marked = len(candidate_marker) == len(marker) and all( + character in "0123456789abcdef" for character in candidate_marker + ) + if marked and candidate_marker != marker: + continue try: descriptor = self._open_readonly(path) with os.fdopen(descriptor, "rb") as stream: raw = stream.read(self.max_bytes + 1) - value = json.loads(raw.decode("utf-8", errors="strict")) + value = json.loads(raw.decode("utf-8", errors="strict"), parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value))) except (OSError, ValueError, UnicodeDecodeError, json.JSONDecodeError): continue if isinstance(value, dict) and value.get("event_id") == event_id: @@ -298,7 +349,7 @@ def _load_counters(self) -> dict[str, int]: try: if self._stats_path.is_symlink() or self._stats_path.stat().st_size > 4096: return counters - value = json.loads(self._stats_path.read_text(encoding="utf-8")) + value = json.loads(self._stats_path.read_text(encoding="utf-8"), parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value))) except (FileNotFoundError, OSError, UnicodeDecodeError, json.JSONDecodeError): return counters if not isinstance(value, dict): @@ -336,7 +387,9 @@ def _write_payload_locked(self, target: Path, payload: bytes) -> None: stream.flush() os.fsync(stream.fileno()) _chmod_private(temporary, 0o600) - os.replace(temporary, target) + # link() publishes the complete inode and fails if target exists; + # unlike replace(), it cannot overwrite an admitted queue item. + os.link(temporary, target, follow_symlinks=False) _chmod_private(target, 0o600) _fsync_directory(self.root) finally: diff --git a/tests/test_authority_release.py b/tests/test_authority_release.py new file mode 100644 index 0000000..de63324 --- /dev/null +++ b/tests/test_authority_release.py @@ -0,0 +1,760 @@ +"""Authority citations, frozen vendor bytes, launcher, and release closure.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import time +import zipfile + +import pytest +from io import BytesIO +from pathlib import Path + +from claude_code_memory.contract import ( + CLAUDE_BASE, + HERMES_AUTHORITY_MAIN, + HERMES_AUTHORITY_RELEASE, + SUPPORTED_CLAUDE_CODE_VERSIONS, + VERSION, +) + +ROOT = Path(__file__).parents[1] + + +def _head_sha() -> str: + return subprocess.run( + ("git", "-C", str(ROOT), "rev-parse", "HEAD"), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + +def _load_builder() -> object: + path = ROOT / "scripts" / "build_release.py" + spec = importlib.util.spec_from_file_location("release_builder", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_hermes_authority_and_base_are_explicit_not_inferred() -> None: + assert HERMES_AUTHORITY_MAIN == "db4ddc2f093f833ffa62f4246860de96ba398713" + assert HERMES_AUTHORITY_RELEASE == "e4ad07cfc858618edfd69e1f3be5e8345b253037" + assert CLAUDE_BASE == "2a5051304ac7e8b960e3cdd9d4d2e96dcba76b80" + for name in ("README.md", "docs/architecture.md", "docs/releasing.md"): + text = (ROOT / name).read_text() + assert HERMES_AUTHORITY_MAIN in text + assert HERMES_AUTHORITY_RELEASE in text + + +def test_one_version_and_exact_host_gate_are_consistent() -> None: + plugin = json.loads((ROOT / ".claude-plugin" / "plugin.json").read_text()) + market = json.loads((ROOT / ".claude-plugin" / "marketplace.json").read_text()) + pyproject = (ROOT / "pyproject.toml").read_text() + assert plugin["version"] == market["plugins"][0]["version"] == VERSION + assert f'version = "{VERSION}"' in pyproject + assert SUPPORTED_CLAUDE_CODE_VERSIONS == ("2.1.237",) + + +def test_shared_vendor_manifest_matches_narrow_security_fixes() -> None: + root = ROOT / "src" / "substrate_capture" + manifest = json.loads((root / "_vendor.json").read_text()) + actual = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in root.iterdir() + if path.is_file() and path.name != "_vendor.json" + } + assert manifest["files"] == actual + + +def test_release_zip_is_deterministic_closed_and_provenanced() -> None: + builder = _load_builder() + source_sha = subprocess.run( + ("git", "-C", str(ROOT), "rev-parse", "HEAD"), + capture_output=True, + text=True, + check=True, + ).stdout.strip() + first = builder._zip_bytes(source_sha) # type: ignore[attr-defined] + second = builder._zip_bytes(source_sha) # type: ignore[attr-defined] + assert first == second + with zipfile.ZipFile(BytesIO(first)) as archive: + names = archive.namelist() + assert len(names) == len(set(names)) + provenance = json.loads(archive.read("claude_code_substrate_memory/PROVENANCE.json")) + assert provenance["source_commit"] == source_sha + assert provenance["plugin_version"] == VERSION + assert provenance["provider_id"] == "claude_code_memory" + archived_files = { + name.removeprefix("claude_code_substrate_memory/") + for name in names + if not name.endswith("PROVENANCE.json") + } + assert archived_files == set(provenance["files"]) + + +def _load_installer() -> object: + path = ROOT / "scripts" / "install_release.py" + spec = importlib.util.spec_from_file_location("release_installer", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_installer_exact_claude_version_parser(monkeypatch: pytest.MonkeyPatch) -> None: + installer = _load_installer() + + class Result: + returncode = 0 + stdout = "2.1.237 (Claude Code)\n" + + monkeypatch.setattr( # type: ignore[attr-defined] + installer.subprocess, "run", lambda *_args, **_kwargs: Result() + ) + assert installer._claude_version("synthetic-claude") == "2.1.237" # type: ignore[attr-defined] + Result.stdout = "2.1.238 (Claude Code)\n" + try: + installer._claude_version("synthetic-claude") # type: ignore[attr-defined] + except ValueError as exc: + assert str(exc) == "unsupported Claude Code version" + else: + raise AssertionError("unsupported host version was accepted") + + +def _fake_claude(tmp_path: Path) -> Path: + script = tmp_path / "fake-claude.py" + script.write_text( + f"""#!{sys.executable} +import json +import os +import shutil +import sys +import time +import time +from pathlib import Path +config = Path(os.environ["CLAUDE_CONFIG_DIR"]) +state_path = config / "fake-host-state.json" +log_path = config / "fake-host-log.jsonl" +config.mkdir(parents=True, exist_ok=True) +args = sys.argv[1:] +with log_path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(args) + "\\n") +if args == ["--version"]: + print("2.1.237 (Claude Code)") + raise SystemExit(0) +state = json.loads(state_path.read_text()) if state_path.exists() else {{"mutations": 0}} +def save(): + state["mutations"] = int(state.get("mutations", 0)) + 1 + delay = float(os.environ.get("FAKE_CLAUDE_MUTATION_DELAY", "0")) + if delay: + time.sleep(delay) + state_path.write_text(json.dumps(state, sort_keys=True)) +def source_version(): + source = Path(state["target"]) + return json.loads((source / ".claude-plugin" / "plugin.json").read_text())["version"] +if args[:2] == ["plugin", "validate"]: + source = Path(args[2]) + valid = (source / ".claude-plugin" / "plugin.json").is_file() + raise SystemExit(0 if valid else 2) +if args[:3] == ["plugin", "marketplace", "add"]: + state.update(marketplace=True, target=args[3]) + save() + registry = config / "plugins" / "known_marketplaces.json" + registry.parent.mkdir(parents=True, exist_ok=True) + registry.write_text(json.dumps({{"claude-code-substrate-memory": {{"source": {{"source": "directory", "path": args[3]}}}}}})) + raise SystemExit(0) +if args[:3] == ["plugin", "marketplace", "remove"]: + state["marketplace"] = False + save() + registry = config / "plugins" / "known_marketplaces.json" + registry.write_text("{{}}") + raise SystemExit(0) +if args == ["plugin", "list", "--json"]: + records = [] + if state.get("installed"): + records.append({{ + "id": "claude-code-substrate-memory@claude-code-substrate-memory", + "version": source_version(), + "enabled": bool(state.get("enabled")), + "scope": "user", + "installedAt": state.get("installed_at"), + "lastUpdated": state.get("installed_at"), + "mcpServers": {{"substrate": {{ + "type": "stdio", + "command": "node", + "args": ["${{CLAUDE_PLUGIN_ROOT}}/scripts/plugin_runtime.cjs", "mcp"], + }}}}, + }}) + print(json.dumps(records)) + raise SystemExit(0) +if args[:2] == ["plugin", "uninstall"]: + state.update(installed=False, enabled=False) + save() + raise SystemExit(0) +if args[:2] == ["plugin", "install"]: + if not state.get("marketplace"): + raise SystemExit(2) + state.update(installed=True, enabled=True, mcp_dead=False) + state["installed_at"] = f"stamp-{{int(state.get('mutations', 0)) + 1}}" + save() + raise SystemExit(0) +if args[:2] == ["plugin", "enable"]: + if state.get("enabled"): + raise SystemExit(2) + state["enabled"] = True + save() + raise SystemExit(0) +if args[:2] == ["plugin", "disable"]: + if not state.get("enabled"): + raise SystemExit(2) + state["enabled"] = False + save() + raise SystemExit(0) +if args[:2] == ["plugin", "details"]: + raise SystemExit(0 if state.get("installed") else 2) +if args == ["mcp", "list"]: + configured_output = os.environ.get("FAKE_CLAUDE_MCP_OUTPUT") + if configured_output is not None: + print(configured_output, end="" if configured_output.endswith("\\n") else "\\n") + raise SystemExit(0) + if state.get("installed") and state.get("enabled") and not state.get("mcp_dead"): + print("plugin:claude-code-substrate-memory:substrate: fake - ✔ Connected") + raise SystemExit(0) + print("No MCP servers configured") + raise SystemExit(0) +raise SystemExit(2) +""", + encoding="utf-8", + ) + script.chmod(0o700) + if os.name != "nt": + return script + executable = tmp_path / "fake-claude.cmd" + executable.write_text(f'@echo off\r\n"{sys.executable}" "{script}" %*\r\n', encoding="utf-8") + return executable + + +def _generated_fixture(tmp_path: Path) -> tuple[Path, Path, Path, dict[str, str]]: + builder = _load_builder() + source_commit = _head_sha() + archive_bytes = builder._zip_bytes(source_commit) # type: ignore[attr-defined] + archive = tmp_path / "candidate.zip" + archive.write_bytes(archive_bytes) + generated = tmp_path / "install_claude_plugin.py" + generated.write_text( + (ROOT / "scripts" / "install_release.py") + .read_text() + .replace("@ARCHIVE_SHA256@", hashlib.sha256(archive_bytes).hexdigest()) + .replace("@SOURCE_COMMIT@", source_commit) + .replace("@PLUGIN_VERSION@", VERSION) + ) + fake = _fake_claude(tmp_path) + environment = os.environ.copy() + environment.update( + HOME=str(tmp_path / "split-home"), + CLAUDE_CONFIG_DIR=str(tmp_path / "claude-config"), + SUBSTRATE_STATE_HOME=str(tmp_path / "external-substrate-state"), + PYTHONIOENCODING="utf-8", + ) + return archive, generated, fake, environment + + +def _run_generated( + generated: Path, + fake: Path, + environment: dict[str, str], + *arguments: str, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(generated), *arguments, "--claude", str(fake)], + capture_output=True, + text=True, + timeout=30, + check=False, + env=environment, + ) + + +def test_generated_installer_first_noop_update_rollback_and_uninstall(tmp_path: Path) -> None: + archive, generated, fake, environment = _generated_fixture(tmp_path) + first = _run_generated(generated, fake, environment, str(archive)) + assert first.returncode == 0, first.stdout + assert json.loads(first.stdout)["plugin_version"] == VERSION + + config = Path(environment["CLAUDE_CONFIG_DIR"]) + source = next((config / "plugins" / "substrate-installer" / "releases").glob("*/source")) + registration = config / "fake-host-state.json" + journal = config / "plugins" / "substrate-installer" / "journal.json" + state = config / "plugins" / "substrate-installer" / "state.json" + before = ( + source.stat().st_ino, + source.stat().st_mtime_ns, + registration.read_bytes(), + registration.stat().st_mtime_ns, + state.read_bytes(), + state.stat().st_mtime_ns, + ) + second = _run_generated(generated, fake, environment, str(archive)) + assert second.returncode == 0 + assert json.loads(second.stdout)["already_installed"] is True + after = ( + source.stat().st_ino, + source.stat().st_mtime_ns, + registration.read_bytes(), + registration.stat().st_mtime_ns, + state.read_bytes(), + state.stat().st_mtime_ns, + ) + assert after == before + assert not journal.exists() + + # Replace the active registration with a synthetic older valid source. + uninstalled = _run_generated(generated, fake, environment, "--uninstall") + assert uninstalled.returncode == 0 + old_source = tmp_path / "old-source" + with zipfile.ZipFile(archive) as bundle: + bundle.extractall(tmp_path / "old-extract") + shutil.move(tmp_path / "old-extract" / "claude_code_substrate_memory", old_source) + plugin_manifest = old_source / ".claude-plugin" / "plugin.json" + plugin_value = json.loads(plugin_manifest.read_text()) + plugin_value["version"] = "0.1.0" + plugin_manifest.write_text(json.dumps(plugin_value)) + market_manifest = old_source / ".claude-plugin" / "marketplace.json" + market_value = json.loads(market_manifest.read_text()) + market_value["plugins"][0]["version"] = "0.1.0" + market_manifest.write_text(json.dumps(market_value)) + for command in ( + ["plugin", "marketplace", "add", str(old_source), "--scope", "user"], + [ + "plugin", + "install", + "claude-code-substrate-memory@claude-code-substrate-memory", + "--scope", + "user", + ], + ): + assert subprocess.run([str(fake), *command], env=environment, check=False).returncode == 0 + + updated = _run_generated(generated, fake, environment, str(archive)) + assert updated.returncode == 0 + assert json.loads(updated.stdout)["rollback_available"] is True + rolled_back = _run_generated(generated, fake, environment, "--rollback") + assert rolled_back.returncode == 0 + rollback_value = json.loads(rolled_back.stdout) + assert rollback_value["plugin_version"] == "0.1.0" + assert rollback_value["source"] == str(old_source) + + external = Path(environment["SUBSTRATE_STATE_HOME"]) + external.mkdir() + (external / "credential-spool-checkpoint.marker").write_text("preserve") + removed = _run_generated(generated, fake, environment, "--uninstall") + assert removed.returncode == 0 + removed_value = json.loads(removed.stdout) + assert removed_value["registered"] is removed_value["marketplace_registered"] is False + assert removed_value["active_registration"] == "absent" + assert removed_value["remaining_claude_cache_paths"] == [] + assert removed_value["server_revoked"] is False + assert (external / "credential-spool-checkpoint.marker").read_text() == "preserve" + refused_purge = _run_generated( + generated, fake, environment, "--uninstall", "--purge", "--confirm-purge", "wrong" + ) + assert refused_purge.returncode == 2 + purged = _run_generated( + generated, + fake, + environment, + "--uninstall", + "--purge", + "--confirm-purge", + "claude-code-substrate-memory@claude-code-substrate-memory", + ) + assert purged.returncode == 0 + assert json.loads(purged.stdout)["remaining_release_sources"] == [] + assert (external / "credential-spool-checkpoint.marker").read_text() == "preserve" + + +def test_generated_installer_noop_verifies_live_mcp_and_repairs_dead_activation( + tmp_path: Path, +) -> None: + archive, generated, fake, environment = _generated_fixture(tmp_path) + first = _run_generated(generated, fake, environment, str(archive)) + assert first.returncode == 0, first.stderr + config = Path(environment["CLAUDE_CONFIG_DIR"]) + source = next((config / "plugins" / "substrate-installer" / "releases").glob("*/source")) + registration = config / "fake-host-state.json" + state_path = config / "plugins" / "substrate-installer" / "state.json" + healthy_before = ( + source.stat().st_ino, + source.stat().st_mtime_ns, + registration.read_bytes(), + registration.stat().st_mtime_ns, + state_path.read_bytes(), + state_path.stat().st_mtime_ns, + ) + healthy = _run_generated(generated, fake, environment, str(archive)) + assert healthy.returncode == 0, healthy.stderr + assert json.loads(healthy.stdout)["already_installed"] is True + healthy_after = ( + source.stat().st_ino, + source.stat().st_mtime_ns, + registration.read_bytes(), + registration.stat().st_mtime_ns, + state_path.read_bytes(), + state_path.stat().st_mtime_ns, + ) + assert healthy_after == healthy_before + commands = [ + json.loads(line) for line in (config / "fake-host-log.jsonl").read_text().splitlines() + ] + assert [ + "plugin", + "details", + "claude-code-substrate-memory@claude-code-substrate-memory", + ] in commands + assert ["mcp", "list"] in commands + + host_state = json.loads(registration.read_text()) + host_state["mcp_dead"] = True + registration.write_text(json.dumps(host_state, sort_keys=True)) + repair = _run_generated(generated, fake, environment, str(archive)) + assert repair.returncode == 0, repair.stderr + result = json.loads(repair.stdout) + assert result["already_installed"] is False + assert result["registered"] is True and result["enabled"] is True + repaired_host = json.loads(registration.read_text()) + assert repaired_host["mcp_dead"] is False + assert not (config / "plugins" / "substrate-installer" / "journal.json").exists() + + +@pytest.mark.parametrize( + ("mcp_output", "healthy"), + [ + ("plugin:claude-code-substrate-memory:substrate: fake - ✔ Connected", True), + ("plugin:claude-code-substrate-memory:substrate: fake - ✘ Not Connected", False), + ("plugin:claude-code-substrate-memory:substrate: fake - ✘ Disconnected", False), + ("plugin:claude-code-substrate-memory:substrate: fake - ◌ Connecting", False), + ("plugin:other-server:substrate: fake - ✔ Connected", False), + ("plugin:claude-code-substrate-memory:substrate: - ✔ Connected", False), + ( + "plugin:claude-code-substrate-memory:substrate: fake - ✔ Connected\n" + "plugin:claude-code-substrate-memory:substrate: fake - ✔ Connected", + False, + ), + ( + "plugin:claude-code-substrate-memory:substrate: fake - ✔ Connected\n" + "plugin:claude-code-substrate-memory:substrate: fake - ✘ Disconnected", + False, + ), + ], +) +def test_generated_installer_accepts_only_one_exact_connected_mcp_record( + tmp_path: Path, mcp_output: str, healthy: bool +) -> None: + archive, generated, fake, environment = _generated_fixture(tmp_path) + first = _run_generated(generated, fake, environment, str(archive)) + assert first.returncode == 0, first.stderr + hostile = dict(environment) + hostile["FAKE_CLAUDE_MCP_OUTPUT"] = mcp_output + result = _run_generated(generated, fake, hostile, str(archive)) + if healthy: + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["already_installed"] is True + else: + assert result.returncode != 0 + assert "already_installed" not in result.stdout + + +@pytest.mark.parametrize( + "boundary", ["prepare", "uninstall", "marketplace", "register", "verify", "commit"] +) +def test_generated_installer_recovers_every_durable_boundary(tmp_path: Path, boundary: str) -> None: + archive, generated, fake, environment = _generated_fixture(tmp_path) + crashing = dict(environment) + crashing["SUBSTRATE_INSTALLER_TEST_CRASH_AFTER"] = boundary + interrupted = _run_generated(generated, fake, crashing, str(archive)) + assert interrupted.returncode == 86 + journal = ( + Path(environment["CLAUDE_CONFIG_DIR"]) / "plugins" / "substrate-installer" / "journal.json" + ) + assert json.loads(journal.read_text())["phase"] == boundary + recovered = _run_generated(generated, fake, environment, str(archive)) + assert recovered.returncode == 0, recovered.stdout + assert not journal.exists() + host = json.loads((Path(environment["CLAUDE_CONFIG_DIR"]) / "fake-host-state.json").read_text()) + assert host["installed"] is host["enabled"] is host["marketplace"] is True + + +@pytest.mark.parametrize( + "boundary", ["prepare", "uninstall", "marketplace", "register", "verify", "commit"] +) +def test_generated_rollback_itself_recovers_every_boundary(tmp_path: Path, boundary: str) -> None: + archive, generated, fake, environment = _generated_fixture(tmp_path) + assert _run_generated(generated, fake, environment, str(archive)).returncode == 0 + crashing = dict(environment) + crashing["SUBSTRATE_INSTALLER_TEST_CRASH_AFTER"] = boundary + interrupted = _run_generated(generated, fake, crashing, "--rollback") + assert interrupted.returncode == 86 + recovered = _run_generated(generated, fake, environment, "--rollback") + assert recovered.returncode == 0, recovered.stdout + value = json.loads(recovered.stdout) + assert value["recovered"] == "rollback" + assert value["registered"] is value["enabled"] is False + root = Path(environment["CLAUDE_CONFIG_DIR"]) / "plugins" / "substrate-installer" + assert not (root / "journal.json").exists() + + +def test_generated_installer_ignores_stale_lock_metadata(tmp_path: Path) -> None: + archive, generated, fake, environment = _generated_fixture(tmp_path) + lock = ( + Path(environment["CLAUDE_CONFIG_DIR"]) + / "plugins" + / "substrate-installer" + / "transaction.lock" + ) + lock.parent.mkdir(parents=True) + lock.write_text("stale metadata is not ownership") + installed = _run_generated(generated, fake, environment, str(archive)) + assert installed.returncode == 0 + assert not lock.with_name("journal.json").exists() + + +def test_generated_installer_serializes_concurrent_fresh_root(tmp_path: Path) -> None: + archive, generated, fake, environment = _generated_fixture(tmp_path) + delayed = dict(environment) + delayed["FAKE_CLAUDE_MUTATION_DELAY"] = "0.05" + commands = [sys.executable, str(generated), str(archive), "--claude", str(fake)] + first = subprocess.Popen( + commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=delayed + ) + second = subprocess.Popen( + commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=delayed + ) + outputs = [process.communicate(timeout=30) for process in (first, second)] + assert first.returncode == second.returncode == 0, outputs + values = [json.loads(output[0]) for output in outputs] + assert sorted(value["already_installed"] for value in values) == [False, True] + root = Path(environment["CLAUDE_CONFIG_DIR"]) / "plugins" / "substrate-installer" + assert not (root / "journal.json").exists() + + +@pytest.mark.skipif(os.name != "posix", reason="symlink semantics are POSIX-specific") +def test_installer_rejects_archive_and_config_symlinks_without_touching_victim( + tmp_path: Path, +) -> None: + archive, generated, fake, environment = _generated_fixture(tmp_path) + victim = tmp_path / "victim" + victim.mkdir() + marker = victim / "marker" + marker.write_text("must survive") + + archive_link = tmp_path / "archive-link.zip" + archive_link.symlink_to(archive) + rejected = _run_generated(generated, fake, environment, str(archive_link)) + assert rejected.returncode == 2 + assert marker.read_text() == "must survive" + + dangling_source = ( + Path(environment["CLAUDE_CONFIG_DIR"]) + / "plugins" + / "substrate-installer" + / "releases" + / f"{VERSION}-{_head_sha()}" + / "source" + ) + dangling_source.parent.mkdir(parents=True, exist_ok=True) + dangling_source.symlink_to(tmp_path / "missing-victim", target_is_directory=True) + rejected = _run_generated(generated, fake, environment, str(archive)) + assert rejected.returncode == 2 + assert dangling_source.is_symlink() + + config_link = tmp_path / "config-link" + config_link.symlink_to(victim, target_is_directory=True) + linked_environment = dict(environment) + linked_environment["CLAUDE_CONFIG_DIR"] = str(config_link / "nested") + rejected = _run_generated(generated, fake, linked_environment, str(archive)) + assert rejected.returncode == 2 + assert marker.read_text() == "must survive" + assert list(victim.iterdir()) == [marker] + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX process-group semantics") +def test_hook_launcher_kills_real_sigterm_ignoring_grandchild(tmp_path: Path) -> None: + fake_python = tmp_path / "fake-python" + fake_python.write_text( + f"""#!{sys.executable} +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +if "-c" in sys.argv: + print("3.11") + raise SystemExit(0) +pids = Path(os.environ["SUBSTRATE_TEST_PID_DIR"]) +(pids / "child.pid").write_text(str(os.getpid())) +signal.signal(signal.SIGTERM, signal.SIG_IGN) +code = "import os,signal,time; from pathlib import Path; Path(os.environ['SUBSTRATE_TEST_PID_DIR']).joinpath('grandchild.pid').write_text(str(os.getpid())); signal.signal(signal.SIGTERM,signal.SIG_IGN); time.sleep(60)" +subprocess.Popen([sys.executable, "-c", code], env=os.environ.copy()) +time.sleep(60) +""" + ) + fake_python.chmod(0o700) + environment = os.environ.copy() + environment["SUBSTRATE_PYTHON"] = str(fake_python) + environment["SUBSTRATE_TEST_PID_DIR"] = str(tmp_path) + started = time.monotonic() + result = subprocess.run( + ("node", str(ROOT / "scripts" / "plugin_runtime.cjs"), "hook", "stop"), + input="{}", + capture_output=True, + text=True, + env=environment, + timeout=15, + check=False, + ) + elapsed = time.monotonic() - started + assert result.returncode == 2 + assert 10 <= elapsed < 15 + child_pid = int((tmp_path / "child.pid").read_text()) + grandchild_pid = int((tmp_path / "grandchild.pid").read_text()) + + def alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True + + deadline = time.monotonic() + 2 + while (alive(child_pid) or alive(grandchild_pid)) and time.monotonic() < deadline: + time.sleep(0.02) + assert not alive(child_pid) + assert not alive(grandchild_pid) + + +@pytest.mark.skipif(os.name == "nt", reason="non-Windows cleanup self-test") +def test_windows_cleanup_state_machine_self_test_is_deterministic() -> None: + started = time.monotonic() + result = subprocess.run( + ("node", str(ROOT / "scripts" / "plugin_runtime.cjs"), "self-test-windows-cleanup"), + capture_output=True, + text=True, + timeout=3, + check=False, + ) + assert result.returncode == 0, result.stderr + assert time.monotonic() - started < 3 + + +@pytest.mark.skipif(os.name != "nt", reason="real Windows launcher failure injection") +@pytest.mark.parametrize("behavior", ["failure", "stall"]) +def test_windows_taskkill_failure_or_stall_has_hard_deadline(tmp_path: Path, behavior: str) -> None: + runtime_root = tmp_path / "runtime" + scripts = runtime_root / "scripts" + package = runtime_root / "src" / "claude_code_memory" + scripts.mkdir(parents=True) + package.mkdir(parents=True) + launcher = scripts / "plugin_runtime.cjs" + shutil.copy2(ROOT / "scripts" / "plugin_runtime.cjs", launcher) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "hook.py").write_text("import time\ntime.sleep(60)\n", encoding="utf-8") + environment = os.environ.copy() + environment.update( + SUBSTRATE_PYTHON=sys.executable, + SUBSTRATE_LAUNCHER_SELF_TEST="1", + SUBSTRATE_TASKKILL_TEST_MODE=behavior, + ) + started = time.monotonic() + result = subprocess.run( + ("node", str(launcher), "hook", "stop"), + input="{}", + capture_output=True, + text=True, + env=environment, + timeout=15, + check=False, + ) + elapsed = time.monotonic() - started + assert result.returncode == 2 + assert 10 <= elapsed < 15 + assert "direct child fallback used" in result.stderr + + +def test_portable_launcher_runs_current_isolated_host(tmp_path: Path) -> None: + environment = os.environ.copy() + environment["SUBSTRATE_STATE_HOME"] = str(tmp_path / "state") + result = subprocess.run( + ("node", str(ROOT / "scripts" / "plugin_runtime.cjs"), "status"), + capture_output=True, + text=True, + env=environment, + timeout=10, + check=False, + ) + assert result.returncode == 0 + status = json.loads(result.stdout) + assert status["pending"] == 0 + assert result.stderr == "" + + +def test_release_publish_depends_on_every_platform_gate_and_rechecks_main() -> None: + workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text() + assert "os: [ubuntu-latest, macos-latest, windows-latest]" in workflow + assert 'python: ["3.11", "3.12"]' in workflow + assert "needs: [verify, platform-gates, full-continuation]" in workflow + assert "needs.platform-gates.result == 'success'" in workflow + assert "needs.full-continuation.result == 'success'" in workflow + assert workflow.count("name: full-continuation") == 1 + assert 'SUBSTRATE_FULL_CONTINUATION: "1"' in workflow + assert 'grep -F "1 passed" full-continuation.log' in workflow + assert "grep -Fx 'full-continuation'" in workflow + assert "Recheck protected current main after environment approval" in workflow + assert workflow.count('branches/main" --jq .protected') >= 3 + assert workflow.count('branches/main" --jq .commit.sha') >= 3 + assert "group: release-v0.2.0" in workflow + assert "commits/$SOURCE_SHA/check-runs" in workflow + assert '$1 == "full-continuation"' in workflow + ci = (ROOT / ".github" / "workflows" / "ci.yml").read_text() + assert "name: full-continuation" in ci + assert 'SUBSTRATE_FULL_CONTINUATION: "1"' in ci + assert "needs: [publication-scan, host-contract, full-continuation]" in ci + assert "required_status_checks" in ci + assert "--draft" in workflow + assert "--draft=false" in workflow + assert ( + workflow.index("--draft") + < workflow.index("gh release download") + < workflow.index("--draft=false") + ) + + +def test_windows_default_launcher_probes_supported_py_versions_and_ci_does_not_bypass() -> None: + launcher = (ROOT / "scripts" / "plugin_runtime.cjs").read_text() + assert launcher.index('prefix: ["-3.12"]') < launcher.index('prefix: ["-3.11"]') + assert launcher.index('prefix: ["-3.11"]') < launcher.index('prefix: ["-3"]') + assert 'PYTHONDONTWRITEBYTECODE: "1"' in launcher + assert 'const taskkillCommand = testMode ? process.execPath : "taskkill"' in launcher + assert "TASKKILL_TIMEOUT_MS = 1000" in launcher + assert "result.status === 0 && !result.error && !result.signal" in launcher + assert 'signalTree("SIGTERM")' in launcher + assert 'signalTree("SIGKILL")' in launcher + assert "}, 10000)" in launcher + ci = (ROOT / ".github" / "workflows" / "ci.yml").read_text() + release = (ROOT / ".github" / "workflows" / "release.yml").read_text() + assert "SUBSTRATE_PYTHON:" not in ci + assert "SUBSTRATE_PYTHON:" not in release + setup_node = "actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020" + assert setup_node in ci + assert setup_node in release + assert ci.count('node-version: "22"') >= 2 + assert release.count('node-version: "22"') >= 3 diff --git a/tests/test_continuation_process.py b/tests/test_continuation_process.py new file mode 100644 index 0000000..a6309bf --- /dev/null +++ b/tests/test_continuation_process.py @@ -0,0 +1,192 @@ +"""Real MCP-process continuation and restart acceptance.""" + +from __future__ import annotations + +import io +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from claude_code_memory import hook +from claude_code_memory.profile import state_home + +ROOT = Path(__file__).parents[1] + + +def _record(identifier: str, content: str) -> str: + return ( + json.dumps( + { + "type": "user", + "uuid": identifier, + "message": {"role": "user", "content": content}, + } + ) + + "\n" + ) + + +def _approve() -> Path: + state = state_home() + onboarding = state / "onboarding" + onboarding.mkdir(parents=True, exist_ok=True) + (onboarding / "state.json").write_text(json.dumps({"version": 1, "phase": "ready"})) + return state + + +def _start_mcp(environment: dict[str, str]) -> subprocess.Popen[str]: + return subprocess.Popen( + [sys.executable, "-m", "claude_code_memory.server"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=environment, + cwd=ROOT, + ) + + +def _wait_for_completion(state: Path, process: subprocess.Popen[str], timeout: float) -> None: + deadline = time.monotonic() + timeout + jobs = state / "hook-jobs" + while time.monotonic() < deadline: + assert process.poll() is None, process.stderr.read() if process.stderr else "MCP exited" + if not list(jobs.glob("*.json")): + return + time.sleep(0.05) + raise AssertionError("continuation did not reach its durable target EOF") + + +def _events(state: Path) -> list[dict[str, object]]: + return [json.loads(path.read_text()) for path in sorted((state / "spool").glob("*.json"))] + + +def _assert_complete(state: Path, transcript: Path, expected_records: int) -> None: + jobs = list((state / "hook-jobs").glob("*.json")) + assert jobs == [] + checkpoints = list((state / "checkpoints").glob("*.json")) + assert len(checkpoints) == 1 + checkpoint = json.loads(checkpoints[0].read_text()) + assert checkpoint["cursor"] == transcript.stat().st_size + assert checkpoint["message_index"] == expected_records + assert checkpoint["session_end"] is True + events = _events(state) + event_ids = [event["event_id"] for event in events] + assert len(event_ids) == len(set(event_ids)) + assert sum(event.get("kind") == "session_end" for event in events) == 1 + indexes = { + index + for event in events + for index in event.get("payload", {}).get("message_indexes", []) # type: ignore[union-attr] + } + assert indexes == set(range(expected_records)) + + +@pytest.mark.timeout(240) +@pytest.mark.skipif( + os.name != "posix" or os.environ.get("SUBSTRATE_FULL_CONTINUATION") != "1", + reason="explicit POSIX 50 MiB hostile acceptance gate", +) +def test_full_posix_mcp_sigkill_restart_combined_corpus( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state-root")) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "claude-config")) + state = _approve() + transcript = tmp_path / "combined.jsonl" + with transcript.open("w", encoding="utf-8") as stream: + for index in range(8002): + stream.write(_record(f"small-{index}", f"small message {index}")) + for index in range(500): + stream.write(_record(f"large-{index}", f"large {index} " + "x" * 100_000)) + hook_input = json.dumps({"session_id": "combined-process", "transcript_path": str(transcript)}) + environment = os.environ.copy() + started = time.monotonic() + launched = subprocess.run( + ["node", str(ROOT / "scripts" / "plugin_runtime.cjs"), "hook", "session-end"], + input=hook_input, + capture_output=True, + text=True, + env=environment, + timeout=15, + check=False, + ) + assert launched.returncode == 0 + assert time.monotonic() - started < 5 + checkpoint_path = next((state / "checkpoints").glob("*.json")) + initial_cursor = json.loads(checkpoint_path.read_text())["cursor"] + assert initial_cursor < transcript.stat().st_size + + first = _start_mcp(environment) + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + checkpoint = json.loads(checkpoint_path.read_text()) + if ( + checkpoint["cursor"] > initial_cursor + and checkpoint["cursor"] < transcript.stat().st_size + ): + break + time.sleep(0.01) + else: + first.kill() + raise AssertionError("MCP continuation did not enter the unread suffix") + os.kill(first.pid, signal.SIGKILL) + first.wait(timeout=5) + assert first.returncode == -signal.SIGKILL + assert list((state / "hook-jobs").glob("*.json")) + + second = _start_mcp(environment) + _wait_for_completion(state, second, 150) + assert second.stdin is not None + second.stdin.close() + second.wait(timeout=5) + assert second.returncode == 0 + _assert_complete(state, transcript, 8502) + + +def test_scaled_cross_platform_mcp_restart_without_windows_sigkill_claim( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state-root")) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "split-claude-config")) + state = _approve() + transcript = tmp_path / "scaled.jsonl" + with transcript.open("w", encoding="utf-8") as stream: + for index in range(257): + stream.write(_record(f"small-{index}", f"small message {index}")) + for index in range(20): + stream.write(_record(f"large-{index}", "y" * 20_000)) + started = time.monotonic() + assert ( + hook.run( + "session-end", + stdin=io.StringIO( + json.dumps({"session_id": "scaled-process", "transcript_path": str(transcript)}) + ), + stdout=io.StringIO(), + stderr=io.StringIO(), + authorization_fn=lambda: True, + ) + == 0 + ) + assert time.monotonic() - started < 5 + environment = os.environ.copy() + first = _start_mcp(environment) + time.sleep(0.05) + # This is only a portable restart test. It intentionally makes no claim + # that terminate() on Windows has POSIX SIGKILL durability semantics. + first.terminate() + first.wait(timeout=5) + second = _start_mcp(environment) + _wait_for_completion(state, second, 30) + assert second.stdin is not None + second.stdin.close() + second.wait(timeout=5) + assert second.returncode == 0 + _assert_complete(state, transcript, 277) diff --git a/tests/test_hook_durability.py b/tests/test_hook_durability.py new file mode 100644 index 0000000..6d9de63 --- /dev/null +++ b/tests/test_hook_durability.py @@ -0,0 +1,628 @@ +"""Failure injection for checkpoint truth, cursors, suppression, and concurrency.""" + +from __future__ import annotations + +import io +import json +import multiprocessing +import os +import signal +import time +import threading +from pathlib import Path +from typing import Any + +import pytest + +from claude_code_memory import hook +from claude_code_memory.profile import state_home +from claude_code_memory.transcript import TranscriptWindow, read_message_window +from substrate_capture import CaptureEventBuilder, DurableSpool + + +class SpoolOnly: + def __init__(self, spool: DurableSpool, reject_kind: str = "") -> None: + self.spool = spool + self.reject_kind = reject_kind + + def enqueue(self, event: dict[str, Any]) -> Path | None: + if event.get("kind") == self.reject_kind: + return None + try: + return self.spool.append(event) + except (OSError, ValueError): + return None + + def drain(self) -> dict[str, int]: + return {"pending": len(self.spool)} + + +class Harness: + def __init__( + self, root: Path, *, spool: DurableSpool | None = None, reject_kind: str = "" + ) -> None: + self.spool = spool if spool is not None else DurableSpool(root / "spool") + self.deliverer = SpoolOnly(self.spool, reject_kind) + self.builder = CaptureEventBuilder( + { + "platform": "claude_code", + "agent_id": "test-tenant", + "subject_id": "test-tenant", + }, + provider_id="claude_code_memory", + ) + + def runtime(self) -> tuple[None, DurableSpool, SpoolOnly, CaptureEventBuilder]: + return None, self.spool, self.deliverer, self.builder + + +def transcript(path: Path, count: int = 1, *, sidechain: bool = False) -> Path: + with path.open("w", encoding="utf-8") as stream: + for index in range(count): + stream.write( + json.dumps( + { + "type": "user", + "uuid": f"u-{index}", + "isSidechain": sidechain, + "message": {"role": "user", "content": f"synthetic message {index}"}, + } + ) + + "\n" + ) + return path + + +def invoke(event: str, source: Path, harness: Harness, **extra: object) -> int: + data = { + "session_id": "session-1", + "transcript_path": str(source), + "cwd": str(source.parent), + **extra, + } + return hook.run( + event, + stdin=io.StringIO(json.dumps(data)), + stdout=io.StringIO(), + stderr=io.StringIO(), + runtime_factory=harness.runtime, # type: ignore[arg-type] + authorization_fn=lambda: True, + ) + + +def checkpoint_files() -> list[Path]: + root = state_home() / "checkpoints" + return list(root.glob("*.json")) if root.exists() else [] + + +def test_preapproval_hook_never_reads_or_spools_transcript(tmp_path: Path) -> None: + source = transcript(tmp_path / "transcript.jsonl") + + def forbidden_runtime() -> Any: + raise AssertionError("runtime must not be built before approval") + + result = hook.run( + "stop", + stdin=io.StringIO(json.dumps({"session_id": "s", "transcript_path": str(source)})), + stdout=io.StringIO(), + stderr=io.StringIO(), + runtime_factory=forbidden_runtime, + authorization_fn=lambda: False, + ) + assert result == 0 + + +def test_first_primary_session_start_triggers_onboarding_without_recall( + monkeypatch: pytest.MonkeyPatch, +) -> None: + called: list[bool] = [] + monkeypatch.setattr(hook, "_bootstrap_onboarding", lambda _errors: called.append(True)) + result = hook.run( + "session-start", + stdin=io.StringIO(json.dumps({"session_id": "primary"})), + stdout=io.StringIO(), + stderr=io.StringIO(), + recall_fn=lambda _limit, _data: (_ for _ in ()).throw(AssertionError("no recall")), + authorization_fn=lambda: False, + ) + assert result == 0 + assert called == [True] + + +def test_full_spool_does_not_advance_checkpoint( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + full = DurableSpool(tmp_path / "runtime" / "spool", max_items=1) + harness = Harness(tmp_path / "runtime", spool=full) + invoke("stop", transcript(tmp_path / "t.jsonl"), harness) + assert checkpoint_files() == [] + assert len(full) == 0 + + +def test_spool_fsync_error_does_not_advance_checkpoint( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + harness = Harness(tmp_path / "runtime") + monkeypatch.setattr( + harness.spool, "append", lambda _event: (_ for _ in ()).throw(OSError("fsync")) + ) + invoke("stop", transcript(tmp_path / "t.jsonl"), harness) + assert checkpoint_files() == [] + + +def test_checkpoint_rename_error_leaves_stable_event_for_duplicate_retry( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + harness = Harness(tmp_path / "runtime") + source = transcript(tmp_path / "t.jsonl") + import claude_code_memory.state as state_module + + original = state_module.secure_atomic_json_write + monkeypatch.setattr( + state_module, + "secure_atomic_json_write", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("rename")), + ) + invoke("stop", source, harness) + assert len(harness.spool) == 1 + assert checkpoint_files() == [] + monkeypatch.setattr(state_module, "secure_atomic_json_write", original) + invoke("stop", source, harness) + assert len(harness.spool) == 1 + assert len(checkpoint_files()) == 1 + assert harness.spool.statistics()["duplicates"] == 1 + + +def test_session_end_marker_commits_only_after_boundary_admission( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + harness = Harness(tmp_path / "runtime", reject_kind="session_end") + source = transcript(tmp_path / "t.jsonl") + invoke("session-end", source, harness) + state = json.loads(checkpoint_files()[0].read_text()) + assert state["session_end"] is False + harness.deliverer.reject_kind = "" + invoke("session-end", source, harness) + state = json.loads(checkpoint_files()[0].read_text()) + assert state["session_end"] is True + kinds = [harness.spool.load(path)["kind"] for path in sorted(harness.spool.root.glob("*.json"))] + assert kinds.count("session_end") == 1 + + +def test_concurrent_hooks_admit_one_deterministic_event( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + source = transcript(tmp_path / "t.jsonl") + harness = Harness(tmp_path / "runtime") + threads = [threading.Thread(target=invoke, args=("stop", source, harness)) for _ in range(6)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert len(harness.spool) == 1 + assert len(checkpoint_files()) == 1 + + +def test_unreadable_source_does_not_mint_state( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + harness = Harness(tmp_path / "runtime") + invoke("stop", tmp_path / "missing.jsonl", harness) + assert checkpoint_files() == [] + assert len(harness.spool) == 0 + + +def test_cursor_reaches_suffix_after_two_thousand_records(tmp_path: Path) -> None: + source = transcript(tmp_path / "large.jsonl", 2001) + first = read_message_window(source) + second = read_message_window( + source, + cursor=first.next_cursor, + expected_source_id=first.source_id, + expected_anchor=first.anchor, + ) + assert len(first.messages) == 2000 + assert first.complete is False + assert len(second.messages) == 1 + assert second.complete is True + assert second.next_cursor > first.next_cursor + assert second.skipped_records == 0 + assert second.messages[0]["content"] == "synthetic message 2000" + + +def test_truncated_final_record_is_retried_without_cursor_advance(tmp_path: Path) -> None: + source = tmp_path / "growing.jsonl" + first_line = json.dumps({"type": "user", "message": {"role": "user", "content": "complete"}}) + source.write_bytes((first_line + "\n" + '{"type":"user","message":').encode("utf-8")) + first = read_message_window(source) + assert len(first.messages) == 1 + assert first.complete is False + assert first.next_cursor == len((first_line + "\n").encode()) + with source.open("a", encoding="utf-8") as stream: + stream.write('{"role":"user","content":"now complete"}}\n') + second = read_message_window( + source, + cursor=first.next_cursor, + expected_source_id=first.source_id, + expected_anchor=first.anchor, + ) + assert second.complete is True + assert len(second.messages) == 1 + + +def test_sidechain_and_subagent_are_always_suppressed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + harness = Harness(tmp_path / "runtime") + invoke("stop", transcript(tmp_path / "side.jsonl", sidechain=True), harness) + assert len(harness.spool) == 0 + before = list(checkpoint_files()) + invoke("stop", transcript(tmp_path / "primary.jsonl"), harness, agent_id="agent-123") + assert len(harness.spool) == 0 + assert checkpoint_files() == before + + +def _capture_process( + state_root: str, + runtime_root: str, + source: str, + start: Any, + outcome: Any, + *, + kill_after_admission: bool = False, + admission_signal: Any = None, +) -> None: + os.environ["SUBSTRATE_STATE_HOME"] = state_root + spool = DurableSpool(Path(runtime_root) / "spool") + builder = CaptureEventBuilder( + {"platform": "claude_code", "agent_id": "process", "subject_id": "process"}, + provider_id="claude_code_memory", + ) + + class ProcessDeliverer: + def enqueue(self, event: dict[str, Any]) -> Path | None: + admitted = spool.append(event) + if kill_after_admission: + if admission_signal is not None: + admission_signal.set() + os.kill(os.getpid(), signal.SIGKILL) + return admitted + + def drain(self) -> dict[str, int]: + return {"pending": len(spool)} + + def factory() -> tuple[None, DurableSpool, ProcessDeliverer, CaptureEventBuilder]: + return None, spool, ProcessDeliverer(), builder + + start.wait() + completed = hook._capture_transcript( # type: ignore[attr-defined] + "stop", + {"session_id": "process-session", "transcript_path": source}, + factory, # type: ignore[arg-type] + ) + outcome.put(("completed", completed)) + + +@pytest.mark.skipif(os.name != "posix", reason="SIGKILL durability test is POSIX-specific") +def test_real_process_kill_after_admission_then_restart_commits_checkpoint( + tmp_path: Path, +) -> None: + source = transcript(tmp_path / "kill.jsonl") + state_root = tmp_path / "state" + runtime_root = tmp_path / "runtime" + context = multiprocessing.get_context("fork") + first_start = context.Event() + outcomes = context.Queue() + admission_signal = context.Event() + first = context.Process( + target=_capture_process, + args=(str(state_root), str(runtime_root), str(source), first_start, outcomes), + kwargs={"kill_after_admission": True, "admission_signal": admission_signal}, + ) + first.start() + first_start.set() + assert admission_signal.wait(timeout=5) + first.join(timeout=5) + assert first.exitcode == -signal.SIGKILL + assert len(DurableSpool(runtime_root / "spool")) == 1 + assert not list((state_root).rglob("checkpoints/*.json")) + + second_start = context.Event() + second = context.Process( + target=_capture_process, + args=(str(state_root), str(runtime_root), str(source), second_start, outcomes), + ) + second.start() + second_start.set() + second.join(timeout=10) + assert second.exitcode == 0 + assert outcomes.get(timeout=2) == ("completed", True) + assert len(DurableSpool(runtime_root / "spool")) == 1 + checkpoints = list(state_root.rglob("checkpoints/*.json")) + assert len(checkpoints) == 1 + assert json.loads(checkpoints[0].read_text())["cursor"] == source.stat().st_size + + +@pytest.mark.skipif(os.name != "posix", reason="process flock test is POSIX-specific") +def test_independent_hook_processes_share_one_checkpoint_transaction(tmp_path: Path) -> None: + source = transcript(tmp_path / "concurrent.jsonl") + state_root = tmp_path / "state" + runtime_root = tmp_path / "runtime" + context = multiprocessing.get_context("fork") + start = context.Event() + outcomes = context.Queue() + processes = [ + context.Process( + target=_capture_process, + args=(str(state_root), str(runtime_root), str(source), start, outcomes), + ) + for _ in range(4) + ] + for process in processes: + process.start() + start.set() + for process in processes: + process.join(timeout=10) + assert process.exitcode == 0 + assert [outcomes.get(timeout=2) for _ in processes].count(("completed", True)) == 4 + assert len(DurableSpool(runtime_root / "spool")) == 1 + assert len(list(state_root.rglob("checkpoints/*.json"))) == 1 + + +def test_production_session_end_queues_durable_job_below_host_bound( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + source = transcript(tmp_path / "large.jsonl", 8001) + started = time.monotonic() + result = hook.run( + "session-end", + stdin=io.StringIO( + json.dumps({"session_id": "large-session", "transcript_path": str(source)}) + ), + stdout=io.StringIO(), + stderr=io.StringIO(), + authorization_fn=lambda: True, + ) + elapsed = time.monotonic() - started + assert result == 0 + assert elapsed < 5 + jobs = list((state_home() / "hook-jobs").glob("*.json")) + assert len(jobs) == 1 + value = json.loads(jobs[0].read_text()) + assert value["event"] == "session-end" + assert value["target_cursor"] == source.stat().st_size + checkpoint = json.loads(next((state_home() / "checkpoints").glob("*.json")).read_text()) + assert 0 < checkpoint["message_index"] <= 32 + assert checkpoint["session_end"] is False + + +def test_mcp_continuation_protocol_completes_without_detached_hook_worker( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + source = transcript(tmp_path / "worker.jsonl") + harness = Harness(tmp_path / "runtime") + identifier = hook._queue_job( # type: ignore[attr-defined] + "session-end", {"session_id": "worker-session", "transcript_path": str(source)} + ) + assert identifier is not None + assert ( + hook.process_pending_jobs( + runtime_factory=harness.runtime, # type: ignore[arg-type] + authorization_fn=lambda: True, + maximum=4, + ) + == 1 + ) + assert not (state_home() / "hook-jobs" / f"{identifier}.json").exists() + checkpoint = next((state_home() / "checkpoints").glob("*.json")) + assert json.loads(checkpoint.read_text())["session_end"] is True + kinds = {harness.spool.load(path)["kind"] for path in harness.spool.root.glob("*.json")} + assert kinds == {"turn", "session_end"} + + +def test_session_end_has_no_eight_thousand_record_cap_with_fast_windows( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + harness = Harness(tmp_path / "runtime") + windows = 63 + calls = 0 + + def fake_window(*_args: Any, **_kwargs: Any) -> TranscriptWindow: + nonlocal calls + calls += 1 + count = 65 if calls == windows else 128 + messages = [{"role": "user", "content": "x"} for _ in range(count)] + return TranscriptWindow( + messages=messages, + next_cursor=calls, + source_id="synthetic-source", + readable=True, + complete=calls == windows, + skipped_records=0, + reset=False, + anchor=f"anchor-{calls}", + ) + + monkeypatch.setattr(hook, "read_message_window", fake_window) + completed = hook._capture_transcript( # type: ignore[attr-defined] + "session-end", + {"session_id": "large-fast", "transcript_path": str(tmp_path / "unused")}, + harness.runtime, # type: ignore[arg-type] + ) + assert completed is True + assert calls == windows + checkpoint = json.loads(checkpoint_files()[0].read_text()) + assert checkpoint["message_index"] == 8001 + assert checkpoint["session_end"] is True + assert any( + harness.spool.load(path)["kind"] == "session_end" + for path in harness.spool.root.glob("*.json") + ) + + +def test_malformed_record_does_not_admit_or_advance_prior_valid_content( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + source = transcript(tmp_path / "malformed.jsonl") + with source.open("a", encoding="utf-8") as stream: + stream.write("{malformed}\n") + harness = Harness(tmp_path / "runtime") + invoke("stop", source, harness) + assert checkpoint_files() == [] + assert len(harness.spool) == 0 + + +def test_partial_snapshot_fairness_and_exact_boundary_recovery( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + harness = Harness(tmp_path / "runtime") + partials: list[tuple[Path, str]] = [] + for index in range(4): + complete = json.dumps( + { + "type": "user", + "uuid": f"partial-{index}", + "message": {"role": "user", "content": f"recovered-{index}"}, + } + ) + split = len(complete) // 2 + source = tmp_path / f"partial-{index}.jsonl" + source.write_text(complete[:split], encoding="utf-8") + partials.append((source, complete[split:])) + assert ( + hook.run( + "session-end", + stdin=io.StringIO( + json.dumps( + {"session_id": f"partial-session-{index}", "transcript_path": str(source)} + ) + ), + stdout=io.StringIO(), + stderr=io.StringIO(), + runtime_factory=harness.runtime, # type: ignore[arg-type] + authorization_fn=lambda: True, + ) + == 0 + ) + + good = transcript(tmp_path / "good.jsonl") + assert ( + hook.run( + "session-end", + stdin=io.StringIO( + json.dumps({"session_id": "later-good", "transcript_path": str(good)}) + ), + stdout=io.StringIO(), + stderr=io.StringIO(), + runtime_factory=harness.runtime, # type: ignore[arg-type] + authorization_fn=lambda: True, + ) + == 0 + ) + jobs = list((state_home() / "hook-jobs").glob("*.json")) + assert len(jobs) == 4 + good_state = json.loads( + next( + path + for path in (state_home() / "checkpoints").glob("*.json") + if json.loads(path.read_text())["session_end"] is True + ).read_text() + ) + assert good_state["message_index"] == 1 + + for index, (source, suffix) in enumerate(partials): + with source.open("a", encoding="utf-8") as stream: + stream.write(suffix + "\n") + stream.write( + json.dumps( + { + "type": "user", + "uuid": f"too-late-{index}", + "message": {"role": "user", "content": f"must-not-capture-{index}"}, + } + ) + + "\n" + ) + time.sleep(0.06) + assert ( + hook.process_pending_jobs( + runtime_factory=harness.runtime, # type: ignore[arg-type] + authorization_fn=lambda: True, + maximum=4, + ) + == 4 + ) + assert list((state_home() / "hook-jobs").glob("*.json")) == [] + events = [harness.spool.load(path) for path in harness.spool.root.glob("*.json")] + serialized = json.dumps(events) + for index in range(4): + assert serialized.count(f"recovered-{index}") == 1 + assert f"must-not-capture-{index}" not in serialized + assert len([event for event in events if event["kind"] == "session_end"]) == 5 + + +def test_scheduler_does_not_slice_four_blocked_jobs_before_runnable_session_end( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + harness = Harness(tmp_path / "runtime") + for index in range(4): + source = tmp_path / f"mid-record-{index}.jsonl" + source.write_text('{"type":"user"', encoding="utf-8") + assert hook._queue_job( # type: ignore[attr-defined] + "session-end", {"session_id": f"blocked-{index}", "transcript_path": str(source)} + ) + good = transcript(tmp_path / "runnable.jsonl") + assert hook._queue_job( # type: ignore[attr-defined] + "session-end", {"session_id": "runnable", "transcript_path": str(good)} + ) + assert ( + hook.process_pending_jobs( + runtime_factory=harness.runtime, # type: ignore[arg-type] + authorization_fn=lambda: True, + maximum=4, + ) + == 1 + ) + assert len(list((state_home() / "hook-jobs").glob("*.json"))) == 4 + checkpoint = json.loads(next((state_home() / "checkpoints").glob("*.json")).read_text()) + assert checkpoint["session_end"] is True + + +def test_bounded_job_queue_reports_non_admission( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + monkeypatch.setattr(hook, "_MAX_PENDING_JOBS", 4) + for index in range(4): + source = tmp_path / f"blocked-{index}.jsonl" + source.write_text('{"type":"user"', encoding="utf-8") + assert hook._queue_job( # type: ignore[attr-defined] + "session-end", + {"session_id": f"blocked-{index}", "transcript_path": str(source)}, + ) + overflow = tmp_path / "overflow.jsonl" + overflow.write_text('{"type":"user"', encoding="utf-8") + errors = io.StringIO() + result = hook.run( + "session-end", + stdin=io.StringIO(json.dumps({"session_id": "overflow", "transcript_path": str(overflow)})), + stdout=io.StringIO(), + stderr=errors, + authorization_fn=lambda: True, + ) + assert result == 0 + assert "queue unavailable; capture not admitted" in errors.getvalue() + assert len(list((state_home() / "hook-jobs").glob("*.json"))) == 4 diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 73893ee..7e8f5cb 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -88,10 +88,15 @@ def _invoke(event: str, transcript: Path, harness: RuntimeHarness) -> tuple[int, stdout=stdout, stderr=stderr, runtime_factory=harness.build, + authorization_fn=lambda: True, ) return result, stdout.getvalue(), stderr.getvalue() +def _spooled(harness: RuntimeHarness) -> list[dict[str, Any]]: + return [harness.spool.load(path) for path in harness.spool._files_locked()] + + @pytest.mark.parametrize( ("event", "kind", "endpoint"), [ @@ -113,8 +118,10 @@ def test_hook_events_map_to_capture_kind_and_endpoint( assert result == 0 assert stdout == "" assert stderr == "" - assert harness.client.requests[0]["path"] == endpoint - assert harness.client.requests[0]["body"]["kind"] == kind + matching = [event for event in _spooled(harness) if event["kind"] == kind] + assert len(matching) == 1 + assert harness.client.requests == [] + assert endpoint.startswith("/api/v1/hermes/") def test_second_identical_stop_emits_no_duplicate( @@ -125,7 +132,7 @@ def test_second_identical_stop_emits_no_duplicate( harness = RuntimeHarness(tmp_path / "runtime") assert _invoke("stop", transcript, harness)[0] == 0 assert _invoke("stop", transcript, harness)[0] == 0 - assert len(harness.client.requests) == 1 + assert len(harness.spool) == 1 def test_session_end_is_content_free_and_emits_at_most_once( @@ -136,29 +143,50 @@ def test_session_end_is_content_free_and_emits_at_most_once( harness = RuntimeHarness(tmp_path / "runtime") _invoke("session-end", transcript, harness) _invoke("session-end", transcript, harness) - assert len(harness.client.requests) == 1 - event = harness.client.requests[0]["body"] + boundaries = [event for event in _spooled(harness) if event["kind"] == "session_end"] + assert len(boundaries) == 1 + event = boundaries[0] assert event["capture_boundary"] == {"start": 0, "end": 1} assert event["payload"]["summary"] == { "message_count": 1, "boundary": "session_end", + "loss_count": 0, } assert "hello from transcript" not in json.dumps(event) -def test_session_start_prints_only_recall_context( +def test_session_start_never_runs_generic_recall_and_prompt_hook_uses_exact_prompt( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + calls: list[dict[str, Any]] = [] stdout = io.StringIO() - result = hook.run( - "session-start", - stdin=io.StringIO("{}"), - stdout=stdout, - stderr=io.StringIO(), - recall_fn=lambda limit: f"remembered {limit}\n", + assert ( + hook.run( + "session-start", + stdin=io.StringIO(json.dumps({"session_id": "s", "cwd": "/tmp/project"})), + stdout=stdout, + stderr=io.StringIO(), + recall_fn=lambda _limit, data: calls.append(data) or "wrong", + authorization_fn=lambda: True, + ) + == 0 ) - assert result == 0 + assert stdout.getvalue() == "" + assert calls == [] + + assert ( + hook.run( + "user-prompt-submit", + stdin=io.StringIO(json.dumps({"session_id": "s", "prompt": "exact question"})), + stdout=stdout, + stderr=io.StringIO(), + recall_fn=lambda limit, data: calls.append(data) or f"remembered {limit}\n", + authorization_fn=lambda: True, + ) + == 0 + ) + assert calls == [{"session_id": "s", "prompt": "exact question"}] assert stdout.getvalue() == "remembered 5\n" @@ -223,4 +251,53 @@ def test_hook_honors_sidechain_capture_kill_switch( ) harness = RuntimeHarness(tmp_path / "runtime") assert _invoke("stop", transcript, harness)[0] == 0 - assert harness.client.requests == [] + assert len(harness.spool) == 0 + + +@pytest.mark.parametrize( + "marker", + [ + {"agent_id": "agent-abc123", "agent_type": "Explore"}, + {"agent_transcript_path": "/tmp/agent.jsonl"}, + {"hook_event_name": "SubagentStop"}, + {"isSidechain": "true"}, + {"is_subagent": "TRUE"}, + ], +) +def test_actual_host_subagent_shapes_fail_before_authorization_or_runtime( + marker: dict[str, Any], +) -> None: + data = {"session_id": "s", "transcript_path": "/must/not/read", **marker} + assert ( + hook.run( + "stop", + stdin=io.StringIO(json.dumps(data)), + stdout=io.StringIO(), + stderr=io.StringIO(), + runtime_factory=lambda: (_ for _ in ()).throw(AssertionError("runtime")), + authorization_fn=lambda: (_ for _ in ()).throw(AssertionError("authorization")), + ) + == 0 + ) + + +@pytest.mark.parametrize( + "shape", + [ + {"agent_type": "Explore"}, + {"source": "fork", "parent_session_id": "parent-primary"}, + ], +) +def test_actual_primary_agent_and_fork_shapes_are_not_suppressed(shape: dict[str, Any]) -> None: + called: list[bool] = [] + assert ( + hook.run( + "session-start", + stdin=io.StringIO(json.dumps({"session_id": "s", **shape})), + stdout=io.StringIO(), + stderr=io.StringIO(), + authorization_fn=lambda: called.append(True) or True, + ) + == 0 + ) + assert called == [True] diff --git a/tests/test_host_delivery.py b/tests/test_host_delivery.py new file mode 100644 index 0000000..92cacdc --- /dev/null +++ b/tests/test_host_delivery.py @@ -0,0 +1,163 @@ +"""Auth-safe durable delivery, scheduling, and persistent health tests.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from claude_code_memory.delivery import HostedDeliverer +from claude_code_memory.onboarding import OnboardingManager +from claude_code_memory.credentials import CredentialStore +from substrate_capture import DurableSpool, SubstrateAPIError + + +class MemoryStore(CredentialStore): + backend = "memory" + + def __init__(self, value: str = "test-access-placeholder") -> None: + self.values = {"access-token": value} if value else {} + + def get(self, slot: str = "access-token") -> str: + return self.values.get(slot, "") + + def put(self, value: str, slot: str = "access-token") -> None: + self.values[slot] = value + + def delete(self, slot: str = "access-token") -> None: + self.values.pop(slot, None) + + +class FakeClient: + def __init__(self, failures: list[tuple[str, float | None]] | None = None) -> None: + self.failures = list(failures or []) + self.requests: list[dict[str, Any]] = [] + + def request(self, method: str, path: str, **kwargs: Any) -> dict[str, bool]: + self.requests.append({"method": method, "path": path, **kwargs}) + if self.failures: + category, retry_after = self.failures.pop(0) + raise SubstrateAPIError(category, retry_after=retry_after) + return {"accepted": True} + + +def event(event_id: str = "event-1") -> dict[str, object]: + return {"event_id": event_id, "kind": "turn", "capture_kind": "turn", "payload": {}} + + +def deliverer( + root: Path, + client: FakeClient | None, + store: MemoryStore, + *, + now: list[float] | None = None, +) -> HostedDeliverer: + manager = OnboardingManager(root, store=store) + manager.begin(open_browser=False) + wall = now or [100.0] + return HostedDeliverer( + DurableSpool(root / "spool"), + client, # type: ignore[arg-type] + manager, + root, + wall_clock=lambda: wall[0], + ) + + +@pytest.mark.parametrize("category", ["http_401", "http_403"]) +def test_auth_rejection_keeps_event_deletes_auth_and_requires_repair( + tmp_path: Path, category: str +) -> None: + store = MemoryStore() + worker = deliverer(tmp_path, FakeClient([(category, None)]), store) + assert worker.enqueue(event()) is not None + status = worker.drain() + assert status["pending"] == 1 + assert status["auth_repairs"] == 1 + assert status["onboarding_phase"] == "repair_required" + assert store.get() == "" + assert not list((tmp_path / "spool" / "corrupt").glob("*.bad")) + + +def test_timeout_and_retry_after_are_durable_across_restart(tmp_path: Path) -> None: + now = [100.0] + store = MemoryStore() + first = deliverer(tmp_path, FakeClient([("http_429", 120.0)]), store, now=now) + first.enqueue(event()) + status = first.drain() + assert status["pending"] == 1 + assert status["retry_scheduled"] is True + second_client = FakeClient() + second = deliverer(tmp_path, second_client, store, now=now) + assert second.drain()["pending"] == 1 + assert second_client.requests == [] + now[0] = 221.0 + assert second.drain()["pending"] == 0 + assert len(second_client.requests) == 1 + + +def test_timeout_is_retained_but_permanent_event_rejection_is_quarantined(tmp_path: Path) -> None: + store = MemoryStore() + transient = deliverer(tmp_path / "transient", FakeClient([("timeout", None)]), store) + transient.enqueue(event()) + assert transient.drain()["pending"] == 1 + + other_store = MemoryStore() + permanent = deliverer(tmp_path / "permanent", FakeClient([("http_422", None)]), other_store) + permanent.enqueue(event()) + status = permanent.drain() + assert status["pending"] == 0 + assert status["quarantined"] == 1 + + +def test_status_is_persistent_content_free_and_pending_is_actual_depth(tmp_path: Path) -> None: + store = MemoryStore() + first = deliverer(tmp_path, None, store) + first.enqueue(event("opaque-event-id")) + first.drain() + second = deliverer(tmp_path, None, store) + status = second.status() + assert status["pending"] == len(second.spool) == 1 + assert status["deferred"] == 1 + rendered = repr(status) + assert "opaque-event-id" not in rendered + assert "payload" not in rendered + + +def test_disconnect_clears_current_connection_and_capability_truth(tmp_path: Path) -> None: + store = MemoryStore() + worker = deliverer(tmp_path, FakeClient(), store) + worker.enqueue(event()) + connected = worker.drain() + assert connected["connected"] is True + assert connected["capability_validated"] is True + worker.onboarding.disconnect_local() + disconnected = worker.status() + assert disconnected["credential_present"] is False + assert disconnected["onboarding_phase"] == "new" + assert disconnected["connected"] is False + assert disconnected["capability_validated"] is False + assert disconnected["last_success_recorded"] is True + assert disconnected["capability_validation_recorded"] is True + + +def test_success_receipt_survives_power_loss_before_event_removal(tmp_path: Path) -> None: + store = MemoryStore() + client = FakeClient() + first = deliverer(tmp_path, client, store) + first.enqueue(event("receipt-event")) + original_remove = first.spool.remove + first.spool.remove = lambda _path: (_ for _ in ()).throw(OSError("power loss")) # type: ignore[method-assign] + with pytest.raises(OSError, match="power loss"): + first.drain() + assert len(first.spool) == 1 + assert len(client.requests) == 1 + + restarted_client = FakeClient() + restarted = deliverer(tmp_path, restarted_client, store) + restarted.spool.remove = original_remove # type: ignore[method-assign] + status = restarted.drain() + assert status["pending"] == 0 + assert status["delivered"] == 1 + assert restarted_client.requests == [] diff --git a/tests/test_host_runtime.py b/tests/test_host_runtime.py new file mode 100644 index 0000000..465f855 --- /dev/null +++ b/tests/test_host_runtime.py @@ -0,0 +1,94 @@ +"""Fixed origin, custody-only credentials, and stable tenant-local identity.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +import claude_code_memory +from claude_code_memory.contract import HOSTED_ORIGIN +from claude_code_memory.profile import capture_scope + + +class Store: + backend = "test" + + def get(self, slot: str = "access-token") -> str: + return "test-custody-placeholder" if slot == "access-token" else "" + + +def test_runtime_ignores_environment_bearer_origin_and_agent_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path)) + monkeypatch.setenv("SUBSTRATE_API_URL", "https://foreign.example.test") + monkeypatch.setenv("SUBSTRATE_API_KEY", "test-environment-placeholder") + monkeypatch.setenv("CLAUDE_CODE_AGENT_ID", "unstable-agent") + monkeypatch.setattr(claude_code_memory, "credential_store", lambda _home: Store()) + client, _spool, _deliverer, builder = claude_code_memory.runtime() + assert client is not None + assert client.base_url == HOSTED_ORIGIN + assert client.api_key == "test-custody-placeholder" + assert "test-environment-placeholder" not in builder.secrets + assert builder.scope["agent_id"] != "unstable-agent" + assert builder.scope["agent_id"] == builder.scope["agent_identity"] + assert builder.scope["user_id"] == builder.scope["agent_id"] + assert builder.scope["subject_id"] != builder.scope["user_id"] + + +def test_profile_identity_is_stable_and_shared_by_capture_and_recall( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "one")) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "claude-profile")) + first = capture_scope() + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "two")) + monkeypatch.setenv("CLAUDE_CODE_AGENT_ID", "different") + second = capture_scope() + assert first == second + assert first["user_id"] == first["agent_id"] == first["agent_identity"] + assert "subject_id" not in first + + +def test_capture_and_private_recall_share_exact_server_accepted_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "claude")) + scope = capture_scope() + assert set(scope) == {"platform", "user_id", "agent_id", "agent_identity", "profile"} + assert scope["user_id"] == scope["agent_id"] == scope["agent_identity"] + + from substrate_capture import CaptureEventBuilder, SubstrateClient + import hashlib + + builder = CaptureEventBuilder(scope, provider_id="claude_code_memory") + expected_subject = hashlib.sha256( + f"claude_code\0{scope['user_id']}".encode("utf-8") + ).hexdigest()[:24] + assert builder.scope["subject_id"] == expected_subject + captured: list[dict[str, Any]] = [] + + class RecordingClient(SubstrateClient): + def request(self, method: str, path: str, **kwargs: Any) -> Any: + captured.append({"method": method, "path": path, **kwargs}) + return {"results": []} + + client = RecordingClient("https://app.trysubstrate.co", "synthetic") + client._capability_profile = {"memory_search": True} + client.memory_search("exact prompt", limit=5, scope=scope) + assert captured == [ + { + "method": "POST", + "path": "/api/v1/hermes/memory/search", + "body": { + "q": "exact prompt", + "limit": 5, + "platform": "claude_code", + "user_id": scope["user_id"], + "agent_id": scope["agent_id"], + "agent_identity": scope["agent_identity"], + }, + } + ] diff --git a/tests/test_host_tools.py b/tests/test_host_tools.py new file mode 100644 index 0000000..f8c8c1d --- /dev/null +++ b/tests/test_host_tools.py @@ -0,0 +1,196 @@ +"""Five network operations and truthful Claude operational adapters.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from claude_code_memory.contract import HOSTED_ORIGIN +from claude_code_memory.strict_client import StrictHostedClient +from claude_code_memory.tools import build_tools +from substrate_capture import CaptureEventBuilder + + +class Deliverer: + def __init__(self, admitted: bool) -> None: + self.admitted = admitted + self.events: list[dict[str, Any]] = [] + + def enqueue(self, event: dict[str, Any]) -> Path | None: + if not self.admitted: + return None + self.events.append(event) + return Path("synthetic-spool-entry") + + def drain(self) -> dict[str, int]: + return {"pending": len(self.events)} + + def status(self) -> dict[str, Any]: + return {"pending": len(self.events), "credential_present": False} + + +def tools(deliverer: Deliverer, client: Any = None) -> dict[str, Any]: + builder = CaptureEventBuilder( + {"platform": "claude_code", "agent_id": "tenant", "subject_id": "tenant"}, + provider_id="claude_code_memory", + ) + return { + item.name: item + for item in build_tools(client=client, deliverer=deliverer, builder=builder) # type: ignore[arg-type] + } + + +def test_five_bounded_network_operations_and_two_documented_adapters() -> None: + names = set(tools(Deliverer(True))) + assert { + "substrate_search", + "substrate_read", + "substrate_query", + "substrate_ingest", + "substrate_job_status", + } <= names + assert names == { + "substrate_search", + "substrate_read", + "substrate_query", + "substrate_ingest", + "substrate_job_status", + "substrate_sync", + "substrate_status", + } + + +class RecordingClient: + def __init__(self) -> None: + self.calls: list[tuple[str, tuple[Any, ...], dict[str, Any]]] = [] + + def _record(self, name: str, *args: Any, **kwargs: Any) -> dict[str, Any]: + self.calls.append((name, args, kwargs)) + if name == "search": + return {"results": [{"path": "notes/example.md"}]} + return {"operation": name} + + def search(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + return self._record("search", *args, **kwargs) + + def read_page(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + return self._record("read_page", *args, **kwargs) + + def query_wiki(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + return self._record("query_wiki", *args, **kwargs) + + def ingest(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + return self._record("ingest", *args, **kwargs) + + def job_status(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + return self._record("job_status", *args, **kwargs) + + +def test_five_network_handlers_match_hermes_request_shapes_and_bounds() -> None: + client = RecordingClient() + selected = tools(Deliverer(True), client) + + assert selected["substrate_search"].handler({"query": "needle", "limit": 25}) == { + "results": [{"path": "notes/example.md"}], + } + assert selected["substrate_read"].handler({"path": "notes/example.md"}) == { + "operation": "read_page" + } + assert selected["substrate_query"].handler( + {"question": "q" * 16_384, "save_as_synthesis": True} + ) == {"operation": "query_wiki"} + assert selected["substrate_ingest"].handler( + {"content": "body", "title": "Synthetic", "source_type": "url"} + ) == {"operation": "ingest"} + assert selected["substrate_job_status"].handler({"job_id": "j" * 512}) == { + "operation": "job_status" + } + + assert client.calls == [ + ("search", ("needle",), {"limit": 25}), + ("read_page", ("notes/example.md",), {}), + ("query_wiki", ("q" * 16_384,), {"save_as_synthesis": True}), + ( + "ingest", + ("body",), + {"title": "Synthetic", "source_type": "url"}, + ), + ("job_status", ("j" * 512,), {}), + ] + + before = list(client.calls) + assert selected["substrate_query"].handler({"question": "q" * 16_385}) == { + "error": "invalid_arguments" + } + assert selected["substrate_ingest"].handler({"content": "body", "source_type": "file"}) == { + "error": "invalid_arguments" + } + assert selected["substrate_job_status"].handler({"job_id": "j" * 513}) == { + "error": "invalid_arguments" + } + assert client.calls == before + + assert selected["substrate_search"].handler({"query": "needle", "limit": 99}) == { + "results": [{"path": "notes/example.md"}] + } + assert client.calls[-1] == ("search", ("needle",), {"limit": 25}) + before = list(client.calls) + assert selected["substrate_query"].handler( + {"question": "why", "save_as_synthesis": "false"} + ) == {"error": "invalid_arguments"} + assert client.calls == before + + ingest_schema = selected["substrate_ingest"].input_schema + assert ingest_schema["properties"]["content"]["maxLength"] == 262_144 + assert ingest_schema["properties"]["source_type"]["enum"] == ["text", "url"] + + +def test_all_five_handlers_emit_exact_hermes_http_operations() -> None: + client = StrictHostedClient(HOSTED_ORIGIN, "synthetic-placeholder") + requests: list[dict[str, Any]] = [] + + def request(method: str, path: str, **kwargs: Any) -> dict[str, Any]: + requests.append({"method": method, "path": path, **kwargs}) + return {"results": []} if path.endswith("/search") else {"accepted": True} + + client.request = request # type: ignore[method-assign] + selected = tools(Deliverer(True), client) + selected["substrate_search"].handler({"query": "needle", "limit": 7}) + selected["substrate_read"].handler({"path": "notes/page.md"}) + selected["substrate_query"].handler({"question": "why", "save_as_synthesis": True}) + selected["substrate_ingest"].handler( + {"content": "https://example.test/source", "source_type": "url"} + ) + selected["substrate_job_status"].handler({"job_id": "job-1"}) + + assert requests == [ + { + "method": "POST", + "path": "/api/v1/hermes/wiki/search", + "body": {"q": "needle", "limit": 7}, + }, + { + "method": "POST", + "path": "/api/v1/hermes/wiki/read", + "body": {"path": "notes/page.md"}, + }, + { + "method": "POST", + "path": "/api/v1/hermes/wiki/query", + "body": {"question": "why", "save_as_synthesis": True}, + }, + { + "method": "POST", + "path": "/api/v1/hermes/wiki/ingest", + "body": { + "content": "https://example.test/source", + "title": None, + "source_type": "url", + }, + }, + { + "method": "GET", + "path": "/api/v1/hermes/wiki/job-status", + "query": {"job_id": "job-1"}, + }, + ] diff --git a/tests/test_hosted_onboarding.py b/tests/test_hosted_onboarding.py new file mode 100644 index 0000000..7b254b1 --- /dev/null +++ b/tests/test_hosted_onboarding.py @@ -0,0 +1,280 @@ +"""Hermes-authoritative fixed-origin onboarding and custody tests.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from email.message import Message +from pathlib import Path +from urllib.parse import parse_qs + +import pytest + +from claude_code_memory import onboarding +from claude_code_memory.credentials import CredentialStore, PrivateFileStore +from claude_code_memory.local_security import secure_windows_tree + + +class Response: + def __init__(self, status: int, value: dict[str, object]) -> None: + self.status = status + self.headers = Message() + self.headers["Content-Type"] = "application/json" + self._raw = json.dumps(value).encode() + + def read(self, limit: int) -> bytes: + return self._raw[:limit] + + def close(self) -> None: + pass + + +class Opener: + def __init__(self, responses: list[Response]) -> None: + self.responses = responses + self.requests: list[tuple[object, float]] = [] + + def open(self, request: object, *, timeout: float) -> Response: + self.requests.append((request, timeout)) + return self.responses.pop(0) + + +class MemoryStore(CredentialStore): + backend = "test-memory" + + def __init__(self) -> None: + self.values: dict[str, str] = {} + + def get(self, slot: str = "access-token") -> str: + return self.values.get(slot, "") + + def put(self, value: str, slot: str = "access-token") -> None: + self.values[slot] = value + + def delete(self, slot: str = "access-token") -> None: + self.values.pop(slot, None) + + +def _form(request: object) -> dict[str, list[str]]: + return parse_qs(request.data.decode("ascii"), keep_blank_values=True) # type: ignore[attr-defined] + + +def test_device_and_token_requests_have_exact_identity_scope_origin_and_timeout() -> None: + opener = Opener( + [ + Response( + 200, + { + "device_code": "test-device-placeholder", + "user_code": "TEST-CODE", + "verification_uri": f"{onboarding.HOSTED_ORIGIN}/oauth/device", + "verification_uri_complete": f"{onboarding.HOSTED_ORIGIN}/oauth/device?user_code=TEST-CODE", + "expires_in": 600, + "interval": 5, + }, + ), + Response( + 200, + { + "access_token": "test-access-placeholder", + "token_type": "Bearer", + "scope": "capture retrieve", + }, + ), + ] + ) + api = onboarding.HostedOAuthClient(opener=opener) + grant = api.begin() + token = api.poll(str(grant["device_code"])) + begin, begin_timeout = opener.requests[0] + poll, poll_timeout = opener.requests[1] + assert begin.full_url == onboarding.HOSTED_ORIGIN + "/oauth/device_authorization" + assert _form(begin) == {"client_id": ["substrate-claude-code"], "scope": ["capture retrieve"]} + assert _form(poll) == { + "client_id": ["substrate-claude-code"], + "device_code": ["test-device-placeholder"], + "grant_type": [onboarding.DEVICE_GRANT], + } + assert begin_timeout == poll_timeout == 60 + assert grant["verification_uri_complete"].endswith("user_code=TEST-CODE") + assert set(token) == {"status", "access_token"} + + +def test_complete_url_must_carry_code_and_stay_on_fixed_origin() -> None: + for complete in ( + "https://foreign.example.test/oauth/device?user_code=TEST-CODE", + f"{onboarding.HOSTED_ORIGIN}/oauth/device?user_code=WRONG", + ): + opener = Opener( + [ + Response( + 200, + { + "device_code": "test-device-placeholder", + "user_code": "TEST-CODE", + "verification_uri": f"{onboarding.HOSTED_ORIGIN}/oauth/device", + "verification_uri_complete": complete, + "expires_in": 600, + }, + ) + ] + ) + with pytest.raises(onboarding.OnboardingError, match="invalid_response"): + onboarding.HostedOAuthClient(opener=opener).begin() + + +class FakeAPI: + def begin(self) -> dict[str, object]: + return { + "device_code": "test-device-placeholder", + "user_code": "TEST-CODE", + "verification_uri": f"{onboarding.HOSTED_ORIGIN}/oauth/device", + "verification_uri_complete": f"{onboarding.HOSTED_ORIGIN}/oauth/device?user_code=TEST-CODE", + "expires_in": 600, + "interval": 1, + } + + def poll(self, device_code: str) -> dict[str, str]: + assert device_code == "test-device-placeholder" + return {"status": "approved", "access_token": "test-access-placeholder"} + + +def test_manager_keeps_secrets_out_of_state_and_decline_reaches_ready(tmp_path: Path) -> None: + store = MemoryStore() + checked: list[bool] = [] + manager = onboarding.OnboardingManager( + tmp_path, + api=FakeAPI(), # type: ignore[arg-type] + store=store, + capability_check=lambda token: checked.append(bool(token)), + browser_open=lambda _url: True, + ) + assert manager.begin()["phase"] == "authorization_pending" + assert "test-device-placeholder" not in manager.path.read_text() + approved = manager.poll() + assert approved["phase"] == "awaiting_history_consent" + assert approved["authenticated"] is True + ready = manager.consent_history(False) + assert ready["phase"] == "ready" + assert ready["ready"] is True + assert ready["history_decision"] == "declined" + assert ready["history_import"] == "not_supported" + persisted = manager.path.read_text() + assert "test-device-placeholder" not in persisted + assert "test-access-placeholder" not in persisted + assert checked == [True] + + +def test_approved_history_preference_starts_no_discovery(tmp_path: Path) -> None: + store = MemoryStore() + store.put("test-access-placeholder") + manager = onboarding.OnboardingManager(tmp_path, store=store) + manager.begin(open_browser=False) + result = manager.consent_history(True) + assert result["history_decision"] == "approved" + assert result["history_import"] == "not_supported" + assert not (tmp_path / "history").exists() + + +@pytest.mark.skipif(os.name != "nt", reason="real Windows DACL contract") +def test_windows_private_tree_replaces_and_verifies_real_dacls(tmp_path: Path) -> None: + root = tmp_path / "private" + nested = root / "nested" + nested.mkdir(parents=True) + leaf = nested / "token" + leaf.write_text("synthetic", encoding="utf-8") + shell = shutil.which("powershell.exe") or shutil.which("pwsh.exe") or shutil.which("pwsh") + assert shell is not None + icacls = Path(os.environ["SystemRoot"]) / "System32" / "icacls.exe" + subprocess.run( + (icacls, root, "/grant", "*S-1-1-0:(OI)(CI)R", "/Q"), + timeout=30, + check=True, + ) + + secure_windows_tree(root) + environment = os.environ.copy() + environment["SUBSTRATE_ACL_ROOT"] = str(root) + verifier = r""" +$ErrorActionPreference = 'Stop' +$current = [System.Security.Principal.WindowsIdentity]::GetCurrent().User +$fullControl = [System.Security.AccessControl.FileSystemRights]::FullControl +$allow = [System.Security.AccessControl.AccessControlType]::Allow +$none = [System.Security.AccessControl.PropagationFlags]::None +$paths = @( + [System.IO.Path]::GetFullPath($env:SUBSTRATE_ACL_ROOT), + [System.IO.Path]::Combine($env:SUBSTRATE_ACL_ROOT, 'nested'), + [System.IO.Path]::Combine($env:SUBSTRATE_ACL_ROOT, 'nested', 'token')) +for ($index = 0; $index -lt $paths.Count; $index++) { + $acl = Get-Acl -LiteralPath $paths[$index] + $owner = $acl.GetOwner([System.Security.Principal.SecurityIdentifier]) + $rules = @($acl.GetAccessRules( + $true, $false, [System.Security.Principal.SecurityIdentifier])) + $inheritance = [System.Security.AccessControl.InheritanceFlags]::None + if ($index -lt 2) { + $inheritance = [System.Security.AccessControl.InheritanceFlags]'ContainerInherit, ObjectInherit' + } + if (-not $acl.AreAccessRulesProtected -or + $owner.Value -ne $current.Value -or + $rules.Count -ne 1 -or + $rules[0].IdentityReference.Value -ne $current.Value -or + $rules[0].AccessControlType -ne $allow -or + ($rules[0].FileSystemRights -band $fullControl) -ne $fullControl -or + $rules[0].InheritanceFlags -ne $inheritance -or + $rules[0].PropagationFlags -ne $none) { + throw 'independent ACL verification failed' + } +} +""" + result = subprocess.run( + (shell, "-NoProfile", "-NonInteractive", "-Command", verifier), + capture_output=True, + text=True, + env=environment, + timeout=30, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_private_fallback_is_owner_only_and_symlink_safe(tmp_path: Path) -> None: + store = PrivateFileStore(tmp_path) + store.put("test-access-placeholder") + path = store.root / "access-token" + if os.name == "posix": + assert path.stat().st_mode & 0o077 == 0 + store.delete() + outside = tmp_path / "outside" + outside.write_text("synthetic") + try: + path.symlink_to(outside) + except OSError: + pytest.skip("symlink creation is unavailable on this host") + with pytest.raises(OSError, match="symlink"): + store.put("test-access-placeholder") + + +class UndeletableStore(CredentialStore): + backend = "native-test" + + def get(self, slot: str = "access-token") -> str: + return "rejected-token" if slot == "access-token" else "" + + def put(self, value: str, slot: str = "access-token") -> None: + raise AssertionError("not used") + + def delete(self, slot: str = "access-token") -> None: + raise OSError("vault refused deletion") + + +def test_failed_vault_deletion_remains_repair_required_and_unauthenticated(tmp_path: Path) -> None: + manager = onboarding.OnboardingManager(tmp_path, store=UndeletableStore()) + manager._save({**onboarding._empty_state(), "phase": "ready"}) + status = manager.require_repair("http_401") + assert status["phase"] == "repair_required" + assert status["authenticated"] is False + assert status["ready"] is False + assert status["error_class"] == "credential_delete_failed" + assert manager.begin(open_browser=False)["phase"] == "repair_required" diff --git a/tests/test_manifests.py b/tests/test_manifests.py index 546ad95..e7a38a3 100644 --- a/tests/test_manifests.py +++ b/tests/test_manifests.py @@ -31,7 +31,7 @@ def test_plugin_and_marketplace_self_listing_are_consistent() -> None: plugin = _load(PLUGIN) marketplace = _load(MARKETPLACE) assert plugin["name"] == "claude-code-substrate-memory" - assert plugin["version"] == "0.1.0" + assert plugin["version"] == "0.2.0" assert plugin["author"]["name"] == "Sightline Technologies Inc" listing = marketplace["plugins"][0] assert listing["name"] == plugin["name"] @@ -42,9 +42,10 @@ def test_plugin_and_marketplace_self_listing_are_consistent() -> None: def test_mcp_manifest_references_the_server_module_and_source_root() -> None: server = _load(MCP)["mcpServers"]["substrate"] assert server["type"] == "stdio" - assert server["command"] == "python" - assert server["args"] == ["-m", "claude_code_memory.server"] - assert server["env"]["PYTHONPATH"] == "${CLAUDE_PLUGIN_ROOT}/src" + assert server["command"] == "node" + assert server["args"] == ["${CLAUDE_PLUGIN_ROOT}/scripts/plugin_runtime.cjs", "mcp"] + assert "env" not in server + assert (ROOT / "scripts" / "plugin_runtime.cjs").is_file() assert (ROOT / "src" / "claude_code_memory" / "server.py").is_file() @@ -55,6 +56,7 @@ def test_hooks_reference_existing_module_and_all_events() -> None: "PreCompact": "pre-compact", "SessionEnd": "session-end", "SessionStart": "session-start", + "UserPromptSubmit": "user-prompt-submit", } assert set(hooks) == set(expected) assert (ROOT / "src" / "claude_code_memory" / "hook.py").is_file() @@ -64,8 +66,9 @@ def test_hooks_reference_existing_module_and_all_events() -> None: assert entry["matcher"] == "" assert command["type"] == "command" assert command["timeout"] == 15 - assert "${CLAUDE_PLUGIN_ROOT}/src" in command["command"] - assert f"claude_code_memory.hook {cli_event}" in command["command"] + assert "${CLAUDE_PLUGIN_ROOT}/scripts/plugin_runtime.cjs" in command["command"] + assert f"hook {cli_event}" in command["command"] + assert "PYTHONPATH=" not in command["command"] def test_slash_commands_exist_and_have_description_frontmatter() -> None: diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 47110c5..c548ea7 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -26,7 +26,9 @@ def _tools() -> list[Tool]: schema = {"type": "object", "properties": {}, "additionalProperties": False} return [ Tool("echo", "Echo the arguments back.", schema, lambda args: {"got": args}), - Tool("boom", "Always raises.", schema, lambda args: (_ for _ in ()).throw(RuntimeError("x"))), + Tool( + "boom", "Always raises.", schema, lambda args: (_ for _ in ()).throw(RuntimeError("x")) + ), Tool("bad", "Returns an error result.", schema, lambda args: {"error": "nope"}), ] @@ -46,7 +48,14 @@ def _run(lines: list[dict]) -> list[dict]: def test_initialize_echoes_each_supported_protocol_version() -> None: for version in PROTOCOL_VERSIONS: responses = _run( - [{"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": version}}] + [ + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": {"protocolVersion": version}, + } + ] ) assert responses[0]["result"]["protocolVersion"] == version @@ -153,4 +162,20 @@ def test_embedded_newlines_never_break_framing() -> None: ] ) assert len(responses) == 1 - assert "line one\nline two" in json.loads(responses[0]["result"]["content"][0]["text"])["got"]["text"] + assert ( + "line one\nline two" + in json.loads(responses[0]["result"]["content"][0]["text"])["got"]["text"] + ) + + +def test_non_json_numeric_constants_are_rejected_and_never_emitted() -> None: + stdout = io.StringIO() + raw = '{"jsonrpc":"2.0","id":NaN,"method":"ping"}\n' + assert serve(Server("test", "0.2.0", _tools()), stdin=io.StringIO(raw), stdout=stdout) == 0 + line = stdout.getvalue().strip() + assert "NaN" not in line and "Infinity" not in line + response = json.loads( + line, parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value)) + ) + assert response["error"]["code"] == PARSE_ERROR + assert json.loads(bounded_json({"value": float("nan")})) == {"error": "invalid_result"} diff --git a/tests/test_recall_contract.py b/tests/test_recall_contract.py new file mode 100644 index 0000000..d38c0e4 --- /dev/null +++ b/tests/test_recall_contract.py @@ -0,0 +1,80 @@ +"""Canonical-only recall and session-relevant query tests.""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +import pytest + +from claude_code_memory import recall + + +class Client: + timeout = 6.0 + + def __init__(self, results: list[dict[str, Any]]) -> None: + self.results = results + self.queries: list[tuple[str, dict[str, Any]]] = [] + + def memory_search(self, query: str, *, limit: int, scope: dict[str, Any]) -> dict[str, Any]: + self.queries.append((query, scope)) + return {"results": self.results[:limit]} + + +def runtime_for(client: Client) -> tuple[Client, None, None, None]: + return client, None, None, None + + +def canonical(**changes: Any) -> dict[str, Any]: + item = { + "title": "Synthetic project", + "page_type": "entity", + "entity_id": "project-123", + "entity_type": "project", + "canonical_path": "entities/project/synthetic--1234abcd.md", + "memory_card": "Use the reviewed synthetic decision.", + "quality_version": 2, + } + item.update(changes) + return item + + +def test_only_canonical_v2_memory_card_is_injected(monkeypatch: pytest.MonkeyPatch) -> None: + client = Client( + [ + canonical(), + canonical(entity_id="bad-summary", memory_card="", summary="must not appear"), + canonical(entity_id="bad-path", canonical_path="projects/not-canonical.md"), + canonical(entity_id="bad-type", entity_type="unknown"), + canonical(entity_id="bad-quality", quality_version=1), + ] + ) + monkeypatch.setattr(recall, "runtime", lambda: runtime_for(client)) + block = recall._recall_once( + 10, {"prompt": "synthetic current user question"}, time.monotonic() + 5 + ) + assert "Use the reviewed synthetic decision." in block + assert "must not appear" not in block + assert "projects/not-canonical.md" not in block + assert client.queries[0][0] == "synthetic current user question" + scope = client.queries[0][1] + assert scope["user_id"] == scope["agent_id"] == scope["agent_identity"] + assert "subject_id" not in scope + + +def test_missing_prompt_disables_automatic_recall( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + client = Client([]) + monkeypatch.setattr(recall, "runtime", lambda: runtime_for(client)) + assert ( + recall._recall_once( + 5, + {"cwd": str(tmp_path / "synthetic-project"), "session_id": "session-123"}, + time.monotonic() + 5, + ) + == "" + ) + assert client.queries == [] diff --git a/tests/test_remediation_hostile.py b/tests/test_remediation_hostile.py new file mode 100644 index 0000000..0f5d572 --- /dev/null +++ b/tests/test_remediation_hostile.py @@ -0,0 +1,114 @@ +"""Hostile regressions for the blocked Claude/Hermes remediation.""" + +from __future__ import annotations + +import io +import json +import os +import time +from pathlib import Path + +import pytest + +from claude_code_memory import hook +from claude_code_memory.profile import state_home +from claude_code_memory.transcript import read_message_window +from substrate_capture import DurableSpool, config + + +def _line(content: str) -> str: + return json.dumps({"type": "user", "message": {"role": "user", "content": content}}) + "\n" + + +def test_verified_anchor_accepts_first_suffix_and_detects_same_inode_regrow(tmp_path: Path) -> None: + source = tmp_path / "transcript.jsonl" + first_raw = _line("A" * 32) + second_raw = _line("safe suffix") + source.write_text(first_raw + second_raw) + first = read_message_window(source, limit=1) + second = read_message_window( + source, + cursor=first.next_cursor, + expected_source_id=first.source_id, + expected_anchor=first.anchor, + limit=1, + ) + assert second.reset is False + assert second.messages[0]["content"] == "safe suffix" + assert second.messages[0]["redaction_codes"] == [] + + inode = source.stat().st_ino + replacement = _line("B" * 32) + assert len(replacement.encode()) == len(first_raw.encode()) + with source.open("r+b") as stream: + stream.truncate(0) + stream.write((replacement + second_raw).encode()) + stream.flush() + os.fsync(stream.fileno()) + assert source.stat().st_ino == inode + reset = read_message_window( + source, + cursor=first.next_cursor, + expected_source_id=first.source_id, + expected_anchor=first.anchor, + limit=1, + ) + assert reset.reset is True + assert reset.messages[0]["content"] == "B" * 32 + + +def test_spool_instances_cannot_overwrite_on_clock_collision( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import substrate_capture.spool as spool_module + + monkeypatch.setattr(spool_module.time, "time_ns", lambda: 7) + first = DurableSpool(tmp_path / "spool") + second = DurableSpool(tmp_path / "spool") + a = first.append({"event_id": "a", "kind": "turn"}) + b = second.append({"event_id": "b", "kind": "turn"}) + assert a != b + assert len(first) == 2 + assert {first.load(path)["event_id"] for path in (a, b)} == {"a", "b"} + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX symlink semantics") +def test_state_root_rejects_existing_symlink_ancestor( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + outside = tmp_path / "outside" + outside.mkdir() + redirected = tmp_path / "redirected" + redirected.symlink_to(outside, target_is_directory=True) + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(redirected / "nested")) + with pytest.raises(OSError, match="symlink"): + config.state_home("claude_code_memory") + assert list(outside.iterdir()) == [] + + +def test_large_session_end_hook_has_margin_and_leaves_durable_continuation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SUBSTRATE_STATE_HOME", str(tmp_path / "state")) + source = tmp_path / "large.jsonl" + payload = "x" * 100_000 + with source.open("w", encoding="utf-8") as stream: + for _ in range(500): + stream.write(_line(payload)) + started = time.monotonic() + assert ( + hook.run( + "session-end", + stdin=io.StringIO(json.dumps({"session_id": "large", "transcript_path": str(source)})), + stdout=io.StringIO(), + stderr=io.StringIO(), + authorization_fn=lambda: True, + ) + == 0 + ) + assert time.monotonic() - started < 5 + jobs = list((state_home() / "hook-jobs").glob("*.json")) + assert len(jobs) == 1 + state = json.loads(next((state_home() / "checkpoints").glob("*.json")).read_text()) + assert state["cursor"] < source.stat().st_size + assert state["session_end"] is False diff --git a/tests/test_review185_regressions.py b/tests/test_review185_regressions.py index 9ce7595..808472a 100644 --- a/tests/test_review185_regressions.py +++ b/tests/test_review185_regressions.py @@ -54,17 +54,18 @@ def test_secret_split_across_top_level_tool_result_blocks_must_not_survive(tmp_p def test_secret_split_across_inner_tool_result_text_blocks_must_not_survive(tmp_path, monkeypatch): monkeypatch.setenv("ANTHROPIC_API_KEY", SYNTHETIC_CREDENTIAL) left, right = SYNTHETIC_CREDENTIAL[:13], SYNTHETIC_CREDENTIAL[13:] - blocks = [{ - "type": "tool_result", "tool_use_id": "orphan", - "content": [{"type": "text", "text": left}, {"type": "text", "text": right}], - }] + blocks = [ + { + "type": "tool_result", + "tool_use_id": "orphan", + "content": [{"type": "text", "text": left}, {"type": "text", "text": right}], + } + ] message = read_messages(write_record(tmp_path / "inner-split.jsonl", blocks))[0] assert message["content"] == "" -def test_secret_split_across_adjacent_records_in_same_stream_is_quarantined( - tmp_path, monkeypatch -): +def test_secret_split_across_adjacent_records_in_same_stream_is_quarantined(tmp_path, monkeypatch): monkeypatch.setenv("ANTHROPIC_API_KEY", SYNTHETIC_CREDENTIAL) left, right = SYNTHETIC_CREDENTIAL[:13], SYNTHETIC_CREDENTIAL[13:] records = [ @@ -75,9 +76,7 @@ def test_secret_split_across_adjacent_records_in_same_stream_is_quarantined( "isSidechain": True, "message": { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "left", "content": left} - ], + "content": [{"type": "tool_result", "tool_use_id": "left", "content": left}], }, }, { @@ -103,9 +102,7 @@ def test_secret_split_across_adjacent_records_in_same_stream_is_quarantined( "isSidechain": True, "message": { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "right", "content": right} - ], + "content": [{"type": "tool_result", "tool_use_id": "right", "content": right}], }, }, ] @@ -114,14 +111,10 @@ def test_secret_split_across_adjacent_records_in_same_stream_is_quarantined( messages = read_messages(path) side_a = [ - message - for message in messages - if message["session_ancestry"]["agent_id"] == "side-A" + message for message in messages if message["session_ancestry"]["agent_id"] == "side-A" ] side_b = [ - message - for message in messages - if message["session_ancestry"]["agent_id"] == "side-B" + message for message in messages if message["session_ancestry"]["agent_id"] == "side-B" ] assert len(side_a) == 2 and all(message["content"] == "" for message in side_a) assert side_b[0]["content"] == "safe" @@ -141,9 +134,7 @@ def test_zero_visible_content_records_preserve_same_lineage_overlap(tmp_path, mo "isSidechain": True, "message": { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "left", "content": left} - ], + "content": [{"type": "tool_result", "tool_use_id": "left", "content": left}], }, }, { @@ -167,14 +158,15 @@ def test_zero_visible_content_records_preserve_same_lineage_overlap(tmp_path, mo "isSidechain": True, "message": { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "right", "content": right} - ], + "content": [{"type": "tool_result", "tool_use_id": "right", "content": right}], }, }, ] path = tmp_path / "zero-visible-gap.jsonl" - path.write_text("".join(json.dumps(record, ensure_ascii=False) + "\n" for record in records)) + path.write_text( + "".join(json.dumps(record, ensure_ascii=False) + "\n" for record in records), + encoding="utf-8", + ) messages = read_messages(path) assert len(messages) == 3 @@ -189,9 +181,7 @@ def test_zero_visible_content_records_preserve_same_lineage_overlap(tmp_path, mo provider_records[0]["message"]["content"][0]["content"] = provider_left provider_records[-1]["message"]["content"][0]["content"] = provider_right provider_path = tmp_path / "media-only-gap.jsonl" - provider_path.write_text( - "".join(json.dumps(record) + "\n" for record in provider_records) - ) + provider_path.write_text("".join(json.dumps(record) + "\n" for record in provider_records)) provider_messages = read_messages(provider_path) assert len(provider_messages) == 2 assert all(message["content"] == "" for message in provider_messages) @@ -243,9 +233,7 @@ def test_provider_token_split_across_records_is_quarantined( def test_bearer_credential_split_across_records_is_quarantined(tmp_path): credential = "Bearer " + "Z" * 24 points = (0, 3, 8, len(credential)) - pieces = [ - credential[start:end] for start, end in zip(points, points[1:]) - ] + pieces = [credential[start:end] for start, end in zip(points, points[1:])] records = [ { "type": "user", @@ -276,7 +264,14 @@ def test_bearer_credential_split_across_records_is_quarantined(tmp_path): def test_tool_use_input_full_quarantine_passes(tmp_path, monkeypatch): monkeypatch.setenv("ANTHROPIC_API_KEY", SYNTHETIC_CREDENTIAL) - block = [{"type": "tool_use", "id": "call", "name": "Fetch", "input": {"x": "safe", "arg": SYNTHETIC_CREDENTIAL, "tail": "safe"}}] + block = [ + { + "type": "tool_use", + "id": "call", + "name": "Fetch", + "input": {"x": "safe", "arg": SYNTHETIC_CREDENTIAL, "tail": "safe"}, + } + ] msg = read_messages(write_record(tmp_path / "input.jsonl", block, record_type="assistant"))[0] assert msg["content"] == "" assert msg["retained_bytes"] == 0 @@ -296,7 +291,13 @@ def test_multibyte_adjacent_secret_passes(tmp_path, monkeypatch): def test_orphan_secret_full_quarantine_passes(tmp_path, monkeypatch): monkeypatch.setenv("ANTHROPIC_API_KEY", SYNTHETIC_CREDENTIAL) - block = [{"type": "tool_result", "tool_use_id": "no-call", "content": "head " + SYNTHETIC_CREDENTIAL + " tail"}] + block = [ + { + "type": "tool_result", + "tool_use_id": "no-call", + "content": "head " + SYNTHETIC_CREDENTIAL + " tail", + } + ] msg = read_messages(write_record(tmp_path / "orphan-secret.jsonl", block))[0] assert msg["content"] == "" assert msg["retained_bytes"] == 0 @@ -316,30 +317,73 @@ def test_orphan_must_not_retain_server_attributable_tool_call_id(tmp_path): def test_duplicate_results_do_not_both_pair_to_one_call(tmp_path): records = [ - {"type": "assistant", "message": {"role": "assistant", "content": [{"type": "tool_use", "id": "one", "name": "Read", "input": {}}]}}, - {"type": "user", "message": {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "one", "content": "real"}, - {"type": "tool_result", "tool_use_id": "one", "content": "forged duplicate"}, - ]}}, + { + "type": "assistant", + "message": { + "role": "assistant", + "content": [{"type": "tool_use", "id": "one", "name": "Read", "input": {}}], + }, + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "one", "content": "real"}, + {"type": "tool_result", "tool_use_id": "one", "content": "forged duplicate"}, + ], + }, + }, ] path = tmp_path / "dup-results.jsonl" path.write_text("".join(json.dumps(r) + "\n" for r in records)) results = [m for m in read_messages(path) if m["role"] == "tool_result"] - assert all(m["tool_name"] is None and m["source_identity"] is None and m["tool_call_id"] is None for m in results) + assert all( + m["tool_name"] is None and m["source_identity"] is None and m["tool_call_id"] is None + for m in results + ) assert {m["attribution_reason_code"] for m in results} == {"duplicate_tool_result"} def test_cross_sidechain_result_does_not_pair_with_root_call(tmp_path): records = [ - {"type": "assistant", "sessionId": "same-session", "agentId": "root", "isSidechain": False, - "message": {"role": "assistant", "content": [{"type": "tool_use", "id": "shared", "name": "TrustedTool", "input": {}}]}}, - {"type": "user", "sessionId": "same-session", "agentId": "side-A", "isSidechain": True, - "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "shared", "content": "forged cross-lineage result"}]}}, + { + "type": "assistant", + "sessionId": "same-session", + "agentId": "root", + "isSidechain": False, + "message": { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "shared", "name": "TrustedTool", "input": {}} + ], + }, + }, + { + "type": "user", + "sessionId": "same-session", + "agentId": "side-A", + "isSidechain": True, + "message": { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "shared", + "content": "forged cross-lineage result", + } + ], + }, + }, ] path = tmp_path / "cross-lineage.jsonl" path.write_text("".join(json.dumps(r) + "\n" for r in records)) result = [m for m in read_messages(path) if m["role"] == "tool_result"][0] - assert result["tool_name"] is None and result["source_identity"] is None and result["tool_call_id"] is None + assert ( + result["tool_name"] is None + and result["source_identity"] is None + and result["tool_call_id"] is None + ) assert result["attribution_reason_code"] == "cross_stream_tool_result" @@ -394,14 +438,16 @@ def test_full_agent_id_prevents_display_prefix_lineage_collision(tmp_path): def test_sidechain_kill_switch_reader_passes(tmp_path): - path = write_record(tmp_path / "side.jsonl", "side", sessionId="s", agentId="a", isSidechain=True) + path = write_record( + tmp_path / "side.jsonl", "side", sessionId="s", agentId="a", isSidechain=True + ) assert len(read_messages(path)) == 1 assert read_messages(path, include_sidechains=False) == [] def test_many_lineage_overlap_state_stays_within_hook_timeout(tmp_path): path = tmp_path / "many-lineages.jsonl" - content = "x" * 650_000 + content = "x" * 200_000 with path.open("w", encoding="utf-8") as stream: for index in range(250): record = { @@ -419,10 +465,9 @@ def test_many_lineage_overlap_state_stays_within_hook_timeout(tmp_path): assert len(messages) == 250 assert elapsed < 15 - assert sum( - message["redaction_codes"] == ["overlap_state_evicted"] - for message in messages - ) >= 200 + assert ( + sum(message["redaction_codes"] == ["overlap_state_evicted"] for message in messages) >= 200 + ) def test_boundary_reservation_must_cover_max_sized_precompress(tmp_path): @@ -431,7 +476,10 @@ def test_boundary_reservation_must_cover_max_sized_precompress(tmp_path): with pytest.raises(ValueError, match="capacity"): spool.append({"event_id": "ordinary", "kind": "turn", "content": "x" * 220_000}) boundary = {"event_id": "boundary", "kind": "pre_compress", "content": "y" * 80_000} - assert len(json.dumps(boundary, separators=(",", ":"), sort_keys=True).encode()) < MAX_CAPTURE_BYTES + assert ( + len(json.dumps(boundary, separators=(",", ":"), sort_keys=True).encode()) + < MAX_CAPTURE_BYTES + ) spool.append(boundary) @@ -447,26 +495,38 @@ def test_loss_counters_survive_normal_restart_passes(tmp_path): def test_post_redaction_capture_cap_passes(): - builder = CaptureEventBuilder({"platform": "cli"}, provider_id="claude_code_memory", secrets=(SYNTHETIC_CREDENTIAL,)) - events = builder.message_events("turn", "s", [{"role": "user", "content": "x" * 300_000 + SYNTHETIC_CREDENTIAL}]) + builder = CaptureEventBuilder( + {"platform": "cli"}, provider_id="claude_code_memory", secrets=(SYNTHETIC_CREDENTIAL,) + ) + events = builder.message_events( + "turn", "s", [{"role": "user", "content": "x" * 300_000 + SYNTHETIC_CREDENTIAL}] + ) assert events assert all(len(canonical_bytes(event)) <= MAX_CAPTURE_BYTES for event in events) assert SYNTHETIC_CREDENTIAL not in json.dumps(events) -def test_long_configured_secret_in_new_tool_metadata_is_not_sliced_before_scan(tmp_path, monkeypatch): +def test_long_configured_secret_in_new_tool_metadata_is_not_sliced_before_scan( + tmp_path, monkeypatch +): long_secret = "Q" * 600 monkeypatch.setenv("ANTHROPIC_API_KEY", long_secret) block = [{"type": "tool_use", "id": long_secret, "name": long_secret, "input": {}}] - msg = read_messages(write_record(tmp_path / "metadata.jsonl", block, record_type="assistant"))[0] + msg = read_messages(write_record(tmp_path / "metadata.jsonl", block, record_type="assistant"))[ + 0 + ] serialized = json.dumps(msg) assert "Q" * 32 not in serialized assert msg["content"] == "" def test_builder_does_not_emit_unscanned_session_secret(): - builder = CaptureEventBuilder({"platform": "cli"}, provider_id="claude_code_memory", secrets=(SYNTHETIC_CREDENTIAL,)) - event = builder.payload_event("session_end", SYNTHETIC_CREDENTIAL, {"summary": {"message_count": 0}}) + builder = CaptureEventBuilder( + {"platform": "cli"}, provider_id="claude_code_memory", secrets=(SYNTHETIC_CREDENTIAL,) + ) + event = builder.payload_event( + "session_end", SYNTHETIC_CREDENTIAL, {"summary": {"message_count": 0}} + ) assert SYNTHETIC_CREDENTIAL not in json.dumps(event) diff --git a/tests/test_state.py b/tests/test_state.py index 2806ae2..d97f795 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -97,6 +97,24 @@ def test_spool_refuses_newest_and_reserves_boundary_capacity(tmp_path: Path) -> assert spool.statistics()["evicted"] == 0 +def test_spool_duplicate_lookup_reads_only_matching_private_marker( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + spool = DurableSpool(tmp_path / "s") + matching = spool.append({"event_id": "matching", "kind": "turn"}) + spool.append({"event_id": "other", "kind": "turn"}) + opened: list[Path] = [] + original = spool._open_readonly # noqa: SLF001 + + def tracked(path: Path) -> int: + opened.append(path) + return original(path) + + monkeypatch.setattr(spool, "_open_readonly", tracked) + assert spool.append({"event_id": "matching", "kind": "turn"}) == matching + assert opened == [matching] + + def test_spool_duplicate_and_quarantine_counters_persist(tmp_path: Path) -> None: root = tmp_path / "s" spool = DurableSpool(root) diff --git a/tests/test_strict_contract.py b/tests/test_strict_contract.py new file mode 100644 index 0000000..1247a71 --- /dev/null +++ b/tests/test_strict_contract.py @@ -0,0 +1,72 @@ +"""Strict provider, stream, entity, quality, and floor gates.""" + +from __future__ import annotations + +from copy import deepcopy + +import pytest + +from claude_code_memory.strict_client import validate_capabilities +from substrate_capture import SubstrateAPIError + + +def capabilities() -> dict[str, object]: + return { + "provider": "substrate_wiki", + "providers": ["substrate_wiki", "claude_code_memory"], + "capture_schema_versions": [2], + "max_event_bytes": 262_144, + "history_replay": { + "protocol": "stream-v2", + "min_plugin_version": "2.0.3", + "content_free_completion": True, + "incremental_windows": True, + "status_version": 2, + }, + "entity_memory": { + "protocol": "entity-wiki-v1", + "min_plugin_version": "2.0.3", + "search_endpoint": "/api/v1/hermes/memory/search", + "canonical_wiki_pages": True, + "entity_page_type": "entity", + }, + "entity_quality": { + "protocol": "entity-quality-v2", + "min_plugin_version": "2.0.3", + "memory_card": True, + "quality_version": 2, + "canonical_redirects": True, + }, + } + + +def test_complete_host_specific_contract_passes_even_with_legacy_scalar() -> None: + validate_capabilities(capabilities()) + + +@pytest.mark.parametrize( + ("path", "value"), + [ + (("providers",), ["substrate_wiki"]), + (("capture_schema_versions",), [1]), + (("max_event_bytes",), 262_143), + (("history_replay", "protocol"), "stream-v1"), + (("history_replay", "status_version"), 1), + (("history_replay", "min_plugin_version"), "not-semver"), + (("entity_memory", "canonical_wiki_pages"), False), + (("entity_memory", "entity_page_type"), "page"), + (("entity_quality", "protocol"), "entity-quality-v1"), + (("entity_quality", "memory_card"), False), + (("entity_quality", "canonical_redirects"), False), + ], +) +def test_every_partial_or_malformed_contract_fails_closed( + path: tuple[str, ...], value: object +) -> None: + candidate = deepcopy(capabilities()) + target = candidate + for key in path[:-1]: + target = target[key] # type: ignore[assignment,index] + target[path[-1]] = value # type: ignore[index] + with pytest.raises(SubstrateAPIError, match="server_upgrade_required"): + validate_capabilities(candidate)