From 3362bc8466d6b3ea5529eaeea84542cee9ed96bf Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:35:38 +0100 Subject: [PATCH 01/12] chore(sync): sync marfa v0.0.14 --- openapi.json | 155 +++++++++++++++++++++++++++--- src/generated/cli-download.md | 2 +- src/generated/cli/exit-codes.md | 4 +- src/generated/contract-index.json | 77 ++++++++++++++- src/generated/error-codes.md | 3 +- src/generated/marfa-release.json | 2 +- src/generated/settings.md | 5 +- 7 files changed, 221 insertions(+), 27 deletions(-) diff --git a/openapi.json b/openapi.json index daccc967..b449402e 100644 --- a/openapi.json +++ b/openapi.json @@ -1102,6 +1102,10 @@ "type": "string", "description": "The build the instance runs." }, + "release": { + "type": "string", + "description": "The release the build was made as, such as `0.0.14`. Absent for a build that is not a release." + }, "instance_id": { "type": "string", "description": "Unique identifier for the instance. `GET /config` and an archive's manifest show the same value." @@ -1310,7 +1314,7 @@ ], "description": "An error response." }, - "EdgeTypeNotFoundOrItemNotFoundRefusal": { + "BlobNotFoundOrEdgeTypeNotFoundOrItemNotFoundRefusal": { "type": "object", "properties": { "error": { @@ -1319,6 +1323,7 @@ "code": { "type": "string", "enum": [ + "blob_not_found", "edge_type_not_found", "item_not_found" ], @@ -1813,7 +1818,7 @@ ], "description": "An error response." }, - "EdgeNotFoundOrEdgeTypeNotFoundOrItemNotFoundRefusal": { + "BlobNotFoundOrEdgeNotFoundOrEdgeTypeNotFoundOrItemNotFoundRefusal": { "type": "object", "properties": { "error": { @@ -1822,6 +1827,7 @@ "code": { "type": "string", "enum": [ + "blob_not_found", "edge_not_found", "edge_type_not_found", "item_not_found" @@ -2737,6 +2743,42 @@ ], "description": "An error response." }, + "EdgeTypeNotFoundOrItemNotFoundRefusal": { + "type": "object", + "properties": { + "error": { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "edge_type_not_found", + "item_not_found" + ], + "description": "A machine-readable code for the error. Use it in your logic." + }, + "message": { + "type": "string", + "description": "A description of the error for a person to read. It can change, so don't match on it." + }, + "details": { + "type": "object", + "additionalProperties": {}, + "description": "More about the error, such as the field it concerns. Each code defines its own details." + } + }, + "required": [ + "code", + "message" + ], + "description": "What went wrong." + } + }, + "required": [ + "error" + ], + "description": "An error response." + }, "IdReusedRefusal": { "type": "object", "properties": { @@ -2807,6 +2849,43 @@ ], "description": "An error response." }, + "EdgeNotFoundOrEdgeTypeNotFoundOrItemNotFoundRefusal": { + "type": "object", + "properties": { + "error": { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "edge_not_found", + "edge_type_not_found", + "item_not_found" + ], + "description": "A machine-readable code for the error. Use it in your logic." + }, + "message": { + "type": "string", + "description": "A description of the error for a person to read. It can change, so don't match on it." + }, + "details": { + "type": "object", + "additionalProperties": {}, + "description": "More about the error, such as the field it concerns. Each code defines its own details." + } + }, + "required": [ + "code", + "message" + ], + "description": "What went wrong." + } + }, + "required": [ + "error" + ], + "description": "An error response." + }, "EdgeVersionConflict": { "type": "object", "properties": { @@ -5237,7 +5316,7 @@ } ] }, - "description": "`[name, value]` pairs in the order and case they arrived." + "description": "`[name, value]` pairs in the order and case they arrived, without any header whose name begins with `Tailscale-`." }, "size": { "type": "integer", @@ -6494,6 +6573,42 @@ ], "description": "An error response." }, + "BlobStoreUnavailableOrWriteContentionRefusal": { + "type": "object", + "properties": { + "error": { + "type": "object", + "properties": { + "code": { + "type": "string", + "enum": [ + "blob_store_unavailable", + "write_contention" + ], + "description": "A machine-readable code for the error. Use it in your logic." + }, + "message": { + "type": "string", + "description": "A description of the error for a person to read. It can change, so don't match on it." + }, + "details": { + "type": "object", + "additionalProperties": {}, + "description": "More about the error, such as the field it concerns. Each code defines its own details." + } + }, + "required": [ + "code", + "message" + ], + "description": "What went wrong." + } + }, + "required": [ + "error" + ], + "description": "An error response." + }, "DriftedPlatformTypePage": { "type": "object", "properties": { @@ -8088,7 +8203,7 @@ "Instance" ], "summary": "Describe the instance", - "description": "Describes the instance: its display name, `instance_id`, the build it runs, the contract version it speaks and the features it serves. Needs no credential.", + "description": "Describes the instance: its display name, `instance_id`, the build it runs, the release it was made as when it is one, the contract version it speaks and the features it serves. Needs no credential.", "responses": { "200": { "description": "Returns the instance's description. A request whose `Accept` header prefers `text/html` gets an HTML page instead.", @@ -8408,11 +8523,11 @@ } }, "404": { - "description": "- `edge_type_not_found`: an edge names an edge type that doesn't exist.\n- `item_not_found`: an edge's target doesn't exist, or its type is one you can't read.", + "description": "- `edge_type_not_found`: an edge names an edge type that doesn't exist.\n- `item_not_found`: an edge's target doesn't exist, or its type is one you can't read.\n- `blob_not_found`: `blob_ref` names bytes you didn't upload and can't read.", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/EdgeTypeNotFoundOrItemNotFoundRefusal" + "$ref": "#/components/schemas/BlobNotFoundOrEdgeTypeNotFoundOrItemNotFoundRefusal" } } }, @@ -10225,11 +10340,11 @@ } }, "404": { - "description": "- `item_not_found`: no item has this ID, its type is one you can't read, or an edge target doesn't exist or has a type you can't read. For an item in the trash, `details.trashed` is `true` if you can read its type.\n- `edge_type_not_found`: an edge names an edge type that doesn't exist.\n- `edge_not_found`: a repeat under the `Idempotency-Key` would show an edge you can no longer read.", + "description": "- `item_not_found`: no item you can read has this ID or an edge target's ID. For a trashed item, `details.trashed` is `true` if you can read its type.\n- `edge_type_not_found`: an edge names an edge type that doesn't exist.\n- `edge_not_found`: a repeat under the `Idempotency-Key` would show an edge you can't read now.\n- `blob_not_found`: `blob_ref` names bytes you didn't upload and can't read.", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/EdgeNotFoundOrEdgeTypeNotFoundOrItemNotFoundRefusal" + "$ref": "#/components/schemas/BlobNotFoundOrEdgeNotFoundOrEdgeTypeNotFoundOrItemNotFoundRefusal" } } }, @@ -41532,6 +41647,13 @@ "type": "number", "description": "How many blobs the archive carried whose bytes match their hash, whether or not the instance already had them." }, + "blobs_missing": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The hashes of the blobs the archive names but doesn't carry with matching bytes, and that Marfa holds no copy of. Upload their bytes again with `POST /blobs` to restore them." + }, "types_registered": { "type": "number", "description": "How many types Marfa registered." @@ -41556,6 +41678,7 @@ "edges_skipped", "edges_skipped_reasons", "blobs_imported", + "blobs_missing", "types_registered", "types_skipped", "edge_types_registered", @@ -41794,11 +41917,11 @@ } }, "503": { - "description": "`write_contention`: the database was busy, and Marfa couldn't complete the request in time. Nothing changed. Try the request again.", + "description": "- `blob_store_unavailable`: a blob store failed, or didn't answer in time, while Marfa checked a blob's copy. Nothing was written. Try again later.\n- `write_contention`: the database was busy, and Marfa couldn't complete the request in time. Nothing changed. Try the request again.", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WriteContentionRefusal" + "$ref": "#/components/schemas/BlobStoreUnavailableOrWriteContentionRefusal" } } }, @@ -45861,10 +45984,10 @@ "ndjson", "archive" ], - "description": "`ndjson` for lines of JSON, or `archive` for a `.tar.gz` file that `POST /restore` reads. Defaults to `ndjson`." + "description": "`ndjson` for lines of JSON, or `archive` for a `.tar.gz` file that `POST /restore` reads. Defaults to `ndjson`. An archive's `manifest.json` lists in `missing_blobs` each blob no store could read, which the archive doesn't hold." }, "required": false, - "description": "`ndjson` for lines of JSON, or `archive` for a `.tar.gz` file that `POST /restore` reads. Defaults to `ndjson`.", + "description": "`ndjson` for lines of JSON, or `archive` for a `.tar.gz` file that `POST /restore` reads. Defaults to `ndjson`. An archive's `manifest.json` lists in `missing_blobs` each blob no store could read, which the archive doesn't hold.", "name": "format", "in": "query" } @@ -46057,11 +46180,11 @@ } }, "503": { - "description": "`write_contention`: the database was busy, and Marfa couldn't complete the request in time. Nothing changed. Try the request again.", + "description": "- `blob_store_unavailable`: a blob store failed, or didn't answer in time, while Marfa checked a blob's copy. Nothing was written. Try again later.\n- `write_contention`: the database was busy, and Marfa couldn't complete the request in time. Nothing changed. Try the request again.", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WriteContentionRefusal" + "$ref": "#/components/schemas/BlobStoreUnavailableOrWriteContentionRefusal" } } }, @@ -46932,7 +47055,7 @@ "url": { "type": "string", "minLength": 1, - "description": "The URL Marfa posts events to: `http` or `https`, with no user name or password. Unless the instance allows private addresses, it must reach a public address, which Marfa checks again at each delivery." + "description": "The URL Marfa posts events to: `http` or `https`, with no user name or password. Unless the instance allows private addresses, it must reach a public address: Marfa refuses a URL whose host resolves to a private one, and checks again at each delivery." }, "events": { "type": "array", @@ -47879,7 +48002,7 @@ "properties": { "url": { "type": "string", - "description": "The URL Marfa posts events to: `http` or `https`, with no user name or password. Unless the instance allows private addresses, it must reach a public address, which Marfa checks again at each delivery." + "description": "The URL Marfa posts events to: `http` or `https`, with no user name or password. Unless the instance allows private addresses, it must reach a public address: Marfa refuses a URL whose host resolves to a private one, and checks again at each delivery." }, "events": { "type": "array", diff --git a/src/generated/cli-download.md b/src/generated/cli-download.md index b7bcab0a..64615024 100644 --- a/src/generated/cli-download.md +++ b/src/generated/cli-download.md @@ -1,5 +1,5 @@ ```bash -curl -fsSL https://github.com/withmarfa/marfa/releases/download/v0.0.13/marfa-0.0.13-darwin-arm64.tar.gz | tar -xz +curl -fsSL https://github.com/withmarfa/marfa/releases/download/v0.0.14/marfa-0.0.14-darwin-arm64.tar.gz | tar -xz sudo mkdir -p /usr/local/bin sudo mv marfa /usr/local/bin/ marfa --version diff --git a/src/generated/cli/exit-codes.md b/src/generated/cli/exit-codes.md index 57fe1c37..16e12f48 100644 --- a/src/generated/cli/exit-codes.md +++ b/src/generated/cli/exit-codes.md @@ -1,7 +1,7 @@ | Code | Meaning | | --- | --- | | 0 | Done. | -| 1 | The request was refused, by the server, by the CLI before sending, for an answer on another contract, or for a docs page that does not exist; a retry does not change it. | +| 1 | The request was refused, by the server, by the CLI before sending, for an answer on another contract, or for a docs page that does not exist, or the archive of an export or a restore lacks blobs it names; a retry does not change it. | | 2 | The command line was wrong, or named no store or server. | | 3 | The environment failed (unreachable, timed out, a 5xx, a 429, an answer naming no contract, full local storage, a docs site that cannot be read); try again. | | 4 | The working copy or the queue refused under the device rules, or this system has no keychain. | @@ -18,4 +18,4 @@ With --json a refusal is one JSON object on stderr: {"error":{"code":...,"message":...,"server":{"status":...,"code":...,"details":...}|null,"retry_after_seconds":...},"exit":N} ``` -where error.code is one of: `not_found`, `unauthorized`, `forbidden`, `validation`, `unknown_type`, `rate_limited`, `server`, `io`, `network`, `unnamed_answer`, `decoding`, `store`, `storage_full`, `signed_out`, `no_keychain`, `redirect`, `no_server`, `no_cursor`, `hydration_incomplete`, `no_catalog`, `reading_handle`, `wrong_schema`, `copy_expired`, `stream_incomplete`, `wrong_server`, `bytes_absent`, `contract_mismatch`, `canceled`, `first_sync_waiting`, `invalid`, `not_held`, `watch`, `usage`, `no_store`, `no_credential`, `docs_page_not_found`, `docs_unreachable`, `conflict`, `too_large`. +where error.code is one of: `not_found`, `unauthorized`, `forbidden`, `validation`, `unknown_type`, `rate_limited`, `server`, `io`, `network`, `unnamed_answer`, `decoding`, `store`, `storage_full`, `signed_out`, `no_keychain`, `redirect`, `no_server`, `no_cursor`, `hydration_incomplete`, `no_catalog`, `reading_handle`, `wrong_schema`, `copy_expired`, `stream_incomplete`, `wrong_server`, `bytes_absent`, `contract_mismatch`, `canceled`, `first_sync_waiting`, `invalid`, `not_held`, `watch`, `usage`, `no_store`, `no_credential`, `blobs_missing`, `docs_page_not_found`, `docs_unreachable`, `conflict`, `too_large`. diff --git a/src/generated/contract-index.json b/src/generated/contract-index.json index 012a4d64..08d18775 100644 --- a/src/generated/contract-index.json +++ b/src/generated/contract-index.json @@ -103,6 +103,12 @@ "blobs/lend-extension-unread": "cd6ff89dbc433290", "blobs/lend-copy-edges": "18a61af2681aad1a", "blobs/proof-item": "03d630ebe45fc457", + "blobs/proof-file-refused": "88355f56b774cc49", + "blobs/proof-file-refused-bulk": "f4ce26d5a48b1b4f", + "blobs/proof-file-refused-nothing": "dafac5b9b6c7bdef", + "blobs/proof-file-refused-as-unknown": "826aadbe6804723a", + "blobs/proof-file-text": "b2363fff9d1a8cfd", + "blobs/proof-file-stale-base": "8871e44a4dc054fc", "blobs/proof-folder": "c360b75ec504b26b", "blobs/proof-edge": "01e5bdb850339de6", "blobs/proof-extension": "fdbb2e510dc89032", @@ -594,6 +600,16 @@ "device/command-logout-forgets": "dce28713c72fce3b", "device/command-logout-not-revoked": "90301ca3d6b5c78c", "device/command-keys-forget-offline": "a3486c423a33b49e", + "device/command-keys-keep-checks-current-key": "fd9248faae92165e", + "device/command-keys-keep-keeps-key-without-type": "941d9196539f2e64", + "device/command-keys-keep-refuses-spent-key": "7ad4f2971e6df763", + "device/command-keys-keep-refuses-access-token": "f518a129b2295e93", + "device/command-keys-keep-refuses-wrong-server": "a69f993e182fb19e", + "device/command-events-refuses-cursor": "6e5d097266898d69", + "device/command-events-prints-refusing-frame": "1302d9f2da90011c", + "device/command-events-refuses-incomplete": "1fe49ebbcc65a72d", + "device/command-stream-credential-ended": "9f81678abd044431", + "device/command-events-frame-name-decides": "fa77faf100edc5b3", "device/command-cleartext-warning": "43aca62ac85d31f4", "device/command-cleartext-sends": "c166525a76eb4748", "device/command-cleartext-sign-in": "d40497a38b8a29fb", @@ -1522,10 +1538,16 @@ "events/history-deleted": "b88c1d977b605c97", "events/redeliver-expired": "e0907d422388ce0d", "events/address-private": "4c48d6406e9d8d33", + "events/address-name-refused": "780c2ec208e04bcf", + "events/address-refusal-setting": "b216dd52e04f26b4", + "events/address-name-unresolved": "9f8f2895145cef00", + "events/address-ipv6-global": "95d6bb05db39f25f", "events/address-mapped": "8663584811c46af7", "events/address-update": "4973d64d9e85cb23", + "events/address-name-update": "caa36e545ee8fb3d", "events/address-name-unsent": "c9fa0e04b7f91999", "events/address-name-recorded": "a85779555ad9ca85", + "events/address-name-dead-letter": "b66c0823f427252f", "events/address-resolved-once": "da66be80979705dc", "events/webhook-starts-after-creation": "8da5a611586dc90d", "events/schedule-fails-visibly": "ff0fdafad940bb72", @@ -2228,6 +2250,7 @@ "folders/moved-file-not-trashed": "4029766b41d240d2", "folders/moved-file-not-trashed-item-preserved": "bc3d6766fde35095", "folders/moved-file-count": "b0bec49d07465ab1", + "folders/moved-carried-id-utf16": "aa25b2589c7fe12a", "folders/peer-owned-bytes-not-move": "b5a37ef19c93a44d", "folders/both-folders-copy": "18aa2ad3dde9f685", "folders/missing-file-trashed": "235209e5aa435b25", @@ -2235,6 +2258,7 @@ "folders/missing-file-said": "b3c5c5178e5a1e3f", "folders/uncertain-move-deferred": "26ddcaa6fb20dbd5", "folders/uncertain-move-deferred-unsure-reported": "a44b92875008474a", + "folders/uncertain-move-not-text": "986c81a1f5d0d8b1", "folders/moved-arrival-bytes": "291b20f2e114cf2b", "folders/moved-awaiting-peer": "e8ba48c290b6ec04", "folders/moved-awaiting-peer-no-replacement": "5bd0224352a549f5", @@ -2373,6 +2397,17 @@ "folders/push-server-unavailable": "4d33d62d855ac9f6", "folders/push-credential-stopped": "a9e6dddf823ddf6b", "folders/push-undelivered": "7cd44cf1de8709cd", + "folders/push-rate-limit-no-wait": "bcd4040bd8b24217", + "folders/push-rate-limit-no-wait-send-only": "4b8c65fe0ff1fc22", + "folders/watch-rate-limit-pause": "7782bb8afa454545", + "folders/push-exit-undelivered": "5b0d49beb15334cd", + "folders/push-exit-send-only": "79c9908a9c78bf5b", + "folders/push-exit-credential": "130dc76910e9463b", + "folders/push-exit-catchup-failed": "3abba2a1e4a9dfb7", + "folders/scan-waiting": "4013002f5af64a83", + "folders/scan-waiting-not-dead": "a1ca8deecac4eed1", + "folders/scan-waiting-said": "f29e717a19d33541", + "folders/not-a-folder-refused": "29ef7da86e1ca981", "folders/push-placement-retry": "eac4b4a8407d3aae", "folders/push-placement-verdict": "12977dc546ec8e87", "folders/push-placement-withheld": "698b42a16663352a", @@ -2426,7 +2461,12 @@ "folders/confirm-pull-removes": "f468356d14ab2a44", "folders/confirm-pull-removes-removed-count": "32477b40ddea1fce", "folders/restore-disk-files": "3d2479d0ff5671c9", - "folders/restore-disk-files-put-back-count": "c6f553469be1916b", + "folders/restore-disk-files-put-back-count": "9235238c326acd20", + "folders/restore-not-written-listed": "6fcdcdc9e50492aa", + "folders/restore-written-by-binding": "36521edf6a17096c", + "folders/restore-not-written-flag": "1cee3b3341aee6b9", + "folders/restore-not-written-kept": "bde767785bc4f034", + "folders/restore-not-written-retried": "32a729e32eb7af3c", "folders/restore-departed-binding": "7f0bf4c8c11b604e", "folders/restore-departed-binding-not-put-back": "702869c31bab07cd", "folders/restore-departed-binding-item-preserved": "e1ee98ea4bfd9ce4", @@ -2641,7 +2681,8 @@ "inbound-webhooks/receipt-pending": "78df8c69e9a22c70", "inbound-webhooks/receipt-method": "f9603d559cc6bb0f", "inbound-webhooks/receipt-query": "9bf69cdcdad0f54b", - "inbound-webhooks/receipt-headers": "61e02abe9b70bb96", + "inbound-webhooks/receipt-headers": "976a2027007b3366", + "inbound-webhooks/receipt-no-tailscale": "9e0250742eadc1d6", "inbound-webhooks/receipt-body": "e4a33d2fa7605f29", "inbound-webhooks/receipt-size-hash": "9583c8e3349a4abc", "inbound-webhooks/receipt-received-at": "63c1cb927f40acee", @@ -2840,6 +2881,8 @@ "instance/root-fields": "0aca6005f699f19c", "instance/root-name": "46717bc3472d9a07", "instance/root-version": "c9aa784e6a869141", + "instance/root-release": "2599151a2bc1cb3c", + "instance/root-release-absent": "c20c257359e4e40d", "instance/features-array": "4e9c2ebf1bf3693e", "instance/features-served": "3ca30b6709009d39", "instance/features-snake-case": "bf54a45769ebccd4", @@ -2847,6 +2890,9 @@ "instance/contract-document": "461d7c026e559987", "instance/archive-version": "0d2e49556b22707d", "instance/contract-header": "82b9f1794891bc71", + "instance/replica-under-prefix": "3fb23200932bf5cc", + "instance/replica-restores-own": "87c0601ce15e1834", + "instance/replica-other-prefix": "c78d451887e2baaf", "instance/image-starts": "eab743a9bb8544fa", "instance/image-keeps-writes": "08a6ef06544a8aea", "instance/image-keeps-blobs": "15ef592eec151807", @@ -2953,6 +2999,11 @@ "instance/test-clock-offset": "54d28a72bc621295", "instance/test-clock-forward": "c207432168d82d57", "instance/test-faults-cleared-at-start": "ebf3cdbd390b3e35", + "instance/telemetry-instance-id": "74c9710ddade37ea", + "instance/telemetry-exception-id": "49c3f87daa82ad6c", + "instance/telemetry-attributes": "60305bf4a7a14b60", + "instance/telemetry-service-name": "d2e0b09a92842b4c", + "instance/telemetry-error-environment": "d31ebd162b0f35b7", "instance/settings-edge-accepted": "91747bce5932bd24", "instance/settings-refused-exit": "98cf782ae12ae06b", "instance/settings-refused-no-port": "9b69d9bc16a82161", @@ -3073,6 +3124,7 @@ "items/idempotency-signed-in": "8170f23cf3f5d067", "items/idempotency-key-length": "14039e7aa8ef2d36", "items/idempotency-not-kept": "cf55a95128db13e4", + "items/idempotency-unproven-file-not-kept": "b5666c42e5773b46", "items/replay-reauthorized": "f0365f7ac123e702", "items/replay-write-lost": "58d4f05a8bda6921", "items/replay-read-lost": "6336e4353b7d2304", @@ -3768,6 +3820,7 @@ "queue-and-verdicts/command-queue-not-waiting": "57e98a5c68da56f7", "queue-and-verdicts/command-queue-unlanded": "12e96e0cd59fc47e", "queue-and-verdicts/command-queue-after": "9bd0e45f14eb0036", + "queue-and-verdicts/upload-sent-as-writer": "0abc8cf2dc412977", "queue-and-verdicts/report-sent-write": "7820623b1b0c7c7f", "queue-and-verdicts/report-edge-id": "43b7e666c4eda083", "queue-and-verdicts/report-unsent-verdicts": "2346206835baca0c", @@ -4258,9 +4311,19 @@ "search-and-filters/export-state-trashed": "dbf286ab487be108", "search-and-filters/archive-media-type": "c75b7a9414f11bf0", "search-and-filters/archive-same-selection": "f768633e7225ee4b", - "search-and-filters/archive-entries": "32d50ffffae0d4fd", + "search-and-filters/archive-entries": "0ffd5d85bca9ba04", "search-and-filters/archive-manifest": "820713d2f8e9846e", "search-and-filters/archive-manifest-blobs": "152b1f5c3eba3b30", + "search-and-filters/archive-missing-blobs": "92cb414c3fe3747d", + "search-and-filters/archive-missing-unrecorded": "c05a242665baff85", + "search-and-filters/archive-missing-only-blobs": "6aa910c25709e589", + "search-and-filters/archive-missing-not-carried": "7c469e1cace0c1d3", + "search-and-filters/archive-blob-any-store": "8249880d9342c630", + "search-and-filters/archive-bytes-verified": "f9d2974676fab21f", + "search-and-filters/archive-bytes-struck": "5b719e52eabec0c4", + "search-and-filters/archive-read-fault-kept": "f20fa1e704841b8a", + "search-and-filters/archive-blob-gone": "32b06f36de846746", + "search-and-filters/archive-store-unavailable": "aeccef6ab79c87f7", "search-and-filters/archive-only": "1f55601a751fc60b", "search-and-filters/archive-types": "339f0b45f661cff5", "search-and-filters/archive-history": "f999cd7825a69cd2", @@ -4279,6 +4342,12 @@ "search-and-filters/restore-operator": "d7f0f18b82227e9c", "search-and-filters/restore-roundtrip": "5fb9c51d9951f13f", "search-and-filters/restore-counts": "3146e824b7c324a0", + "search-and-filters/restore-blobs-missing": "6bf4ba1c5be6c593", + "search-and-filters/restore-store-unavailable": "56b419fb7393432b", + "search-and-filters/restore-store-unavailable-unwritten": "e3eceb6a505a779c", + "search-and-filters/cli-export-lacking": "c42c855be7a1fe2e", + "search-and-filters/cli-restore-lacking": "e263a4ac77220af7", + "search-and-filters/cli-export-cut": "44a507870f0529a3", "search-and-filters/restore-edge-skips": "2a1314cde93b14f2", "search-and-filters/restore-items-kept": "da3e8ce050fe34ec", "search-and-filters/restore-edges-kept": "84dce9e9ac2720ba", @@ -4293,7 +4362,7 @@ "search-and-filters/restore-not-archive": "7ffb014b92efcb68", "search-and-filters/restore-keeps-serving": "aaef6fe2b3e1b7f8", "search-and-filters/restore-version": "5812a317951116ed", - "search-and-filters/restore-manifest": "d11f4f84236d4d7d", + "search-and-filters/restore-manifest": "1e0a300a60d636d0", "search-and-filters/restore-state": "841ec4a3fc185d9a", "search-and-filters/restore-state-kept": "f09f2f34dd9b1c68", "search-and-filters/restore-source": "303ceb0c6853ef0b", diff --git a/src/generated/error-codes.md b/src/generated/error-codes.md index caddf1a7..e84a4fcd 100644 --- a/src/generated/error-codes.md +++ b/src/generated/error-codes.md @@ -22,7 +22,7 @@ | `type_not_permitted` | 403 | The credential holds no grant on the type at the level the operation asks for. | | `api_key_not_found` | 404 | No key the caller can reach has this id, or the key was already revoked. | | `blob_location_not_found` | 404 | The named store holds no copy of the blob, or is not attached. | -| `blob_not_found` | 404 | No blob the caller may read has this hash, or no attached store holds its bytes. | +| `blob_not_found` | 404 | No blob the caller may read has this hash, or no attached store holds its bytes. On a write of a file item, no blob the caller uploaded or may read has the hash its `blob_ref` names. | | `bulk_job_not_found` | 404 | No bulk action job the caller can see has this id. | | `connector_not_found` | 404 | No connector registration the credential may read has this id. | | `delivery_not_found` | 404 | The connector's registration has no delivery with this id. | @@ -65,6 +65,7 @@ | `idempotency_result_not_retained` | 422 | An `Idempotency-Key` repeats a request whose first answer was too large to keep. `details.original_status` is the status it carried. | | `rate_limited` | 429 | The credential, or an inbound endpoint, is past its request cap for the current window. | | `internal_error` | 500 | A fault that nothing else names a refusal for. The answer says nothing of what failed. | +| `blob_store_unavailable` | 503 | A blob store failed, or did not answer in time, while Marfa checked a blob's copy. Nothing was written. Retry later. | | `inbound_unavailable` | 503 | An inbound endpoint cannot take a delivery now, because its connector's backlog is full or the instance holds as many bodies in flight as it allows. Retry later. | | `stream_capacity_exhausted` | 503 | The instance is serving as many live event streams as it allows. `details.reason` is `viewer_cap`. | | `write_contention` | 503 | A write could not get the store's write lock within the busy budget. Retry it unchanged. | diff --git a/src/generated/marfa-release.json b/src/generated/marfa-release.json index 3094569a..09ae870c 100644 --- a/src/generated/marfa-release.json +++ b/src/generated/marfa-release.json @@ -1 +1 @@ -{"version":"v0.0.13"} +{"version":"v0.0.14"} diff --git a/src/generated/settings.md b/src/generated/settings.md index 2b425549..c4afa8f5 100644 --- a/src/generated/settings.md +++ b/src/generated/settings.md @@ -19,7 +19,7 @@ | `S3_ACCESS_KEY_ID` | None | Any text | The access key ID for the object store. | | `S3_SECRET_ACCESS_KEY` | None | Text with no whitespace around it | The secret access key for the object store. | | `S3_FORCE_PATH_STYLE` | `true` | `true`, `1`, `yes`, `on`, `false`, `0`, `no` or `off`, in any case | Whether the object store is addressed by path rather than by virtual host. Read only when `S3_ENDPOINT` is set. | -| `S3_PREFIX` | `blobs` | Any text | The key prefix under which the object store keeps its objects. | +| `S3_PREFIX` | `blobs` | `/`-joined segments of letters, digits, `.`, `_` and `-` | The key prefix the instance keeps everything in the bucket under: its blobs and, in the image, its database replica under `db/`. Each instance on a bucket needs its own. A `/` at either end is dropped; a `.` or `..` segment is refused. | | `MARFA_BLOB_MIN_COPIES` | `1` | Whole number, 1 or more | The fewest live copies a blob keeps. A drop that would leave fewer is refused. | | `MARFA_BLOB_READ_IDLE_TIMEOUT_MS` | `30000` ms | Whole number of milliseconds, 1 to 2147483647 | How long a blob read may make no progress before its stream and file or socket are closed. Slow consumers must read within this interval. | | `MARFA_BLOB_REPLICATE_INTERVAL_MS` | `60000` ms | Whole number of milliseconds, 1 to 3153600000000 | How often the `blob-replicate` job gives every attached store its copies. The job has no off switch. | @@ -100,7 +100,8 @@ | `MARFA_PLACEMENT_LOCATION` | None | Any text | Free text naming the location the server runs in, reported at `/health` as `placement.location`. | | `MARFA_PLACEMENT_COUNTRY` | None | Any text | Free text naming the country the server runs in, reported at `/health` as `placement.country`. | | `MARFA_OTEL_SAMPLE_RATIO` | `0.05` | Decimal number, 0 to 1 | The share of traces exported. A trace that contains an error is exported whatever this says: the span that failed, the spans held for the trace (at most 500) and every span after it. | -| `OTEL_SERVICE_NAME` | `marfa-server` | Any text | The service name on exported telemetry. | +| `OTEL_SERVICE_NAME` | `marfa-server` | Any text | The service name on exported telemetry. It takes precedence over a `service.name` in `OTEL_RESOURCE_ATTRIBUTES`. | +| `OTEL_RESOURCE_ATTRIBUTES` | None | Comma-separated `key=value` pairs, values percent-encoded | Attributes added to everything exported, as OpenTelemetry defines them. One named here replaces the server's own of the same name, such as `deployment.environment`, which an error notification names too. | | `OTEL_EXPORTER_OTLP_ENDPOINT` | None | Absolute `http` or `https` URL | The base URL of the OTLP receiver. Setting it turns telemetry on: the server exports traces to `v1/traces` and logs to `v1/logs` under it. Unset, nothing is exported and the telemetry SDK is not loaded. | | `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | None | Absolute `http` or `https` URL | The URL traces post to, used as written. It takes precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`, which must also be set. | | `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | None | Absolute `http` or `https` URL | The URL logs post to, used as written. It takes precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`, which must also be set. | From 0810d81325cecedd4c796aaa1a3148d05a4cabcc Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:40:09 +0100 Subject: [PATCH 02/12] feat(sync): generate the server image commands from the release --- .github/scripts/sync-from-marfa.test.sh | 8 ++++++-- scripts/sync-from-marfa.sh | 14 +++++++++++++- src/generated/server-image.md | 5 +++++ src/lib/docs-structure.test.ts | 18 ++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 src/generated/server-image.md diff --git a/.github/scripts/sync-from-marfa.test.sh b/.github/scripts/sync-from-marfa.test.sh index 5a865cb8..5103c1df 100644 --- a/.github/scripts/sync-from-marfa.test.sh +++ b/.github/scripts/sync-from-marfa.test.sh @@ -727,7 +727,7 @@ sync good echo "a good description" check "the script succeeds" 0 "${status}" check "it names what it synced and the release" \ - "openapi.json, the API overview, GLOSSARY.md, the CLI's global options and exit codes in src/generated/cli, the error codes, the housekeeping jobs, the settings, the edge type, permission and event type catalogues, the Catalog in src/generated/catalog, the contract index, the release (v1.2.3) and its CLI download synced from ${work}/good/marfa" "${out}" + "openapi.json, the API overview, GLOSSARY.md, the CLI's global options and exit codes in src/generated/cli, the error codes, the housekeeping jobs, the settings, the edge type, permission and event type catalogues, the Catalog in src/generated/catalog, the contract index, the release (v1.2.3), its CLI download and its server image commands synced from ${work}/good/marfa" "${out}" check "the overview is the description with each heading one level lower" \ "${expected}" "$(cat "${work}/good/docs/src/generated/api-overview.md")" check "the overview ends with one newline" "" "$(tail -c 1 "${work}/good/docs/src/generated/api-overview.md" | tr -d '\n')" @@ -764,8 +764,12 @@ check "the CLI download names that release and its archive" \ 'curl -fsSL https://github.com/withmarfa/marfa/releases/download/v1.2.3/marfa-1.2.3-darwin-arm64.tar.gz | tar -xz' \ "$(sed -n 2p "${work}/good/docs/src/generated/cli-download.md")" check "the CLI download is one bash block" '```bash|```' "$(head -1 "${work}/good/docs/src/generated/cli-download.md")|$(tail -1 "${work}/good/docs/src/generated/cli-download.md")" +check "the server image commands name that release's image, on loopback" \ + 'docker pull ghcr.io/withmarfa/marfa-server:1.2.3|docker run --name marfa -d --restart unless-stopped -p 127.0.0.1:8600:8600 \| -v marfa-data:/data --env-file marfa.env ghcr.io/withmarfa/marfa-server:1.2.3' \ + "$(sed -n 2p "${work}/good/docs/src/generated/server-image.md")|$(sed -n 3p "${work}/good/docs/src/generated/server-image.md")|$(sed -n 4p "${work}/good/docs/src/generated/server-image.md")" +check "the server image commands are one bash block" '```bash|```' "$(head -1 "${work}/good/docs/src/generated/server-image.md")|$(tail -1 "${work}/good/docs/src/generated/server-image.md")" check "nothing else is left in the generated folder" \ - "GLOSSARY.md api-overview.md catalog cli cli-download.md contract-index.json edge-types.md error-codes.md event-types.md housekeeping.md marfa-release.json permissions.md settings.md" "$(ls "${work}/good/docs/src/generated" | LC_ALL=C sort | tr '\n' ' ' | sed 's/ $//')" + "GLOSSARY.md api-overview.md catalog cli cli-download.md contract-index.json edge-types.md error-codes.md event-types.md housekeeping.md marfa-release.json permissions.md server-image.md settings.md" "$(ls "${work}/good/docs/src/generated" | LC_ALL=C sort | tr '\n' ' ' | sed 's/ $//')" # The Catalog's files: a Markdown file for each top-level type with its subtypes diff --git a/scripts/sync-from-marfa.sh b/scripts/sync-from-marfa.sh index c374d119..bca78d6c 100755 --- a/scripts/sync-from-marfa.sh +++ b/scripts/sync-from-marfa.sh @@ -44,6 +44,9 @@ # src/generated/marfa-release.json the latest release tag, shown in the sidebar # src/generated/cli-download.md the macOS install of that release's CLI, # included by the Quickstart and the CLI page +# src/generated/server-image.md the commands that pull that release's server +# image and start it, included by the Deploy +# guide # # Reads committed files from a checkout of the monorepo, by default the sibling # at ../marfa. Set MARFA_ROOT to use another. Nothing is generated: the @@ -803,5 +806,14 @@ sudo mv marfa /usr/local/bin/ marfa --version \`\`\` DOWNLOAD +# The server image is tagged with the release without its v. The loopback +# publish is deliberate: Docker publishes a bare port on every interface. +cat > "$GENERATED/server-image.md" < { }); }); +describe('the server image commands', () => { + it('come from the generated snippet, so a release sync updates them', () => { + const hardCoded = ALL_PAGES.filter(({ text }) => text.includes('marfa-server:')).map(({ file }) => file); + expect(hardCoded).toEqual([]); + const including = ALL_PAGES.filter(({ text }) => + text.includes('src/generated/server-image.md'), + ).map(({ file }) => file); + expect(including).toEqual([path.join('guides', 'deploy.mdx')]); + }); + + it('name the latest release and publish the port on loopback only', () => { + const { version } = JSON.parse(readFileSync(path.join(ROOT, 'src/generated/marfa-release.json'), 'utf8')); + const snippet = readFileSync(path.join(ROOT, 'src/generated/server-image.md'), 'utf8'); + expect(snippet).toContain(`docker pull ghcr.io/withmarfa/marfa-server:${version.slice(1)}\n`); + expect(snippet).toContain('-p 127.0.0.1:8600:8600'); + }); +}); + describe('the Glossary page', () => { it('defines only words marfa\'s glossary uses', () => { const glossary = readFileSync(path.join(ROOT, 'src/generated/GLOSSARY.md'), 'utf8').toLowerCase(); From f135e2ac67e5f519fdb05426276e408709f029bc Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:46:56 +0100 Subject: [PATCH 03/12] test(content): pin the server image commands to the release --- src/lib/docs-structure.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/docs-structure.test.ts b/src/lib/docs-structure.test.ts index 0a2f6040..96ff5656 100644 --- a/src/lib/docs-structure.test.ts +++ b/src/lib/docs-structure.test.ts @@ -64,7 +64,7 @@ describe('the CLI download', () => { describe('the server image commands', () => { it('come from the generated snippet, so a release sync updates them', () => { - const hardCoded = ALL_PAGES.filter(({ text }) => text.includes('marfa-server:')).map(({ file }) => file); + const hardCoded = ALL_PAGES.filter(({ text }) => /marfa-server:\d/.test(text)).map(({ file }) => file); expect(hardCoded).toEqual([]); const including = ALL_PAGES.filter(({ text }) => text.includes('src/generated/server-image.md'), From cef4c58aa316ab6e6211a2699590a5d78f76ddc1 Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:46:56 +0100 Subject: [PATCH 04/12] docs(guides): run the published server image and give each instance its own prefix --- content/docs/guides/deploy.mdx | 38 +++++++++++++++++--------------- content/docs/guides/settings.mdx | 3 +++ content/docs/guides/upgrade.mdx | 8 +++++-- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/content/docs/guides/deploy.mdx b/content/docs/guides/deploy.mdx index 1156bd86..e449afb0 100644 --- a/content/docs/guides/deploy.mdx +++ b/content/docs/guides/deploy.mdx @@ -12,7 +12,7 @@ An instance is one server process and one SQLite database, so there's no databas | Where | How | Good for | | --- | --- | --- | | A computer you own | Run the server under a supervisor that restarts it, as in [Run it without a container](#run-it-without-a-container). | A Mac or a home server that stays on. | -| A container | Build the image from `deploy/Dockerfile` in the [Marfa repository](https://github.com/withmarfa/marfa) and give it one volume, as in [Run it in a container](#run-it-in-a-container). | A server you already run. | +| A container | Pull the published server image and give it one volume, as in [Run it in a container](#run-it-in-a-container). | A server you already run. | | Railway | Deploy the [Marfa template](https://railway.com/new/template/5z9-Ja). It creates the server, a volume and a private bucket for backups, generates the two secrets, and sets the address. | The quickest way to an instance on the internet. Railway bills the service and its storage to you. | [Create a personal Marfa instance on Railway](https://github.com/withmarfa/marfa/blob/main/deploy/railway.md) has the Railway steps, including claiming the instance. @@ -103,39 +103,37 @@ You reach an instance in one of three ways: When a proxy or tunnel sits in front of the server, **tell the server which client address to believe**: `TRUSTED_PROXY_CIDRS` names the proxy's address ranges, or `TRUSTED_PROXY_HEADER` names a header the platform's edge overwrites with the client's address. Without either, the server ignores `X-Forwarded-For`, and every request seems to come from the proxy. +**That matters for sign-in.** The server limits password sign-in attempts at the owner's account to ten in fifteen minutes from one address, and answers the next `429`. When every caller seems to come from the proxy, they share one address, so anyone's wrong guesses can lock the owner out. + +A proxy on the same machine as a container, such as Tailscale Serve or a tunnel, reaches the container through Docker's network, so the server sees the container network's gateway address and not the proxy's. **Set `TRUSTED_PROXY_CIDRS` to the range of the container network**, such as the subnet that `docker network inspect bridge` shows for the default network, so the server reads each caller's own address from `X-Forwarded-For`. Trust only a range that nothing but your proxy can reach the server through: the server believes the header from every address in it. + ### An object store Name an S3-compatible bucket with `S3_BUCKET` and the other `S3_` settings, and the server keeps a second copy of every blob there. In the container, the bucket also receives the database, so the instance can be rebuilt from the bucket alone. +**Everything an instance keeps in the bucket is under its `S3_PREFIX`**, which is `blobs` unless you set it: its blobs at `/` and its database replica at `/db`. **Give each instance that shares a bucket its own prefix**, such as `trial/blobs`. Two instances with one prefix write one replica, and a new volume of either starts on whichever database was copied last. A prefix is segments of letters, digits, `.`, `_` and `-` joined by `/`, and a `/` at either end is dropped. A prefix keeps replicas apart, not access: each instance holds credentials for the whole bucket, so share a bucket only between instances of one owner. + **Put the bucket on a different site from the instance.** A link the bucket signs serves a blob from the bucket's own host, without the protections the instance adds to its own downloads, so a bucket under the instance's domain could let an uploaded HTML file run as a page of the instance. ## Run it in a container +Every release publishes the server image to `ghcr.io/withmarfa/marfa-server` for `linux/amd64` and `linux/arm64`, tagged with its release number and `latest`. **Pull a release by its tag, not `latest`**, so that a restart never changes the build, and so that the old and the new build of an [upgrade](/guides/upgrade) can each be named. + -### Build the image - -From a clone of the repository: - -```bash -docker build -f deploy/Dockerfile --build-arg VERSION_SHA=$(git rev-parse HEAD) -t marfa-server . -``` - - - +### Pull the image and start it -### Start it +Put `API_KEY_SALT`, `MARFA_AUTH_SECRET`, `MARFA_AUTH_BASE_URL` and the `S3_` settings in `marfa.env`, kept out of any repository. **Leave out `NODE_ENV`, `PORT` and the paths**: the image sets them, with the database, the blobs and the socket under `/data`. Then pull the image and start the container with one volume there: -Put `API_KEY_SALT`, `MARFA_AUTH_SECRET`, `MARFA_AUTH_BASE_URL` and the `S3_` settings in `marfa.env`, kept out of any repository. **Leave out `NODE_ENV`, `PORT` and the paths**: the image sets them, with the database, the blobs and the socket under `/data`. Then start the container with one volume there: +src/generated/server-image.md -```bash -docker run --name marfa -d --restart unless-stopped -p 8600:8600 \ - -v marfa-data:/data --env-file marfa.env marfa-server -``` +**`-p 127.0.0.1:8600:8600` publishes the port on the loopback address only.** A reverse proxy or a tunnel on the same machine reaches it there, and `MARFA_AUTH_BASE_URL` names the address people use. Docker publishes a bare `-p 8600:8600` on every network interface, and on Linux it does so past the host's firewall, so the instance would be open to the whole network. A proxy in front of the container needs `TRUSTED_PROXY_CIDRS`, as [The address](#the-address) says. Without `S3_BUCKET`, the container runs the server alone and says so in its log: that's a trial, because nothing leaves the machine. +To run code that no release holds, build the image from a clone of the repository with `docker build -f deploy/Dockerfile --build-arg VERSION_SHA=$(git rev-parse HEAD) -t marfa-server .`, and use `marfa-server` in place of the image name. It reports no `release`. + @@ -165,9 +163,11 @@ Then start it from `packages/server`, with the settings in its environment: ```bash cd packages/server -node dist/index.js +node --import ./dist/instrumentation.js dist/index.js ``` +**The `--import` loads the server's telemetry before anything else**, as the container does. Plain `node dist/index.js` starts the server but loads no telemetry, whatever the environment says. With `OTEL_EXPORTER_OTLP_ENDPOINT` unset, the preload exports nothing. [Send telemetry](/guides/telemetry) covers turning it on. + Claim it with `marfa --socket setup open`, as the account that runs the server. **The container is the tested route**; this suits a computer you own, under a supervisor such as launchd or systemd. ## Keep it running @@ -183,4 +183,6 @@ When the server can't start, its exit status tells your supervisor whether to tr A server that crashes can leave its socket file behind. On restart, the server replaces a socket that refuses connections. If another listener accepts connections there, the new start refuses and preserves it: check which process is running rather than removing its socket. +`GET /` answers the `release` the image was built as, beside `version`, which is the commit. A build from a clone has no `release`. + The rules behind this page are in the [instance chapter](https://github.com/withmarfa/marfa/blob/main/conformance/spec/instance.md) of the contract, and the container recipe is described in the [deployment guide](https://github.com/withmarfa/marfa/blob/main/deploy/README.md). diff --git a/content/docs/guides/settings.mdx b/content/docs/guides/settings.mdx index f0c41725..2ecc717e 100644 --- a/content/docs/guides/settings.mdx +++ b/content/docs/guides/settings.mdx @@ -34,6 +34,9 @@ Most settings have defaults that suit any instance. Check these before you keep - `NODE_ENV=production` makes the server refuse to start without `API_KEY_SALT`, `MARFA_AUTH_SECRET` and `MARFA_AUTH_BASE_URL`. Outside production, it falls back to a built-in salt and a new secret at each start, which signs every browser out at each restart. - `SQLITE_PATH` and `BLOB_PATH` say where your data lives. [Back up and restore](/guides/back-up-and-restore) covers protecting it. - `TRUSTED_PROXY_CIDRS` or `TRUSTED_PROXY_HEADER` tells the server which client address to believe behind a proxy or tunnel. [Deploy](/guides/deploy#the-address) explains when you need one. +- `S3_PREFIX` is the prefix an instance keeps everything in its bucket under, its blobs and its database replica. **Each instance on a shared bucket needs its own.** [Deploy](/guides/deploy#an-object-store) explains it. +- `MARFA_WEBHOOK_ALLOW_PRIVATE_ADDRESSES` lets outbound webhooks reach loopback, private and carrier-grade NAT addresses, which a receiver on a Tailscale or home network needs. [Receive webhooks](/guides/receive-webhooks#receive-deliveries-on-a-private-network) explains the refusal it removes. +- `OTEL_RESOURCE_ATTRIBUTES` marks an instance on its telemetry, such as `deployment.environment=staging` for a test instance, and `OTEL_SERVICE_NAME` renames the service. [Send telemetry](/guides/telemetry#tell-instances-apart) explains both. - `CORS_ORIGINS` names the browser origins whose pages may call the API. - `PORT` selects the public listener's port; `0` asks the operating system to assign one. `MARFA_LISTEN_HOST` selects its bind address. Leave it unset to use the runtime's unspecified address. diff --git a/content/docs/guides/upgrade.mdx b/content/docs/guides/upgrade.mdx index b8b7782a..6ef538df 100644 --- a/content/docs/guides/upgrade.mdx +++ b/content/docs/guides/upgrade.mdx @@ -26,12 +26,16 @@ Use these steps when the new build refuses your database. A build that accepts t Follow [Export an archive](/guides/back-up-and-restore#export-an-archive). **Stop your apps first**, so the archive holds every write, and check that the file unpacks. +Note which build this is. `GET /` answers its `release` beside `version`, which is the commit, and `marfa status` shows both. **The image of that release, `ghcr.io/withmarfa/marfa-server:`, stays available under that tag**, so you can start the old build again at any time, even on a machine that has pulled a newer one. A build made from a clone has no `release`, so note its `version` and keep that clone. + ### Start the new build on empty storage -Give the new build a new database file and an empty blob folder. In a container, that's a new volume. If you use a bucket, **give the new instance a new bucket too**: on an empty volume, the container restores the old database from its bucket. If that database has the incompatible schema, the new build refuses it again. +Give the new build a new database file and an empty blob folder. In a container, that's a new volume, and the new container can run beside the old one on **its own volume and its own port**, such as `-p 127.0.0.1:8601:8600`. + +If you use a bucket, **give the new instance an `S3_PREFIX` of its own**, such as `v2/blobs`, or a new bucket. On an empty volume, the container restores the database it finds under its prefix, and under the old instance's prefix that is the old database. If that database has the incompatible schema, the new build refuses it again. An instance keeps its blobs and its database replica under its prefix, so a new prefix on the same bucket leaves the old instance's replica as it is. @@ -65,7 +69,7 @@ Make your keys again, register your webhooks again, and set the configuration ag ### Check, then retire the old build -`GET /` answers the build that's running, as `version`. Remove the old build and its data **only when the new instance holds what you expect**. +`GET /` answers the build that's running, as `release` and `version`. Remove the old build and its data **only when the new instance holds what you expect**. From 99a34cc3634b61dbf3e8ae2408515328b32d07c4 Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:46:57 +0100 Subject: [PATCH 05/12] docs(guides): say what a backup lacks and how a restore reports it --- content/docs/guides/back-up-and-restore.mdx | 55 +++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/content/docs/guides/back-up-and-restore.mdx b/content/docs/guides/back-up-and-restore.mdx index e7a8e75a..532d78d5 100644 --- a/content/docs/guides/back-up-and-restore.mdx +++ b/content/docs/guides/back-up-and-restore.mdx @@ -32,9 +32,17 @@ To restore, stop the server, put the copied files where `SQLITE_PATH` and `BLOB_ ### With a bucket -When `S3_BUCKET` is set, **the container copies for you**. It streams each committed database write to the bucket within a second and keeps a day of hourly snapshots, and the `blob-replicate` housekeeping job copies each blob there. A new volume started with the same bucket and secrets restores the database as it starts, and brings the blobs back as replication runs. +When `S3_BUCKET` is set, **the container copies for you**. It streams each committed database write to the bucket within a second and keeps a day of hourly snapshots, and the `blob-replicate` housekeeping job copies each blob there. A new volume started with the same bucket, prefix and secrets restores the database as it starts, and brings the blobs back as replication runs. -**The database reaches the bucket before a blob does**, so a restore to a moment between the two can name a blob the bucket doesn't hold yet. The [deployment guide](https://github.com/withmarfa/marfa/blob/main/deploy/README.md#restoring-by-hand) has the commands to restore to the latest state or to a point in time. +**Everything an instance keeps in the bucket is under its `S3_PREFIX`**: its blobs at `/` and its database replica at `/db`. **Every instance on a shared bucket needs a prefix of its own**, so that a new volume restores its own database and not another instance's. [Deploy](/guides/deploy#an-object-store) says what a prefix may hold. + +**The database reaches the bucket before a blob does**, so a restore to a moment between the two can name a blob the bucket doesn't hold yet. To restore by hand, run Litestream against the replica, and **set `S3_PREFIX` to the instance's prefix** with no `/` at either end, even when it's the default `blobs`: Litestream reads an unset name as nothing, and would look for the replica at `/db`. + +```bash +S3_PREFIX=blobs litestream restore -config deploy/litestream.yml -o restored.db -integrity-check full "$SQLITE_PATH" +``` + +The [deployment guide](https://github.com/withmarfa/marfa/blob/main/deploy/README.md#restoring-by-hand) has the same command for a point in time. ## Export an archive @@ -87,11 +95,49 @@ types.ndjson blobs/sha256:a0feecd7233d9f5f3ec784c6bdb36587c651e2ec60437c27b4bf135de39149d7 ``` -If the server fails partway through an export, it ends the connection early, and the file doesn't unpack. +If the server fails partway through an export, it ends the connection early, and the file doesn't unpack. **`marfa export --output` writes to a file beside the path and renames it only once the stream has ended whole.** A stream cut short removes that file, leaves any earlier file at the path as it was, and ends with exit code 3, so try again. + + + + +### Check what the archive lacks + +The manifest names each file's bytes the archive couldn't carry, under `missing_blobs`. Read it from the archive: + +```bash +tar -xzOf marfa-archive.tar.gz manifest.json +``` + +The manifest's other fields are left out here: + +```json +{ + "blob_count": 0, + "blobs": {}, + "missing_blobs": { + "sha256:a0feecd7233d9f5f3ec784c6bdb36587c651e2ec60437c27b4bf135de39149d7": { + "mime_type": "image/jpeg", + "size_bytes": 2048 + } + } +} +``` + +An empty `missing_blobs` means the archive holds every blob its items name. The next section says what a name in it means. +### When an archive lacks a file's bytes + +An export never claims a blob it doesn't hold. When no store holds a whole copy of a blob, such as after a disk lost the file, the export **leaves the blob out of the archive** and names it in `missing_blobs`, with its `mime_type` and `size_bytes`. The rest of the instance is still backed up: one lost file costs the backup that file, not everything. + +`marfa export` writes the archive, then **exits with code 1 and `blobs_missing`**, naming up to ten of the hashes and counting the rest. A script that reads only the exit code can't take such an archive for a whole one. The HTTP answer is still `200`, so read the manifest if you export without the CLI. + +A file item can name bytes the instance has no record of, as after a restore that lacked them. The export names such a blob too, with `null` for both `mime_type` and `size_bytes`, because nothing knows them. **Every later backup keeps naming a file's blob it lacks until the bytes are uploaded again**, with `marfa blobs upload` or [Upload a blob](/api-reference/blobs/uploadBlob). Only a file item's `blob_ref` counts: a hash that appears in a note's text is not named. + +A store that fails, or doesn't answer within 15 seconds, is not a lost file. The export, or a restore, is refused with `503 blob_store_unavailable` before anything is written, unless another store holds the blob whole. **Try again once the store answers.** + **An export isn't taken at one instant**: an item created while it runs may not be in it. Stop your apps first if the archive must hold every write. The server prepares the whole archive before it sends the first byte, in a folder inside the blob folder, so leave room on that disk for about as much as the archive's text takes uncompressed. A [source filter](/guides/settings#what-the-enforcement-levers-do) leaves items out of an export as it does out of a listing, so check that neither the instance nor the export key carries one that hides what you want kept. @@ -114,6 +160,7 @@ The answer counts what the restore did. Into a fresh instance: "edges_skipped": 0, "edges_skipped_reasons": {}, "blobs_imported": 1, + "blobs_missing": [], "types_registered": 0, "types_skipped": 0, "edge_types_registered": 0, @@ -127,6 +174,8 @@ A restored `system.connection` item for an app has `status: revoked`. It records An item the instance already holds is counted in `duplicates` and left as it is, so replaying an archive never overwrites work done since. Restoring the same archive again into the same instance answers `"imported": 0` and `"duplicates": 5`. +`blobs_missing` lists the hashes the archive names but doesn't carry bytes for, that no store of the instance holds either. The restore still writes the items, and **writes no record for those blobs**, so a file that names one is a file with no bytes until you upload them again with `marfa blobs upload`. `marfa restore` prints the answer and exits with code 1 and `blobs_missing` when the list isn't empty, naming up to ten of the hashes. The same check covers an archive whose blob entry is missing or doesn't hash to its name. + A restore writes in one transaction. If it's refused, fails or is interrupted, it leaves nothing behind, so you can run it again. While it runs, other writes wait for it, so let it answer before you start your apps. **An archive never holds keys, webhooks, the configuration or the audit log**: make keys, register webhooks and set the configuration again on the new instance. From 671a285a424de4a9530da2fb2ff7d8b64917dd47 Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:46:57 +0100 Subject: [PATCH 06/12] docs(guides): explain webhooks to private addresses and unstored Tailscale headers --- content/docs/guides/connector-webhooks.mdx | 4 +++- content/docs/guides/receive-webhooks.mdx | 14 +++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/content/docs/guides/connector-webhooks.mdx b/content/docs/guides/connector-webhooks.mdx index 6c217dea..a4407384 100644 --- a/content/docs/guides/connector-webhooks.mdx +++ b/content/docs/guides/connector-webhooks.mdx @@ -92,7 +92,7 @@ const { data: page } = await marfa.GET("/connectors/{id}/deliveries", { }); ``` -Each delivery is the request as it arrived, with every header in the order and case it came in: +Each delivery is the request as it arrived, with every header in the order and case it came in, **except the headers a Tailscale proxy adds**: ```json { @@ -116,6 +116,8 @@ Each delivery is the request as it arrived, with every header in the order and c A page can be short and still have a `next_cursor` to follow, so read until it's `null`. **Only the connector's own key reads deliveries**: any other caller is refused, the owner included. +**Marfa doesn't store a header whose name begins with `Tailscale-`**, in any case, such as `Tailscale-User-Login`. Tailscale adds these to a request from your tailnet when it serves your instance, and they name the person who sent it, so a connector never sees them. A header the sender chose, such as `X-Tailscale-Note`, is stored. + ## Check the signature Anyone who learns the address can post to it, so **a stored delivery proves nothing about who sent it**. Most vendors sign the body with the secret they share with you, and send the signature in a header. [Get an inbound delivery's body](/api-reference/connectors/getInboundDeliveryBody) answers the body byte for byte, so the signature you compute over it is the one the vendor made: diff --git a/content/docs/guides/receive-webhooks.mdx b/content/docs/guides/receive-webhooks.mdx index f3e2c31b..7452da68 100644 --- a/content/docs/guides/receive-webhooks.mdx +++ b/content/docs/guides/receive-webhooks.mdx @@ -60,6 +60,8 @@ To use a secret of your own instead, send `secret` with at least 32 characters. The webhook receives events written after you registered it, and **nothing earlier**. +**Marfa refuses a `url` that points at a private address**, at registration and when you change the `url` of a webhook. [Receive deliveries on a private network](#receive-deliveries-on-a-private-network) says which addresses and how to allow them. + ## Check each delivery A delivery is a `POST` whose body is the event, as the stream would send it, with three more members: `event_id`, the event's ID in the log; `delivery_id`, which stays the same on every attempt to deliver this event; and `delivered_at`, when Marfa sent this attempt. Here's one, as it arrives: @@ -141,7 +143,7 @@ Each delivery's `event_id` is the event's ID in the log, and IDs rise with every Marfa tries a delivery again when your receiver can't be reached, doesn't answer in time, or answers `5xx`, `408` or `429`. It waits longer before each attempt, starting at a second, and **makes eight attempts over about five hours**; a `Retry-After` header asks it to wait longer, up to five minutes. **Any other `4xx`, or a redirect, ends the delivery at once.** -A delivery Marfa gives up on is `dead_letter`. That's why the receiver above answers `401` for a bad signature: if you've configured the wrong secret, Marfa stops at once rather than retrying for hours, and you can send the deliveries again once you've fixed it. +A delivery Marfa gives up on is `dead_letter`. A delivery whose receiver's address isn't public is one: see [Receive deliveries on a private network](#receive-deliveries-on-a-private-network). That's why the receiver above answers `401` for a bad signature: if you've configured the wrong secret, Marfa stops at once rather than retrying for hours, and you can send the deliveries again once you've fixed it. [List webhook deliveries](/api-reference/webhooks/listWebhookDeliveries) shows each delivery, newest first, with its `status` and the last answer your receiver gave: @@ -198,8 +200,14 @@ await marfa.POST("/webhooks/{id}/deliveries/{delivery_id}/redeliver", { **Only a `dead_letter` delivery can be sent again**; any other answers `409 conflict`. A delivery's record is kept for as long as the instance keeps its audit log. -## Receive deliveries on your own machine +## Receive deliveries on a private network + +By default, **Marfa posts only to public addresses**. A host that isn't public is a loopback address such as `localhost`, a private address, a link-local or unique-local address, or a carrier-grade NAT address such as the `100.64.0.0/10` range that Tailscale uses. Marfa refuses a webhook whose `url` has such a host, both when you register it and when you change its `url`, with `400 validation_error`. The message names the setting that allows it, `MARFA_WEBHOOK_ALLOW_PRIVATE_ADDRESSES`. + +**A name counts as well as an address.** Marfa looks up a host written as a name and refuses it when any address it resolves to isn't public. A receiver that you reach by a Tailscale name, or by a name on your home network, resolves to a private address, so this is the refusal you meet first. It needs `MARFA_WEBHOOK_ALLOW_PRIVATE_ADDRESSES=true` in the instance's environment, and a restart. [Deploy](/guides/deploy#the-address) describes reaching an instance over Tailscale, and [Settings](/guides/settings) the setting. + +To try a receiver on your own machine, start a development instance with the setting on, register `http://localhost:3000/marfa`, and run the receiver above. **Leave the setting off on an instance you keep**, unless its receivers run on a private network. -By default, **Marfa posts only to public addresses**, and refuses a webhook whose URL is `localhost` or a private address. To try a receiver on your own machine, start a development instance with `MARFA_WEBHOOK_ALLOW_PRIVATE_ADDRESSES=true`, register `http://localhost:3000/marfa`, and run the receiver above. **Leave the setting off on an instance you keep**, unless its receivers run on a private network: [Settings](/guides/settings) describes it. +Marfa accepts a name that doesn't resolve, or that its lookup doesn't answer within three seconds, because the receiver may not exist yet. It checks the name again at each delivery. **A delivery to a name that resolves to an address that isn't public is never sent.** It's recorded as `dead_letter` at that attempt, with the error "The receiver's address is not public.", and not retried. Once you've allowed private addresses or fixed the name, [redeliver it](#when-a-delivery-fails). The rules behind this guide are in the [events](https://github.com/withmarfa/marfa/blob/main/conformance/spec/events.md) chapter of the contract. From 6faeb57f6ffa655047d018dd1266fd6ebaddacd1 Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:46:57 +0100 Subject: [PATCH 07/12] docs(guides): tell instances apart in telemetry --- content/docs/guides/telemetry.mdx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/content/docs/guides/telemetry.mdx b/content/docs/guides/telemetry.mdx index ee7d1ce0..00d8e8b3 100644 --- a/content/docs/guides/telemetry.mdx +++ b/content/docs/guides/telemetry.mdx @@ -17,12 +17,24 @@ export MARFA_OTEL_SAMPLE_RATIO=0.1 - **`OTEL_EXPORTER_OTLP_HEADERS`** holds the headers the receiver wants, as comma-separated `key=value` pairs with percent-encoded values. It is a secret: keep it out of any file you commit. - **`MARFA_OTEL_SAMPLE_RATIO`** is the share of traces exported, from 0 to 1. **A trace that contains an error is always exported**, whatever the ratio. -- **`OTEL_SERVICE_NAME`** names the service on what Marfa sends. +- **`OTEL_SERVICE_NAME`** names the service on what Marfa sends. It is `marfa-server` unless you set it, and it wins over a `service.name` in `OTEL_RESOURCE_ATTRIBUTES`. The settings also take an address and headers for traces or logs alone, which override the shared ones: `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` and `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` are used as written, and `OTEL_EXPORTER_OTLP_TRACES_HEADERS` and `OTEL_EXPORTER_OTLP_LOGS_HEADERS` win over `OTEL_EXPORTER_OTLP_HEADERS`. **The two address settings need `OTEL_EXPORTER_OTLP_ENDPOINT` set too**, or the server refuses to start. The header settings have no such check, and without the shared address nothing reads them. Marfa drops what carries a credential, a request body, raw SQL or an email address, and strips the query string from a URL, before it exports anything. +### Tell instances apart + +Several instances can send to one project, such as your own, a demo and a test. **Every log record and span names its instance by `service.instance.id`**, which is the `instance_id` that `GET /` answers. So does every exception reported to PostHog once the server has read the id, which it does as soon as its database opens. Look the id up with `GET /` to find which instance an export came from. The instance's name isn't sent: it can change while the server runs, and it can be personal. + +`OTEL_RESOURCE_ATTRIBUTES` adds attributes of your own, as comma-separated `key=value` pairs with percent-encoded values. **An attribute named there replaces the server's own of the same name**, `service.instance.id` and `service.version` included. Mark a test or demo instance with `deployment.environment`, which is `production` unless you set it: + +```bash +export OTEL_RESOURCE_ATTRIBUTES=deployment.environment=staging +``` + +The `environment` of a [failure notice](#post-a-notice-when-a-request-fails) takes the value of `deployment.environment.name` if the list names it, and otherwise of `deployment.environment`. The server refuses to start when the list is malformed, and names the setting. + **Telemetry loads only when the server starts with its preload.** The container already does. When you run the server with Node yourself, start it this way, because plain `node dist/index.js` exports nothing whatever the variables say: ```bash @@ -53,4 +65,4 @@ A system can't report its own death, so something outside the instance has to no ## What the CLI sends -**The CLI sends nothing unless the environment names an OTLP endpoint**, `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`. With one set, each command sends one trace, named for the command, with its exit code. The trace holds no argument, address, credential, item content or message. A failed command also sends one exception to PostHog when an OTLP endpoint and both `MARFA_POSTHOG_HOST` and `MARFA_POSTHOG_PROJECT_TOKEN` are set, as the server does. `OTEL_SDK_DISABLED=true` turns the CLI's telemetry off. It also reads the standard variables for headers, timeout, sampler and service name, as [the CLI reference](https://github.com/withmarfa/marfa/blob/main/core/marfa-cli/COMMANDS.md#telemetry) lists. +**The CLI sends nothing unless the environment names an OTLP endpoint**, `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`. With one set, each command sends one trace, named for the command, with its exit code. The trace holds no argument, address, credential, item content or message. A failed command also sends one exception to PostHog when an OTLP endpoint and both `MARFA_POSTHOG_HOST` and `MARFA_POSTHOG_PROJECT_TOKEN` are set, as the server does. `OTEL_SDK_DISABLED=true` turns the CLI's telemetry off. It adds the attributes `OTEL_RESOURCE_ATTRIBUTES` names to its trace and its exception, and a list with one pair it can't read is ignored whole. `OTEL_SERVICE_NAME`, which names the trace `marfa-cli` unless you set it, wins over a `service.name` in that list. It also reads the standard variables for headers, timeout, sampler and service name, as [the CLI reference](https://github.com/withmarfa/marfa/blob/main/core/marfa-cli/COMMANDS.md#telemetry) lists. From 093bcae04889c7ed96541ee9e2bd47b454cc2a90 Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:46:57 +0100 Subject: [PATCH 08/12] docs(cli): keep any valid key, report events exits and name the release --- content/docs/(reference)/cli.mdx | 10 ++++++---- content/docs/guides/authenticate.mdx | 2 +- content/docs/guides/keep-in-sync.mdx | 2 ++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/content/docs/(reference)/cli.mdx b/content/docs/(reference)/cli.mdx index 28ca16e1..85c4e0e9 100644 --- a/content/docs/(reference)/cli.mdx +++ b/content/docs/(reference)/cli.mdx @@ -54,12 +54,14 @@ Waiting for the approval... The token holds only the scopes you approve. The CLI asks for everything an owner can grant except `instance.read`, `instance.maintain`, `connectors.manage`, `blobs.manage` and `keys.manage`, so the commands that need one of those need a key instead. `marfa logout` revokes the token and forgets it. -Or **keep a key** you already have with `marfa keys keep`. It reads the key from `MARFA_API_KEY`, or, if that isn't set, from standard input, so you can paste it and press Return. It checks the key with the server, then keeps it in the keychain: +Or **keep a key** you already have with `marfa keys keep`. It reads the key from `MARFA_API_KEY`, or, if that isn't set, from standard input, so you can paste it and press Return. It checks the key with the server, then keeps it in the keychain. ```bash marfa keys keep --url https://marfa.example.com ``` +**Any valid key is kept**, including one that holds only permissions such as `keys.mint`, `instance.read` or `audit.read` and reaches no type. A key the server doesn't accept, because it's revoked, expired or unknown, is kept nowhere and ends the command with exit code 5. An app's access token isn't a key: the command keeps nothing and exits 1, because the token ends within the hour and `marfa login` keeps a sign-in that renews it. + [Manage keys](/guides/manage-keys) shows how to make a key that reaches only what you need. Either way, the server you sign in to or keep a key for becomes the current server, which a command uses when nothing names another. The keychain holds one credential for each server. For each command, the CLI takes the first server and the first credential that's set, in this order: @@ -111,11 +113,11 @@ marfa whoami ```text server https://marfa.example.com -instance 01a12312-7e62-70f2-a656-1c9eef418991 (Marfa dev) +instance 01a12312-7e62-70f2-a656-1c9eef418991 (Marfa 0.0.14, build 37eff92) a key from the keychain ``` -The instance name comes from its live configuration and defaults to Marfa. [Settings](/guides/settings#change-settings-while-it-runs) shows how to change it. +The instance name comes from its live configuration and defaults to Marfa. [Settings](/guides/settings#change-settings-while-it-runs) shows how to change it. After the name comes the **release** the server was built as, then the commit it was built from. `marfa status` names them the same way. A server built from a clone shows only the commit, because it has no release. When a command reaches the wrong instance, or is refused, start here. @@ -137,7 +139,7 @@ marfa --socket /path/to/data/control/marfa.sock setup status That's how the owner makes the first key, as in the [Quickstart](/quickstart), and recovers a lost password. A few commands work only this way: `marfa setup status`, `marfa setup code`, `marfa setup open`, `marfa owner recover`, `marfa owner update`, the `marfa owner passkeys` commands, and the `marfa sign-ins` commands. -`marfa owner passkeys add` opens a fresh owner sign-in in a browser, followed by passkey registration. Use `--no-browser` to print the sign-in URL instead. The command does not sign you in or make a setup credential. [Authenticate](/guides/authenticate#sign-in-as-the-owner) covers passkey management. +`marfa owner passkeys add` opens the ordinary owner sign-in page in a browser, where you sign in yourself with the password or a passkey, and then reach the passkey page. Use `--no-browser` to print the sign-in URL instead. The command does not sign you in or make a setup credential. [Authenticate](/guides/authenticate#sign-in-as-the-owner) covers passkey management. `marfa --socket owner update --email owner@example.com --name "Reader"` changes the owner's sign-in address and display name. `marfa profile update --name "Reader"` uses a key or app token with profile write access instead; it can change name and avatar, never email. [Authenticate](/guides/authenticate#change-the-owners-email-or-name) explains both operations. diff --git a/content/docs/guides/authenticate.mdx b/content/docs/guides/authenticate.mdx index fb4faad4..b88a40f5 100644 --- a/content/docs/guides/authenticate.mdx +++ b/content/docs/guides/authenticate.mdx @@ -11,7 +11,7 @@ The owner signs in in a browser to manage the instance and approve apps. For dat To add a passkey, open `/auth/owner/passkeys` with a browser session authenticated within the last five minutes. This page also lists and removes individual passkeys. **Renaming or removing a passkey requires authentication within the last five minutes**; listing them needs only a live owner session. Removing one prevents further sign-in with it, while existing sign-ins and other passkeys keep working. -On the instance's own machine, `marfa --socket owner passkeys list` lists the passkeys. Use `marfa --socket owner passkeys rename --name "Laptop"` to rename one, or `marfa --socket owner passkeys remove ` to remove it. `marfa --socket owner passkeys add` opens a fresh browser sign-in, then the passkey page; `--no-browser` prints the URL instead. **The add command issues no sign-in credential**: you still authenticate in the browser and register the passkey there. [Recover the owner's password](/guides/manage-keys#recover-the-owners-password) remains the way to recover a lost password. +On the instance's own machine, `marfa --socket owner passkeys list` lists the passkeys. Use `marfa --socket owner passkeys rename --name "Laptop"` to rename one, or `marfa --socket owner passkeys remove ` to remove it. `marfa --socket owner passkeys add` opens the ordinary sign-in page, where you sign in with the password or a passkey, and then the passkey page; `--no-browser` prints the URL instead. **The add command issues no sign-in credential**: you still authenticate in the browser and register the passkey there. [Recover the owner's password](/guides/manage-keys#recover-the-owners-password) remains the way to recover a lost password. Passkeys need a supported browser and HTTPS, or localhost for development. **A valid passkey works even when password attempts are locked.** It creates the same owner browser authority used to approve apps and manage the instance; an app still needs its own token to read and write data. diff --git a/content/docs/guides/keep-in-sync.mdx b/content/docs/guides/keep-in-sync.mdx index 0eceaaf9..0d38fb24 100644 --- a/content/docs/guides/keep-in-sync.mdx +++ b/content/docs/guides/keep-in-sync.mdx @@ -164,6 +164,8 @@ A stream doesn't end of its own accord. What you see when it does tells you what | `catchup_too_old` | The log no longer holds the events after your cursor. By default it keeps them for at least seven days. | Read the state again and start from the head of the log. | | `cursor_ahead` | Your cursor is past the log's head, as after the instance was restored from a backup. | Discard the cursor, read the state again and start from the head of the log. | +`marfa events` ends with a status that says which of these happened, so a script need not read the frames. It prints the frame it read, then exits `1` for `cursor_ahead` and `catchup_too_old`, with the error on standard error, `3` for a `stream_incomplete` that a new stream can mend, such as `reader_behind` or `server_stopping`, and `5` for `credential_ended`, which `marfa device hydrate` and `marfa device catch-up` report the same way. + `rebuild` does the three steps at the top of this guide, with the [TypeScript SDK](/typescript-sdk) for the listing, and returns the head it started from. The listing and the stream both ask for `core.note` in every state, so the copy holds exactly what the stream keeps current. The loop after it follows the stream for as long as the program runs. `clear` and `loadCursor` are yours too: From b21beacaa210b96c69c48d5b30c3f31e39220714 Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:46:57 +0100 Subject: [PATCH 09/12] docs(guides): report how a folder push ends and what a restore could not write --- content/docs/(overview)/devices.mdx | 2 ++ content/docs/guides/sync-a-folder.mdx | 32 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/content/docs/(overview)/devices.mdx b/content/docs/(overview)/devices.mdx index f3f2e5d4..4eb5718c 100644 --- a/content/docs/(overview)/devices.mdx +++ b/content/docs/(overview)/devices.mdx @@ -50,6 +50,8 @@ Each write the instance answers gets one verdict: A device's working copy can also be a directory of files on your computer: **a folder**. Each item the folder holds is a file, a note becomes a Markdown file, and an edit to either the file or the item reaches the other. +A file moved to another folder on the same computer stays the same item, because the file carries its item's ID, even when an editor saves the moved file again as UTF-16. + A folder keeps a working copy and a queue like any device. [Sync a folder](/guides/sync-a-folder) sets one up, and [Keep folders running](/guides/keep-folders-running) keeps every folder in step from login. The rules behind this page are in the [device](https://github.com/withmarfa/marfa/blob/main/conformance/spec/device.md), [queue and verdicts](https://github.com/withmarfa/marfa/blob/main/conformance/spec/queue-and-verdicts.md) and [folders](https://github.com/withmarfa/marfa/blob/main/conformance/spec/folders.md) chapters of the contract. diff --git a/content/docs/guides/sync-a-folder.mdx b/content/docs/guides/sync-a-folder.mdx index 66aa3382..3756f866 100644 --- a/content/docs/guides/sync-a-folder.mdx +++ b/content/docs/guides/sync-a-folder.mdx @@ -193,6 +193,29 @@ The folder leaves a name alone when it can't tell the name from your choice: a f **A file you delete moves its item to the bin** after a few seconds' grace, so a save that replaces a file isn't taken for a delete; the item stays in the bin until you restore or purge it. +## Know whether a push finished + +`marfa folders push` ends with an exit code that says whether it finished, so a script need not read the report: + +| Exit code | Meaning | +| --- | --- | +| `0` | The push finished. | +| `3` | A write is left unsent, or the catch-up with the instance failed. Try again. | +| `5` | The server refused the credential. Use a valid key. | + +`marfa folders push --send-only` follows the same rule for a write left unsent and for a refused credential. + +**A push under a rate limit doesn't wait.** When the server refuses writes because the key is over its rate limit, the push leaves them queued, reports them as not delivered, names how long the server asked for, and exits `3`. The end of the report reads: + +```text +2 write(s) not delivered, waiting for the next drain +the server asked for 60s before the next drain; push again after that to send the rest +``` + +Run the push again after that wait, as a script would. A large first sync takes one push for each window. **A watch waits on its own**: after a pass leaves writes undelivered under a named wait, it starts no pass until the wait is over, and then sends the rest. + +The first line of a report counts files. A file whose create or edit was sent earlier and hasn't been answered yet counts as **waiting**, not unchanged, and the line adds `; 2 queued earlier and not delivered yet`. With `--json`, the count is `waiting`. A file whose write was refused for good counts as unchanged, because Marfa never sends that write again. + ## A missing or unreadable directory A scan that cannot read or identify an entry is incomplete. **It does not treat files it failed to see as deleted**, and it defers deletions already waiting until a complete scan succeeds. Fix the directory or disk access and sync again. @@ -201,6 +224,8 @@ Each directory has its own working copy and queue, recorded in the machine's fol A scan also leaves another folder's state directory and contents alone. If the registry cannot be read, deletion waits when the device cannot rule out a file having moved. +**A file moved to another folder on the same Mac stays the same item.** The file carries its `marfa_id`, and the folder it left reads it there, so the folder it left doesn't take the move for a delete. That holds when an editor saves the moved file again as UTF-16, which changes its bytes: Marfa reads the `marfa_id` of a file that starts with a UTF-16 byte order mark. A file in another folder that isn't text doesn't hold up the deletes of this folder's files. + ## A large removal waits for you When many files go at once, such as when a directory is moved away or a disk isn't mounted, **the folder doesn't move their items to the bin**. It pauses the removal and waits for you to decide: @@ -228,6 +253,13 @@ if status.paused.isPaused { } ``` +`restore` counts only the files it wrote back. **A file it couldn't write**, such as one whose bytes it couldn't fetch, is listed with the flag the pull gave it, or `unwritten` where the pull gave none, and the command says so. Its item stays: the next push doesn't send it to the bin, and every later pull writes the file once the bytes can be fetched. + +```text +2 file(s) written back; 0 item(s) restored, sent at the next push; 1 file(s) not written back yet, which every pull tries again +Reading list.md: the pull did not write it +``` + `marfa folders status Notes` shows where every file stands, and whether a removal waits, without asking the instance. To see how every computer that follows the folder last reported, from any of them, `marfa folders health` takes the folder's ID and reads those reports from the instance. A report's age says when it arrived, not whether that computer is reachable now. ## What a folder never takes in From 035bd119172f313b952cc9435aa31fe959dbdf6a Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:46:57 +0100 Subject: [PATCH 10/12] docs(guides): refuse a file item naming bytes its writer did not upload --- content/docs/(overview)/files.mdx | 2 +- content/docs/guides/handle-files.mdx | 4 +++- content/docs/guides/write-safely.mdx | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/content/docs/(overview)/files.mdx b/content/docs/(overview)/files.mdx index 90fca03b..d0a0039c 100644 --- a/content/docs/(overview)/files.mdx +++ b/content/docs/(overview)/files.mdx @@ -39,7 +39,7 @@ The same bytes uploaded again answer the same hash and are stored once, so two i **Knowing a hash isn't enough to read the bytes.** A credential reads a blob through an item that names it: if you can read a file item, you can read its blob. A hash anywhere else in an item counts too, such as an image link inside a note's Markdown body. -A reference counts only if whoever wrote it had uploaded the bytes or could already read them, so naming a hash you found somewhere gives you nothing. +A reference counts only if whoever wrote it had uploaded the bytes or could already read them, so naming a hash you found somewhere gives you nothing. **A file item can't name such a hash at all**: the write is refused `404 blob_not_found`, so the item is never made with bytes that nobody can read. Upload the bytes with the same credential first, as [Handle files](/guides/handle-files#point-an-item-at-the-bytes) shows. A blob you may not read answers `404 blob_not_found`, exactly as a hash that doesn't exist does, so a refusal never confirms that a file exists. That includes a blob you uploaded yourself, until an item names it. diff --git a/content/docs/guides/handle-files.mdx b/content/docs/guides/handle-files.mdx index a7ffd009..f7170940 100644 --- a/content/docs/guides/handle-files.mdx +++ b/content/docs/guides/handle-files.mdx @@ -94,7 +94,9 @@ Marfa sets the item's `size_bytes` from the stored bytes. Soon after, a housekee `marfa items add reading-list.txt` uploads the bytes and creates the item in one command. To make a file belong to another item, such as a photo attached to a note, link the two with an `attached-to` edge from the file item: `marfa items attach photo.jpg` and the Swift SDK's `attach(to:file:)` do all three steps. -**Until an item names the hash, the bytes can't be read**, even by the credential that uploaded them; only the owner and a credential with `blobs.manage` can. And naming a hash in `blob_ref` lends a read only if you uploaded those bytes or could already read them, so a hash copied from elsewhere gives you an item whose download answers `404 blob_not_found`. +**Until an item names the hash, the bytes can't be read**, even by the credential that uploaded them; only the owner and a credential with `blobs.manage` can. **A file item may name only bytes its writer uploaded or can already read.** A write that names another hash in `blob_ref`, such as one copied from elsewhere or uploaded with another key, is refused `404 blob_not_found` and writes nothing. It is the answer a hash that nothing holds gets, so it never tells you whether the instance has the bytes. **Upload the bytes with the same credential that writes the item, then write again.** Uploading bytes the instance already holds stores nothing new. A hash in any other property, such as an image link in a note's body, is text: the write is taken and the hash lends nothing. + +A working copy follows the same rule for you. If the key changes after a file's bytes were uploaded and before its write is sent, the copy uploads the bytes again under the current key, ahead of the write. ## Download the bytes diff --git a/content/docs/guides/write-safely.mdx b/content/docs/guides/write-safely.mdx index c07c9ccd..6b84f03e 100644 --- a/content/docs/guides/write-safely.mdx +++ b/content/docs/guides/write-safely.mdx @@ -199,7 +199,7 @@ const send = () => }); ``` -If Marfa already answered that request, it returns the first answer again, with the header `Idempotency-Replayed: true`, and writes nothing. A repeat of a request refused `409 version_conflict` gets the same `409`. A repeat of one that failed with a `5xx` runs again. +If Marfa already answered that request, it returns the first answer again, with the header `Idempotency-Replayed: true`, and writes nothing. A repeat of a request refused `409 version_conflict` gets the same `409`. A repeat of one that failed with a `5xx` runs again, and so does a repeat of one refused `404 blob_not_found` because a file item named bytes its writer hadn't uploaded: Marfa keeps no answer for that refusal, so after you upload the bytes, the same key works. A key names one request. The same key on a different request, such as the resolved write after a conflict, answers `422 idempotency_key_reused` and writes nothing, so **send each new write under a new key**. A repeat that arrives while the first is still running answers `409 idempotency_key_in_flight`: wait, then send it again. From bda302b1046e2a2096d129dd0bd64f870bd2722d Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:46:57 +0100 Subject: [PATCH 11/12] docs(changelog): add v0.0.14 --- content/docs/(reference)/changelog.mdx | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/content/docs/(reference)/changelog.mdx b/content/docs/(reference)/changelog.mdx index c4f7d4bb..e5aa6af6 100644 --- a/content/docs/(reference)/changelog.mdx +++ b/content/docs/(reference)/changelog.mdx @@ -3,10 +3,31 @@ title: Changelog description: What each release of Marfa changed for someone running an instance or building on it. --- -Each release of Marfa ships the server, the CLI for macOS on Apple silicon, and the TypeScript SDK on npm. Below is what each one changed that you'd notice when you run an instance or build on it, newest first, with a link to the full list on GitHub. +Each release of Marfa ships the server and its container image, the CLI for macOS on Apple silicon, and the TypeScript SDK on npm. Below is what each one changed that you'd notice when you run an instance or build on it, newest first, with a link to the full list on GitHub. **Until the first public release, Marfa does not upgrade a database in place.** A build refuses a database whose schema differs from its own or that holds retired tables or settings, and leaves the file unchanged. If the build refuses your database, [Upgrade](/guides/upgrade) covers exporting with the old build and restoring into fresh storage. A version change alone does not cause a refusal. +## v0.0.14 + +Released October 19, 2026. [v0.0.14 on GitHub](https://github.com/withmarfa/marfa/releases/tag/v0.0.14) + +**Before you update, if your container keeps its database in a bucket:** the replica moves from `db` at the bucket's root to `/db`. On its next start, an existing deployment streams its database to the new path and takes a full snapshot there at once. The old `db/` objects are no longer read or pruned, and you can delete them once the new replica exists. **Every instance on a shared bucket now needs its own `S3_PREFIX`.** Before this release, instances on one bucket shared one replica, and a new volume of one could start on another's database. [Back up and restore](/guides/back-up-and-restore#with-a-bucket) and [Deploy](/guides/deploy#an-object-store) explain the prefix. + +For running an instance: + +- **Each release publishes the server image** to `ghcr.io/withmarfa/marfa-server`, for `linux/amd64` and `linux/arm64`, tagged with the release and `latest`. [Deploy](/guides/deploy#run-it-in-a-container) pulls it, and [Upgrade](/guides/upgrade) names the old build by its tag. `GET /` answers the `release` of such a build beside `version`, and `marfa status` and `marfa whoami` show both. A build made from a clone has no `release`. The container's port is published on the loopback address only, because Docker publishes a bare port on every interface. +- **A proxy on the same machine as the container needs `TRUSTED_PROXY_CIDRS`** set to the container network's range. Without it, every caller shares one address, and sign-in lockouts, which count attempts for each address, apply to everyone together. [Deploy](/guides/deploy#the-address) explains it. +- **A webhook to a private address is refused when you register it.** A `url` whose host is a private address, or a name that resolves to one, such as a Tailscale name, is refused `400 validation_error` that names `MARFA_WEBHOOK_ALLOW_PRIVATE_ADDRESSES`, and so is a change of `url` to one. A delivery refused for its address is `dead_letter` at once and not retried, so redeliver it after you allow private addresses. An outbound webhook reaches only global IPv6 addresses, and other IPv6 blocks are refused. [Receive webhooks](/guides/receive-webhooks#receive-deliveries-on-a-private-network) covers it. +- **Telemetry names the instance.** Every log record, span and reported exception carries `service.instance.id`, the `instance_id` that `GET /` answers. `OTEL_RESOURCE_ATTRIBUTES` marks an instance, such as `deployment.environment=staging` for a test one, for the server and the CLI, and `OTEL_SERVICE_NAME` wins over a `service.name` in it. The bare-metal start in [Deploy](/guides/deploy#run-it-without-a-container) loads the telemetry preload. [Send telemetry](/guides/telemetry#tell-instances-apart) covers it. +- **A backup names the files it lacks.** An archive leaves out a blob that no store holds whole and names it under `missing_blobs` in its manifest. `marfa export` then exits `1` with `blobs_missing`, a restore answers `blobs_missing`, and `marfa restore` exits `1` when it lists any. A cut `marfa export --output` leaves the path as it was and exits `3`. A blob store that fails answers `503 blob_store_unavailable`. [Back up and restore](/guides/back-up-and-restore#when-an-archive-lacks-a-files-bytes) covers it. +- **Headers a Tailscale proxy adds, those named `Tailscale-`, are not stored** with a connector's inbound delivery. [Receive vendor webhooks](/guides/connector-webhooks) covers it. + +For apps and devices: + +- **A file item names only bytes its writer uploaded or can already read.** Any other write of `blob_ref` is refused `404 blob_not_found`, where it was taken and left a file that nobody could download. Upload with the same credential first. The refusal isn't kept against an `Idempotency-Key`, and a working copy uploads again under its current key when the key changed. [Handle files](/guides/handle-files#point-an-item-at-the-bytes) covers it. +- **`marfa folders push` says whether it finished.** It exits `3` when a write is left unsent or the catch-up fails, and `5` when the credential is refused. Under a rate limit it doesn't wait: it reports the writes as not delivered, names the wait and exits `3`, and a watch waits on its own. The report counts files with an unanswered write as waiting. `marfa folders restore` lists the files it couldn't write back. A file moved to another folder and saved again as UTF-16 stays the same item, and a file that isn't text no longer holds up deletes in other folders. [Sync a folder](/guides/sync-a-folder#know-whether-a-push-finished) covers it. +- **`marfa keys keep` keeps any valid key**, including one that holds only `keys.mint`, `instance.read` or `audit.read`. `marfa events` ends with exit `1` for `cursor_ahead` and `catchup_too_old`, `3` for a stream that stopped early, and `5` for `credential_ended`. `marfa owner passkeys add` opens the ordinary sign-in page. [The CLI](/cli#connect-to-a-server) and [Keep a copy in sync](/guides/keep-in-sync#when-the-stream-ends) cover them. + ## Swift SDK v0.0.21 Released October 18, 2026. [v0.0.21 on GitHub](https://github.com/withmarfa/marfa-swift/releases/tag/v0.0.21) From 10b7a8948e0837623ee6bc8350e813e2c9cc787d Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Mon, 19 Oct 2026 03:51:21 +0100 Subject: [PATCH 12/12] docs(guides): say what folders restore prints for a file it could not write --- content/docs/guides/sync-a-folder.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/guides/sync-a-folder.mdx b/content/docs/guides/sync-a-folder.mdx index 3756f866..d3fbafc7 100644 --- a/content/docs/guides/sync-a-folder.mdx +++ b/content/docs/guides/sync-a-folder.mdx @@ -253,7 +253,7 @@ if status.paused.isPaused { } ``` -`restore` counts only the files it wrote back. **A file it couldn't write**, such as one whose bytes it couldn't fetch, is listed with the flag the pull gave it, or `unwritten` where the pull gave none, and the command says so. Its item stays: the next push doesn't send it to the bin, and every later pull writes the file once the bytes can be fetched. +`restore` counts only the files it wrote back. **A file it couldn't write**, such as one whose bytes it couldn't fetch, is listed with the reason the pull gave, or "the pull did not write it" where it gave none, and the command says so. With `--json`, each file carries the pull's flag, or `unwritten` where the pull gave none. Its item stays: the next push doesn't send it to the bin, and every later pull writes the file once the bytes can be fetched. ```text 2 file(s) written back; 0 item(s) restored, sent at the next push; 1 file(s) not written back yet, which every pull tries again