From c377b2fa4e8dd014821e40612a7995cbe262e9ed Mon Sep 17 00:00:00 2001 From: Pablo Rodriguez Date: Thu, 24 Sep 2026 09:16:56 -0700 Subject: [PATCH] seal payloads that are already files without a base64 detour through JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HD1 v2 is JSON-only, so a PDF has to be base64'd into it at a 33% cost — the difference between a 24 MB upload cap and a 32 MB one. v3 is the same envelope over opaque bytes; isHD1 lets a store read plaintext and sealed objects side by side while it migrates between them. Co-Authored-By: Claude Opus 5 (1M context) --- ARCHITECTURE.md | 19 +++++++++++++ CHANGELOG.md | 7 +++++ crypto.ts | 63 ++++++++++++++++++++++++++++++++++---------- tests/crypto.test.ts | 62 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 14 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index df9ffc9..24a177f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -74,6 +74,25 @@ v2 separates *encrypting the data* from *granting access to it*: shared between principals, and revoking one principal's *future* access means deleting their wrapped-DEK row — it does not touch anyone else's. +### HD1 v3 — the same envelope over opaque bytes + +``` +MAGIC(3) | VERSION(1)=3 | vaultKeyId(16) | iv(12) | ciphertext +``` + +Identical to v2 in every respect except the payload: v2 carries UTF-8 JSON, v3 carries raw bytes. +It exists for payloads that are **already files** — a PDF, an image — which would otherwise have +to be base64'd into v2's JSON at a 33% cost. + +`encryptBytes`/`decryptBytes` are the v3 pair. `isHD1(blob)` reads the magic alone and says whether +an object is sealed at all, which is what a store migrating from plaintext to sealed objects needs +while both formats are live. + +The natural shape for per-file encryption is one random content key per file, kept inside a v2 +vault blob that the file's owner already controls. The file is then sealed under a key that only a +principal who can open that vault can recover, and re-keying the vault does not mean re-uploading +the files. + `vaultKeyId` is an opaque reference into the caller's own storage (a row ID, not key material) — resolving it to an actual wrapped DEK is the storage layer's job, not this module's. diff --git a/CHANGELOG.md b/CHANGELOG.md index a4d0400..1391136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ semver, but is pre-1.0 — expect breaking changes between minor versions until ### Added +- `crypto.ts` — HD1 **v3**, the same authenticated envelope over an opaque byte payload instead of + UTF-8 JSON, for sealing payloads that are already files. `encryptBytes(data, dek)` / + `decryptBytes(blob, dek)`, plus `isHD1(blob)` for a store that must read plaintext and sealed + objects side by side while migrating between them. Header, key handling and GCM tag are + identical to v2; only the version byte and the payload codec differ, so `MIN_BYTES = 32` and + magic checks stay valid across all three versions. + - `blob-store.ts` — `BlobStore`, a generic conditional-write interface for storing an opaque encrypted blob by key. Server-side counterpart to `vault-sink.ts`'s browser-side `VaultSink`. - `adapters/d1/` — `D1AccountStore`/`D1CredentialStore`/`D1EnvelopeStore`/`D1ProviderLinkStore`/ diff --git a/crypto.ts b/crypto.ts index 5b3858b..9fd9be0 100644 --- a/crypto.ts +++ b/crypto.ts @@ -58,6 +58,7 @@ export async function decryptVault>(blob: Uint8Array let o = MAGIC.length; const version = blob[o++]; if (version === VERSION_V2) throw new Error("v2 envelope blob: open it with decryptVaultV2 and a DEK"); + if (version === VERSION_BYTES) throw new Error("v3 bytes blob: open it with decryptBytes and a DEK"); if (version !== VERSION) throw new Error(`unsupported version ${version}`); const salt = blob.slice(o, o + SALT_LEN); o += SALT_LEN; const iv = blob.slice(o, o + IV_LEN); o += IV_LEN; @@ -80,6 +81,7 @@ export async function decryptVault>(blob: Uint8Array // re-wrap of the DEK rather than a shared secret. const VERSION_V2 = 2; +const VERSION_BYTES = 3; // same envelope, opaque byte payload instead of UTF-8 JSON const KEYID_LEN = 16; // opaque per-vault key id, reserved for future re-key tracking const EC_PARAMS = { name: "ECDH", namedCurve: "P-256" } as const; @@ -242,38 +244,71 @@ export async function unwrapDEKWithKek(blob: Uint8Array, kek: CryptoKey): Promis return subtle.importKey("raw", raw, { name: "AES-GCM", length: 256 }, true, ["encrypt", "decrypt"]); } -// HD1 v2 blob: magic(3) + version=2(1) + vaultKeyId(16) + iv(12) + AES-GCM(DEK, JSON). -// Header = 32 B, equal to v1, so the Functions' MIN_BYTES=32 + magic checks stay valid. -export async function encryptVaultV2>(data: T, dek: CryptoKey): Promise { +// HD1 v2/v3 blob: magic(3) + version(1) + vaultKeyId(16) + iv(12) + AES-GCM(DEK, payload). +// Header = 32 B, equal to v1, so the Functions' MIN_BYTES=32 + magic checks stay valid for all three. +// +// The two differ only in what the payload is: v2 carries UTF-8 JSON, v3 carries opaque bytes. A +// file routed through v2 would have to be base64'd into that JSON, which costs 33% — the difference +// between a 24 MB upload cap and a 32 MB one. The version byte is what tells a reader which it holds, +// so pack and unpack are written once here rather than per format. +const HD1_HEADER_LEN = MAGIC.length + 1 + KEYID_LEN + IV_LEN; + +async function sealHD1(version: number, plaintext: Uint8Array, dek: CryptoKey): Promise { const vaultKeyId = globalThis.crypto.getRandomValues(new Uint8Array(KEYID_LEN)); const iv = globalThis.crypto.getRandomValues(new Uint8Array(IV_LEN)); - const plaintext = enc.encode(JSON.stringify(data)); const ciphertext = new Uint8Array(await subtle.encrypt({ name: "AES-GCM", iv: toAB(iv) }, dek, toAB(plaintext))); - const out = new Uint8Array(MAGIC.length + 1 + KEYID_LEN + IV_LEN + ciphertext.length); + const out = new Uint8Array(HD1_HEADER_LEN + ciphertext.length); let o = 0; out.set(MAGIC, o); o += MAGIC.length; - out[o++] = VERSION_V2; + out[o++] = version; out.set(vaultKeyId, o); o += KEYID_LEN; out.set(iv, o); o += IV_LEN; out.set(ciphertext, o); return out; } -export async function decryptVaultV2>(blob: Uint8Array, dek: CryptoKey): Promise { - const header = MAGIC.length + 1 + KEYID_LEN + IV_LEN; - if (blob.length < header) throw new Error("blob too short"); +async function openHD1(blob: Uint8Array, version: number, label: string, dek: CryptoKey): Promise { + if (blob.length < HD1_HEADER_LEN) throw new Error("blob too short"); for (let i = 0; i < MAGIC.length; i++) if (blob[i] !== MAGIC[i]) throw new Error("not an HD1 blob"); let o = MAGIC.length; - const version = blob[o++]; - if (version !== VERSION_V2) throw new Error(`expected HD1 v2, got version ${version}`); + const got = blob[o++]; + if (got !== version) throw new Error(`expected HD1 ${label}, got version ${got}`); o += KEYID_LEN; // vaultKeyId — reserved/opaque const iv = blob.slice(o, o + IV_LEN); o += IV_LEN; const ciphertext = blob.slice(o); - let plaintext: ArrayBuffer; try { - plaintext = await subtle.decrypt({ name: "AES-GCM", iv: toAB(iv) }, dek, toAB(ciphertext)); + return new Uint8Array(await subtle.decrypt({ name: "AES-GCM", iv: toAB(iv) }, dek, toAB(ciphertext))); } catch { throw new Error("wrong DEK or corrupt blob"); } - return JSON.parse(dec.decode(plaintext)) as T; +} + +export async function encryptVaultV2>(data: T, dek: CryptoKey): Promise { + return sealHD1(VERSION_V2, enc.encode(JSON.stringify(data)), dek); +} + +export async function decryptVaultV2>(blob: Uint8Array, dek: CryptoKey): Promise { + return JSON.parse(dec.decode(await openHD1(blob, VERSION_V2, "v2", dek))) as T; +} + +// v3 — the same envelope over a payload that is already a file (a PDF, an image), so it is sealed +// as-is rather than base64'd through v2's JSON. Key handling and the GCM tag are identical to v2. +export async function encryptBytes(data: Uint8Array, dek: CryptoKey): Promise { + return sealHD1(VERSION_BYTES, data, dek); +} + +export async function decryptBytes(blob: Uint8Array, dek: CryptoKey): Promise { + return openHD1(blob, VERSION_BYTES, "v3", dek); +} + +/** + * Whether `blob` carries the HD1 magic — one of this package's envelopes rather than plaintext. + * + * For a store migrating from plaintext to sealed objects, where a reader must accept both until the + * last plaintext object is gone. Cheap and total: it reads the magic, never the key. + */ +export function isHD1(blob: Uint8Array): boolean { + if (blob.length < HD1_HEADER_LEN) return false; + for (let i = 0; i < MAGIC.length; i++) if (blob[i] !== MAGIC[i]) return false; + return true; } diff --git a/tests/crypto.test.ts b/tests/crypto.test.ts index c33fbe3..895d263 100644 --- a/tests/crypto.test.ts +++ b/tests/crypto.test.ts @@ -13,6 +13,9 @@ import { encryptVaultV2, decryptVaultV2, deriveAuthHash, + encryptBytes, + decryptBytes, + isHD1, } from "../crypto"; const rand = (n: number) => globalThis.crypto.getRandomValues(new Uint8Array(n)); @@ -183,3 +186,62 @@ describe("deriveAuthHash", () => { expect(hash).toMatch(/^[A-Za-z0-9_-]{40,44}$/); }); }); + +// ── v3: the same envelope over opaque bytes ────────────────────────────────── +// What the file-at-rest path needs from it: a PDF comes back byte-identical, a blob sealed to +// someone else's key does not open, and v2 and v3 are never mistaken for one another. +describe("HD1 v3 (bytes)", () => { + const pdf = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x37, ...rand(4096)]); + + it("round-trips arbitrary bytes unchanged", async () => { + const dek = await generateDEK(); + expect(await decryptBytes(await encryptBytes(pdf, dek), dek)).toEqual(pdf); + }); + + it("round-trips an empty payload", async () => { + const dek = await generateDEK(); + expect(await decryptBytes(await encryptBytes(new Uint8Array(0), dek), dek)).toEqual(new Uint8Array(0)); + }); + + it("writes an HD1 v3 header and hides the plaintext", async () => { + const blob = await encryptBytes(pdf, await generateDEK()); + expect([blob[0], blob[1], blob[2], blob[3]]).toEqual([0x48, 0x44, 0x31, 3]); + // The decisive property for at-rest storage: the stored object is not the document. + expect(blob.slice(32, 36)).not.toEqual(pdf.slice(0, 4)); + expect(blob.length).toBe(32 + pdf.length + 16); // header + ciphertext + GCM tag + }); + + it("refuses a wrong key rather than returning garbage", async () => { + const blob = await encryptBytes(pdf, await generateDEK()); + await expect(decryptBytes(blob, await generateDEK())).rejects.toThrow(/wrong DEK or corrupt blob/); + }); + + it("refuses a tampered payload (the GCM tag is load-bearing)", async () => { + const dek = await generateDEK(); + const blob = await encryptBytes(pdf, dek); + blob[40] ^= 0xff; + await expect(decryptBytes(blob, dek)).rejects.toThrow(/wrong DEK or corrupt blob/); + }); + + it("refuses a truncated blob and a non-HD1 blob", async () => { + const dek = await generateDEK(); + await expect(decryptBytes(new Uint8Array(20), dek)).rejects.toThrow(/blob too short/); + await expect(decryptBytes(new Uint8Array(64), dek)).rejects.toThrow(/not an HD1 blob/); + }); + + it("keeps v2 and v3 unconfusable in both directions", async () => { + const dek = await generateDEK(); + const v2 = await encryptVaultV2(sample, dek); + const v3 = await encryptBytes(pdf, dek); + await expect(decryptBytes(v2, dek)).rejects.toThrow(/expected HD1 v3, got version 2/); + await expect(decryptVaultV2(v3, dek)).rejects.toThrow(/expected HD1 v2, got version 3/); + await expect(decryptVault(v3, "pw")).rejects.toThrow(/v3 bytes blob/); + }); + + it("isHD1 separates a sealed object from plaintext, for a store mid-migration", async () => { + expect(isHD1(await encryptBytes(pdf, await generateDEK()))).toBe(true); + expect(isHD1(await encryptVaultV2(sample, await generateDEK()))).toBe(true); + expect(isHD1(pdf)).toBe(false); + expect(isHD1(new Uint8Array(4))).toBe(false); // shorter than a header + }); +});