Add HD1 v3: the same envelope over opaque bytes

ARCHITECTURE.md

@@ -74,6 +74,25 @@ v2 separates *encrypting the data* from *granting access to it*:
    shared between principals, and revoking one principal's *future* access means deleting their
    wrapped-DEK row — it does not touch anyone else's.
 
+### HD1 v3 — the same envelope over opaque bytes
+
+```
+MAGIC(3) | VERSION(1)=3 | vaultKeyId(16) | iv(12) | ciphertext
+```
+
+Identical to v2 in every respect except the payload: v2 carries UTF-8 JSON, v3 carries raw bytes.
+It exists for payloads that are **already files** — a PDF, an image — which would otherwise have
+to be base64'd into v2's JSON at a 33% cost.
+
+`encryptBytes`/`decryptBytes` are the v3 pair. `isHD1(blob)` reads the magic alone and says whether
+an object is sealed at all, which is what a store migrating from plaintext to sealed objects needs
+while both formats are live.
+
+The natural shape for per-file encryption is one random content key per file, kept inside a v2
+vault blob that the file's owner already controls. The file is then sealed under a key that only a
+principal who can open that vault can recover, and re-keying the vault does not mean re-uploading
+the files.
+
 `vaultKeyId` is an opaque reference into the caller's own storage (a row ID, not key material)
 — resolving it to an actual wrapped DEK is the storage layer's job, not this module's.
 

CHANGELOG.md

@@ -7,6 +7,13 @@ semver, but is pre-1.0 — expect breaking changes between minor versions until
 
 ### Added
 
+- `crypto.ts` — HD1 **v3**, the same authenticated envelope over an opaque byte payload instead of
+  UTF-8 JSON, for sealing payloads that are already files. `encryptBytes(data, dek)` /
+  `decryptBytes(blob, dek)`, plus `isHD1(blob)` for a store that must read plaintext and sealed
+  objects side by side while migrating between them. Header, key handling and GCM tag are
+  identical to v2; only the version byte and the payload codec differ, so `MIN_BYTES = 32` and
+  magic checks stay valid across all three versions.
+
 - `blob-store.ts` — `BlobStore`, a generic conditional-write interface for storing an opaque
   encrypted blob by key. Server-side counterpart to `vault-sink.ts`'s browser-side `VaultSink`.
 - `adapters/d1/` — `D1AccountStore`/`D1CredentialStore`/`D1EnvelopeStore`/`D1ProviderLinkStore`/

crypto.ts

@@ -58,6 +58,7 @@ export async function decryptVault<T = Record<string, unknown>>(blob: Uint8Array
   let o = MAGIC.length;
   const version = blob[o++];
   if (version === VERSION_V2) throw new Error("v2 envelope blob: open it with decryptVaultV2 and a DEK");
+  if (version === VERSION_BYTES) throw new Error("v3 bytes blob: open it with decryptBytes and a DEK");
   if (version !== VERSION) throw new Error(`unsupported version ${version}`);
   const salt = blob.slice(o, o + SALT_LEN); o += SALT_LEN;
   const iv = blob.slice(o, o + IV_LEN); o += IV_LEN;
@@ -80,6 +81,7 @@ export async function decryptVault<T = Record<string, unknown>>(blob: Uint8Array
 // re-wrap of the DEK rather than a shared secret.
 
 const VERSION_V2 = 2;
+const VERSION_BYTES = 3; // same envelope, opaque byte payload instead of UTF-8 JSON
 const KEYID_LEN = 16; // opaque per-vault key id, reserved for future re-key tracking
 const EC_PARAMS = { name: "ECDH", namedCurve: "P-256" } as const;
 
@@ -242,38 +244,71 @@ export async function unwrapDEKWithKek(blob: Uint8Array, kek: CryptoKey): Promis
   return subtle.importKey("raw", raw, { name: "AES-GCM", length: 256 }, true, ["encrypt", "decrypt"]);
 }
 
-// HD1 v2 blob: magic(3) + version=2(1) + vaultKeyId(16) + iv(12) + AES-GCM(DEK, JSON).
-// Header = 32 B, equal to v1, so the Functions' MIN_BYTES=32 + magic checks stay valid.
-export async function encryptVaultV2<T = Record<string, unknown>>(data: T, dek: CryptoKey): Promise<Uint8Array> {
+// HD1 v2/v3 blob: magic(3) + version(1) + vaultKeyId(16) + iv(12) + AES-GCM(DEK, payload).
+// Header = 32 B, equal to v1, so the Functions' MIN_BYTES=32 + magic checks stay valid for all three.
+//
+// The two differ only in what the payload is: v2 carries UTF-8 JSON, v3 carries opaque bytes. A
+// file routed through v2 would have to be base64'd into that JSON, which costs 33% — the difference
+// between a 24 MB upload cap and a 32 MB one. The version byte is what tells a reader which it holds,
+// so pack and unpack are written once here rather than per format.
+const HD1_HEADER_LEN = MAGIC.length + 1 + KEYID_LEN + IV_LEN;
+
+async function sealHD1(version: number, plaintext: Uint8Array, dek: CryptoKey): Promise<Uint8Array> {
   const vaultKeyId = globalThis.crypto.getRandomValues(new Uint8Array(KEYID_LEN));
   const iv = globalThis.crypto.getRandomValues(new Uint8Array(IV_LEN));
-  const plaintext = enc.encode(JSON.stringify(data));
   const ciphertext = new Uint8Array(await subtle.encrypt({ name: "AES-GCM", iv: toAB(iv) }, dek, toAB(plaintext)));
-  const out = new Uint8Array(MAGIC.length + 1 + KEYID_LEN + IV_LEN + ciphertext.length);
+  const out = new Uint8Array(HD1_HEADER_LEN + ciphertext.length);
   let o = 0;
   out.set(MAGIC, o); o += MAGIC.length;
-  out[o++] = VERSION_V2;
+  out[o++] = version;
   out.set(vaultKeyId, o); o += KEYID_LEN;
   out.set(iv, o); o += IV_LEN;
   out.set(ciphertext, o);
   return out;
 }
 
-export async function decryptVaultV2<T = Record<string, unknown>>(blob: Uint8Array, dek: CryptoKey): Promise<T> {
-  const header = MAGIC.length + 1 + KEYID_LEN + IV_LEN;
-  if (blob.length < header) throw new Error("blob too short");
+async function openHD1(blob: Uint8Array, version: number, label: string, dek: CryptoKey): Promise<Uint8Array> {
+  if (blob.length < HD1_HEADER_LEN) throw new Error("blob too short");
   for (let i = 0; i < MAGIC.length; i++) if (blob[i] !== MAGIC[i]) throw new Error("not an HD1 blob");
   let o = MAGIC.length;
-  const version = blob[o++];
-  if (version !== VERSION_V2) throw new Error(`expected HD1 v2, got version ${version}`);
+  const got = blob[o++];
+  if (got !== version) throw new Error(`expected HD1 ${label}, got version ${got}`);
   o += KEYID_LEN; // vaultKeyId — reserved/opaque
   const iv = blob.slice(o, o + IV_LEN); o += IV_LEN;
   const ciphertext = blob.slice(o);
-  let plaintext: ArrayBuffer;
   try {
-    plaintext = await subtle.decrypt({ name: "AES-GCM", iv: toAB(iv) }, dek, toAB(ciphertext));
+    return new Uint8Array(await subtle.decrypt({ name: "AES-GCM", iv: toAB(iv) }, dek, toAB(ciphertext)));
   } catch {
     throw new Error("wrong DEK or corrupt blob");
   }
-  return JSON.parse(dec.decode(plaintext)) as T;
+}
+
+export async function encryptVaultV2<T = Record<string, unknown>>(data: T, dek: CryptoKey): Promise<Uint8Array> {
+  return sealHD1(VERSION_V2, enc.encode(JSON.stringify(data)), dek);
+}
+
+export async function decryptVaultV2<T = Record<string, unknown>>(blob: Uint8Array, dek: CryptoKey): Promise<T> {
+  return JSON.parse(dec.decode(await openHD1(blob, VERSION_V2, "v2", dek))) as T;
+}
+
+// v3 — the same envelope over a payload that is already a file (a PDF, an image), so it is sealed
+// as-is rather than base64'd through v2's JSON. Key handling and the GCM tag are identical to v2.
+export async function encryptBytes(data: Uint8Array, dek: CryptoKey): Promise<Uint8Array> {
+  return sealHD1(VERSION_BYTES, data, dek);
+}
+
+export async function decryptBytes(blob: Uint8Array, dek: CryptoKey): Promise<Uint8Array> {
+  return openHD1(blob, VERSION_BYTES, "v3", dek);
+}
+
+/**
+ * Whether `blob` carries the HD1 magic — one of this package's envelopes rather than plaintext.
+ *
+ * For a store migrating from plaintext to sealed objects, where a reader must accept both until the
+ * last plaintext object is gone. Cheap and total: it reads the magic, never the key.
+ */
+export function isHD1(blob: Uint8Array): boolean {
+  if (blob.length < HD1_HEADER_LEN) return false;
+  for (let i = 0; i < MAGIC.length; i++) if (blob[i] !== MAGIC[i]) return false;
+  return true;
 }

tests/crypto.test.ts

@@ -13,6 +13,9 @@ import {
   encryptVaultV2,
   decryptVaultV2,
   deriveAuthHash,
+  encryptBytes,
+  decryptBytes,
+  isHD1,
 } from "../crypto";
 
 const rand = (n: number) => globalThis.crypto.getRandomValues(new Uint8Array(n));
@@ -183,3 +186,62 @@ describe("deriveAuthHash", () => {
     expect(hash).toMatch(/^[A-Za-z0-9_-]{40,44}$/);
   });
 });
+
+// ── v3: the same envelope over opaque bytes ──────────────────────────────────
+// What the file-at-rest path needs from it: a PDF comes back byte-identical, a blob sealed to
+// someone else's key does not open, and v2 and v3 are never mistaken for one another.
+describe("HD1 v3 (bytes)", () => {
+  const pdf = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x37, ...rand(4096)]);
+
+  it("round-trips arbitrary bytes unchanged", async () => {
+    const dek = await generateDEK();
+    expect(await decryptBytes(await encryptBytes(pdf, dek), dek)).toEqual(pdf);
+  });
+
+  it("round-trips an empty payload", async () => {
+    const dek = await generateDEK();
+    expect(await decryptBytes(await encryptBytes(new Uint8Array(0), dek), dek)).toEqual(new Uint8Array(0));
+  });
+
+  it("writes an HD1 v3 header and hides the plaintext", async () => {
+    const blob = await encryptBytes(pdf, await generateDEK());
+    expect([blob[0], blob[1], blob[2], blob[3]]).toEqual([0x48, 0x44, 0x31, 3]);
+    // The decisive property for at-rest storage: the stored object is not the document.
+    expect(blob.slice(32, 36)).not.toEqual(pdf.slice(0, 4));
+    expect(blob.length).toBe(32 + pdf.length + 16); // header + ciphertext + GCM tag
+  });
+
+  it("refuses a wrong key rather than returning garbage", async () => {
+    const blob = await encryptBytes(pdf, await generateDEK());
+    await expect(decryptBytes(blob, await generateDEK())).rejects.toThrow(/wrong DEK or corrupt blob/);
+  });
+
+  it("refuses a tampered payload (the GCM tag is load-bearing)", async () => {
+    const dek = await generateDEK();
+    const blob = await encryptBytes(pdf, dek);
+    blob[40] ^= 0xff;
+    await expect(decryptBytes(blob, dek)).rejects.toThrow(/wrong DEK or corrupt blob/);
+  });
+
+  it("refuses a truncated blob and a non-HD1 blob", async () => {
+    const dek = await generateDEK();
+    await expect(decryptBytes(new Uint8Array(20), dek)).rejects.toThrow(/blob too short/);
+    await expect(decryptBytes(new Uint8Array(64), dek)).rejects.toThrow(/not an HD1 blob/);
+  });
+
+  it("keeps v2 and v3 unconfusable in both directions", async () => {
+    const dek = await generateDEK();
+    const v2 = await encryptVaultV2(sample, dek);
+    const v3 = await encryptBytes(pdf, dek);
+    await expect(decryptBytes(v2, dek)).rejects.toThrow(/expected HD1 v3, got version 2/);
+    await expect(decryptVaultV2(v3, dek)).rejects.toThrow(/expected HD1 v2, got version 3/);
+    await expect(decryptVault(v3, "pw")).rejects.toThrow(/v3 bytes blob/);
+  });
+
+  it("isHD1 separates a sealed object from plaintext, for a store mid-migration", async () => {
+    expect(isHD1(await encryptBytes(pdf, await generateDEK()))).toBe(true);
+    expect(isHD1(await encryptVaultV2(sample, await generateDEK()))).toBe(true);
+    expect(isHD1(pdf)).toBe(false);
+    expect(isHD1(new Uint8Array(4))).toBe(false); // shorter than a header
+  });
+});