From d56a0cfbc3d99b6c781f9cb892a447c30171933b Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:08:19 -0700 Subject: [PATCH 1/6] fix(relay): tunnel limit counts abandoned reprovisions and frees failed revokes The live-tunnel branch of the quota count ignored any withdrawn link, so a revoked environment that was reprovisioned but never had its link rewritten held a Cloudflare tunnel while counting as zero. Bound it the same way as the pending branch: a link withdrawn before the allocation's last provisioning write no longer hides it. Teardown claims (claimDeprovision, unmarked claimRelease) no longer move updatedAt, so a revoke whose Cloudflare cleanup fails stays excluded instead of being counted again on every retry. Co-Authored-By: Claude Opus 5.5 --- infra/relay/package.json | 1 + .../ManagedEndpointAllocations.test.ts | 27 +++ .../ManagedEndpointAllocations.ts | 9 +- .../environments/ManagedTunnelLimits.test.ts | 163 +++++++++++++++--- .../src/environments/ManagedTunnelLimits.ts | 97 ++++++----- pnpm-lock.yaml | 6 +- 6 files changed, 225 insertions(+), 78 deletions(-) diff --git a/infra/relay/package.json b/infra/relay/package.json index c75e4b70648d2..197c37819981e 100644 --- a/infra/relay/package.json +++ b/infra/relay/package.json @@ -33,6 +33,7 @@ "@cloudflare/workers-types": "^4.20260601.1", "@effect/platform-node": "catalog:", "@effect/vitest": "catalog:", + "@electric-sql/pglite": "0.3.15", "@types/node": "catalog:", "drizzle-kit": "1.0.0-rc.5-ab785fc", "typescript": "catalog:", diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index 931e8a2b8893d..f0cd86bef9669 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -74,4 +74,31 @@ describe("ManagedEndpointAllocations", () => { ]); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); + + it.effect("teardown claims leave updatedAt to provisioning writes", () => { + // The tunnel limit compares link and allocation updatedAt; a claim that bumps it makes a + // failed revoke cleanup look like a reprovision and keeps holding the user's slot. + const writes: Array> = []; + const fakeDb = { + update: () => ({ + set: (values: Record) => { + writes.push(values); + return { + where: () => ({ returning: () => Effect.succeed([{ generation: 2 }]) }), + }; + }, + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const key = { userId: "user-1", environmentId: "environment-1", generation: 1 }; + yield* allocations.claimDeprovision(key); + yield* allocations.claimRelease({ ...key, tunnelId: "tunnel-1" }); + yield* allocations.claimRelease({ ...key, tunnelId: "tunnel-1", markReleased: true }); + + expect(writes.map((values) => "updatedAt" in values)).toEqual([false, false, true]); + expect(writes[2]?.["updatedAt"]).toBe(writes[2]?.["tunnelReleasedAt"]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); }); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index d432f45c61763..918c410e35176 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -512,9 +512,8 @@ const make = Effect.gen(function* () { const claimed = yield* db .update(relayManagedEndpointAllocations) .set({ - updatedAt: now, generation: sql`${relayManagedEndpointAllocations.generation} + 1`, - ...(input.markReleased === true ? { tunnelReleasedAt: now } : {}), + ...(input.markReleased === true ? { tunnelReleasedAt: now, updatedAt: now } : {}), }) .where( and( @@ -585,10 +584,8 @@ const make = Effect.gen(function* () { ) { const claimed = yield* db .update(relayManagedEndpointAllocations) - .set({ - updatedAt: DateTime.formatIso(yield* DateTime.now), - generation: sql`${relayManagedEndpointAllocations.generation} + 1`, - }) + // Claims move only the generation: the tunnel limit reads updatedAt as the last provisioning write. + .set({ generation: sql`${relayManagedEndpointAllocations.generation} + 1` }) .where( and( whereAllocation(input), diff --git a/infra/relay/src/environments/ManagedTunnelLimits.test.ts b/infra/relay/src/environments/ManagedTunnelLimits.test.ts index 9879e38f18a37..dca75e527dd83 100644 --- a/infra/relay/src/environments/ManagedTunnelLimits.test.ts +++ b/infra/relay/src/environments/ManagedTunnelLimits.test.ts @@ -3,7 +3,8 @@ import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Semaphore from "effect/Semaphore"; -import { PgDialect } from "drizzle-orm/pg-core"; +import { PGlite } from "@electric-sql/pglite"; +import { drizzle } from "drizzle-orm/pglite"; import * as RelayDb from "../db.ts"; import { relayManagedEndpointAllocations } from "../persistence/schema.ts"; @@ -14,7 +15,7 @@ class FakeTransaction extends Context.Service, countConditions: Array = []) { +function makeFakeDb(reserved: ReadonlyArray) { const userLock = Semaphore.makeUnsafe(1); return { $client: { @@ -40,11 +41,10 @@ function makeFakeDb(reserved: ReadonlyArray, countConditions: Array ({ from: (table: unknown) => ({ - where: (condition: unknown) => { + where: () => { if (table !== relayManagedEndpointAllocations) { return { limit: () => Effect.succeed([]) }; } - countConditions.push(condition); return Effect.yieldNow.pipe(Effect.as([{ activeTunnels: reserved.length }])); }, }), @@ -87,29 +87,140 @@ describe("ManagedTunnelLimits", () => { ); }); - it.effect("pending reservations stop counting once released or withdrawn", () => { - const countConditions: Array = []; - - return Effect.gen(function* () { - const limits = yield* ManagedTunnelLimits.ManagedTunnelLimits; - yield* limits.withCapacity({ userId: "user-1", environmentId: "environment-1" }, Effect.void); - - expect(countConditions).toHaveLength(1); - const query = new PgDialect().sqlToQuery(countConditions[0] as never); - // claimRelease writes updatedAt = tunnelReleasedAt; a later reserve moves updatedAt past it. - expect(query.sql).toContain( - '"relay_managed_endpoint_allocations"."updated_at" > "relay_managed_endpoint_allocations"."tunnel_released_at"', + it.effect("counts each allocation by its tunnel and link state", () => + Effect.gen(function* () { + const pg = yield* Effect.acquireRelease( + Effect.promise(() => PGlite.create()), + (client) => Effect.promise(() => client.close()), ); - // A link revoked or made publish-only after the last reservation no longer holds a slot. - expect(query.sql).toContain( - '"relay_environment_links"."updated_at" >= "relay_managed_endpoint_allocations"."updated_at"', + yield* Effect.promise(() => + pg.exec(` + create table relay_managed_endpoint_allocations ( + user_id text not null, environment_id text not null, tunnel_id text, + tunnel_released_at text, updated_at text not null + ); + create table relay_environment_links ( + user_id text not null, environment_id text not null, revoked_at text, + managed_tunnels_enabled boolean not null, updated_at text not null + ); + `), ); - }).pipe( - Effect.provide( - ManagedTunnelLimits.layer.pipe( - Layer.provide(Layer.succeed(RelayDb.RelayDb, makeFakeDb([], countConditions))), + const at = (minutes: number) => `2026-10-27T12:${String(minutes).padStart(2, "0")}:00.000Z`; + const cases: ReadonlyArray<{ + readonly environmentId: string; + readonly counted: boolean; + readonly allocation: { + readonly tunnelId?: string; + readonly releasedAt?: number; + readonly updatedAt: number; + }; + readonly link?: { + readonly revoked?: boolean; + readonly publishOnly?: boolean; + readonly updatedAt: number; + }; + }> = [ + { environmentId: "pending", counted: true, allocation: { updatedAt: 50 } }, + { environmentId: "abandoned-reservation", counted: false, allocation: { updatedAt: 10 } }, + { + environmentId: "live", + counted: true, + allocation: { tunnelId: "t", updatedAt: 10 }, + link: { updatedAt: 12 }, + }, + { + environmentId: "released", + counted: false, + allocation: { tunnelId: "t", releasedAt: 20, updatedAt: 20 }, + }, + { + environmentId: "re-reserved-after-release", + counted: true, + allocation: { tunnelId: "t", releasedAt: 20, updatedAt: 50 }, + }, + { + environmentId: "revoked", + counted: false, + allocation: { tunnelId: "t", updatedAt: 10 }, + link: { revoked: true, updatedAt: 20 }, + }, + { + environmentId: "revoked-pending", + counted: false, + allocation: { updatedAt: 50 }, + link: { revoked: true, updatedAt: 55 }, + }, + { + environmentId: "publish-only", + counted: false, + allocation: { tunnelId: "t", updatedAt: 10 }, + link: { publishOnly: true, updatedAt: 20 }, + }, + // Reprovisioned after a revoke, but the Worker died before rewriting the link. + { + environmentId: "reprovision-before-link", + counted: true, + allocation: { tunnelId: "t", updatedAt: 20 }, + link: { revoked: true, updatedAt: 10 }, + }, + { + environmentId: "reserved-before-link-rewrite", + counted: true, + allocation: { updatedAt: 50 }, + link: { revoked: true, updatedAt: 48 }, + }, + { environmentId: "self", counted: false, allocation: { tunnelId: "t", updatedAt: 10 } }, + ]; + for (const entry of cases) { + yield* Effect.promise(() => + pg.query( + "insert into relay_managed_endpoint_allocations values ('user-1', $1, $2, $3, $4)", + [ + entry.environmentId, + entry.allocation.tunnelId ?? null, + entry.allocation.releasedAt === undefined ? null : at(entry.allocation.releasedAt), + at(entry.allocation.updatedAt), + ], + ), + ); + if (entry.link !== undefined) { + const link = entry.link; + yield* Effect.promise(() => + pg.query("insert into relay_environment_links values ('user-1', $1, $2, $3, $4)", [ + entry.environmentId, + link.revoked === true ? at(link.updatedAt) : null, + link.publishOnly !== true, + at(link.updatedAt), + ]), + ); + } + } + yield* Effect.promise(() => + pg.query( + "insert into relay_managed_endpoint_allocations values ('user-2', 'other-user', 't', null, $1)", + [at(10)], ), - ), - ); - }); + ); + + const counted = yield* Effect.promise(() => + drizzle({ client: pg }) + .select({ environmentId: relayManagedEndpointAllocations.environmentId }) + .from(relayManagedEndpointAllocations) + .where( + ManagedTunnelLimits.countedAllocations({ + userId: "user-1", + environmentId: "self", + pendingCutoff: at(45), + }), + ), + ); + + expect(counted.map((row) => row.environmentId).toSorted()).toEqual( + cases + .filter((entry) => entry.counted) + .map((entry) => entry.environmentId) + .toSorted(), + ); + }), + ); }); diff --git a/infra/relay/src/environments/ManagedTunnelLimits.ts b/infra/relay/src/environments/ManagedTunnelLimits.ts index 15c50ae7efb1f..d4ddbb5cddf84 100644 --- a/infra/relay/src/environments/ManagedTunnelLimits.ts +++ b/infra/relay/src/environments/ManagedTunnelLimits.ts @@ -45,6 +45,57 @@ export class ManagedTunnelLimitExceeded extends Schema.TaggedError + exists( + new QueryBuilder() + .select({ userId: relayEnvironmentLinks.userId }) + .from(relayEnvironmentLinks) + .where( + and( + eq(relayEnvironmentLinks.userId, allocation.userId), + eq(relayEnvironmentLinks.environmentId, allocation.environmentId), + or( + isNotNull(relayEnvironmentLinks.revokedAt), + eq(relayEnvironmentLinks.managedTunnelsEnabled, false), + ), + gte(relayEnvironmentLinks.updatedAt, since), + ), + ), + ); + +// Counts allocations, not links: a link is written only after its tunnel exists. +// Excludes this environment so relinking at the limit stays idempotent. +// updatedAt moves only on provisioning writes and release, never on teardown claims, so a link +// withdrawn at or after it means cleanup is owed; an older withdrawal means a relink that never finished. +export const countedAllocations = (input: { + readonly userId: string; + readonly environmentId: string; + readonly pendingCutoff: string; +}) => + and( + eq(allocation.userId, input.userId), + ne(allocation.environmentId, input.environmentId), + or( + // Reserved but not yet holding a tunnel. Release writes updatedAt = tunnelReleasedAt and + // a relink reserves before rewriting its link, so only a newer reservation counts here. + and( + gt(allocation.updatedAt, input.pendingCutoff), + or( + isNull(allocation.tunnelReleasedAt), + gt(allocation.updatedAt, allocation.tunnelReleasedAt), + ), + not(withdrawnLink(allocation.updatedAt)), + ), + and( + isNotNull(allocation.tunnelId), + isNull(allocation.tunnelReleasedAt), + not(withdrawnLink(allocation.updatedAt)), + ), + ), + ); + export class ManagedTunnelLimits extends Context.Service< ManagedTunnelLimits, { @@ -103,52 +154,10 @@ const make = Effect.gen(function* () { minutes: MANAGED_TUNNEL_PENDING_WINDOW_MINUTES, }), ); - // Counts allocations, not links: a link is written only after its tunnel exists. - // Excludes this environment so relinking at the limit stays idempotent. - const allocation = relayManagedEndpointAllocations; - const withdrawnLink = (since?: typeof allocation.updatedAt) => - exists( - new QueryBuilder() - .select({ userId: relayEnvironmentLinks.userId }) - .from(relayEnvironmentLinks) - .where( - and( - eq(relayEnvironmentLinks.userId, allocation.userId), - eq(relayEnvironmentLinks.environmentId, allocation.environmentId), - or( - isNotNull(relayEnvironmentLinks.revokedAt), - eq(relayEnvironmentLinks.managedTunnelsEnabled, false), - ), - since === undefined ? undefined : gte(relayEnvironmentLinks.updatedAt, since), - ), - ), - ); const counted = yield* db .select({ activeTunnels: count() }) - .from(allocation) - .where( - and( - eq(allocation.userId, input.userId), - ne(allocation.environmentId, input.environmentId), - or( - // Reserved but not yet holding a tunnel. Release writes updatedAt = tunnelReleasedAt and - // a relink reserves before rewriting its link, so only a newer reservation counts here. - and( - gt(allocation.updatedAt, pendingCutoff), - or( - isNull(allocation.tunnelReleasedAt), - gt(allocation.updatedAt, allocation.tunnelReleasedAt), - ), - not(withdrawnLink(allocation.updatedAt)), - ), - and( - isNotNull(allocation.tunnelId), - isNull(allocation.tunnelReleasedAt), - not(withdrawnLink()), - ), - ), - ), - ) + .from(relayManagedEndpointAllocations) + .where(countedAllocations({ ...input, pendingCutoff })) .pipe( Effect.mapError( (cause) => diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0adcb65c50062..150b869ef7e3d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1354,6 +1354,9 @@ importers: '@effect/vitest': specifier: 4.0.5 version: 4.0.5(patch_hash=2731c9c3761c595f25ddc625a7d61ca376b389d5b3049ecb837c8940f21cfa02)(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(effect@4.0.5)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) + '@electric-sql/pglite': + specifier: 0.3.15 + version: 0.3.15 '@types/node': specifier: 24.12.4 version: 24.12.4 @@ -13573,8 +13576,7 @@ snapshots: - vue-tsc - yaml - '@electric-sql/pglite@0.3.15': - optional: true + '@electric-sql/pglite@0.3.15': {} '@electron-internal/extract-zip@1.0.5': {} From 93dd2a27fef46b44dfce8c8340f3f75184bb6b8d Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:28:34 -0700 Subject: [PATCH 2/6] fix(relay): backfill pre-deploy failed deprovisions so they stop holding a slot Before this branch, claimDeprovision bumped updated_at past the revoke. A failed Cloudflare delete therefore left rows that the new live-branch bound reads as an unfinished relink, which would hold the slot forever. A one-time migration sets those rows' updated_at to their link's updated_at. The count test now runs on the real migrated schema, covers pre-deploy rows, and adds a tie case that pins gte. Co-Authored-By: Claude Opus 5.5 --- .../migration.sql | 12 + .../snapshot.json | 1571 +++++++++++++++++ .../environments/ManagedTunnelLimits.test.ts | 121 +- 3 files changed, 1666 insertions(+), 38 deletions(-) create mode 100644 infra/relay/migrations/postgres/20261028062605_backfill_withdrawn_tunnel_allocations/migration.sql create mode 100644 infra/relay/migrations/postgres/20261028062605_backfill_withdrawn_tunnel_allocations/snapshot.json diff --git a/infra/relay/migrations/postgres/20261028062605_backfill_withdrawn_tunnel_allocations/migration.sql b/infra/relay/migrations/postgres/20261028062605_backfill_withdrawn_tunnel_allocations/migration.sql new file mode 100644 index 0000000000000..4c822889bd98e --- /dev/null +++ b/infra/relay/migrations/postgres/20261028062605_backfill_withdrawn_tunnel_allocations/migration.sql @@ -0,0 +1,12 @@ +-- Teardown claims used to bump updated_at past the link's revoke or publish-only write, so a +-- failed deprovision now reads as an unfinished relink. Give those rows the updated_at the +-- current write rule would have left. Touches only withdrawn links' live tunnels; idempotent. +UPDATE "relay_managed_endpoint_allocations" AS a +SET "updated_at" = l."updated_at" +FROM "relay_environment_links" AS l +WHERE l."user_id" = a."user_id" + AND l."environment_id" = a."environment_id" + AND a."tunnel_id" IS NOT NULL + AND a."tunnel_released_at" IS NULL + AND (l."revoked_at" IS NOT NULL OR l."managed_tunnels_enabled" = false) + AND l."updated_at" < a."updated_at"; diff --git a/infra/relay/migrations/postgres/20261028062605_backfill_withdrawn_tunnel_allocations/snapshot.json b/infra/relay/migrations/postgres/20261028062605_backfill_withdrawn_tunnel_allocations/snapshot.json new file mode 100644 index 0000000000000..e02f3a0c6f2e7 --- /dev/null +++ b/infra/relay/migrations/postgres/20261028062605_backfill_withdrawn_tunnel_allocations/snapshot.json @@ -0,0 +1,1571 @@ +{ + "id": "4413755a-63f1-4f93-bdc9-d1357283c23e", + "prevIds": ["7d8c1702-6f47-4453-a8ee-66ce5e3bfb41"], + "version": "8", + "dialect": "postgres", + "ddl": [ + { + "isRlsEnabled": false, + "name": "relay_agent_activity_rows", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_delivery_attempts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_dpop_proofs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_credentials", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_links", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_live_activities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_endpoint_allocations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_tunnel_limits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_mobile_devices", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_upstream_api_pauses", + "entityType": "tables", + "schema": "public" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state_json", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(36)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_job_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_suffix", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_status", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_reason", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "transport_error", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thumbprint", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "jti", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "iat", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_hash", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'T3 Environment'", + "generated": null, + "identity": null, + "name": "environment_label", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_http_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_ws_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(32)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_provider_kind", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notifications_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "live_activities_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "managed_tunnels_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "hold_webhooks_while_offline", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activity_push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_start_queued_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_started_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_aggregate_json", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_live_activity_delivery_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "hostname", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_name", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dns_record_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ready_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_enabled_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_released_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "generation", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "max_tunnels", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'iOS device'", + "generated": null, + "identity": null, + "name": "label", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "platform", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ios_major_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "android_api_level", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "bundle_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "aps_environment", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_to_start_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "preferences_json", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "api", + "entityType": "columns", + "schema": "public", + "table": "relay_upstream_api_pauses" + }, + { + "type": "bigint", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "paused_until_ms", + "entityType": "columns", + "schema": "public", + "table": "relay_upstream_api_pauses" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_created", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_source_job", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_dpop_proofs_expires_at", + "entityType": "indexes", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "credential_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_hash", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "environment_public_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment_key", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_links_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_links" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "activity_push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_live_activities_activity_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "hostname", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_hostname", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "tunnel_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_tunnel_name", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_to_start_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_to_start_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["environment_id", "environment_public_key", "thread_id"], + "nameExplicit": false, + "name": "relay_agent_activity_rows_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "columns": ["thumbprint", "jti"], + "nameExplicit": false, + "name": "relay_dpop_proofs_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_environment_links_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_environment_links" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_live_activities_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_live_activities" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_managed_endpoint_allocations_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_mobile_devices_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "relay_delivery_attempts_pkey", + "schema": "public", + "table": "relay_delivery_attempts", + "entityType": "pks" + }, + { + "columns": ["credential_id"], + "nameExplicit": false, + "name": "relay_environment_credentials_pkey", + "schema": "public", + "table": "relay_environment_credentials", + "entityType": "pks" + }, + { + "columns": ["user_id"], + "nameExplicit": false, + "name": "relay_managed_tunnel_limits_pkey", + "schema": "public", + "table": "relay_managed_tunnel_limits", + "entityType": "pks" + }, + { + "columns": ["api"], + "nameExplicit": false, + "name": "relay_upstream_api_pauses_pkey", + "schema": "public", + "table": "relay_upstream_api_pauses", + "entityType": "pks" + } + ], + "renames": [] +} diff --git a/infra/relay/src/environments/ManagedTunnelLimits.test.ts b/infra/relay/src/environments/ManagedTunnelLimits.test.ts index dca75e527dd83..301aa1229204c 100644 --- a/infra/relay/src/environments/ManagedTunnelLimits.test.ts +++ b/infra/relay/src/environments/ManagedTunnelLimits.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Semaphore from "effect/Semaphore"; import { PGlite } from "@electric-sql/pglite"; @@ -10,6 +12,8 @@ import * as RelayDb from "../db.ts"; import { relayManagedEndpointAllocations } from "../persistence/schema.ts"; import * as ManagedTunnelLimits from "./ManagedTunnelLimits.ts"; +const BACKFILL_MIGRATION = "20261028062605_backfill_withdrawn_tunnel_allocations"; + class FakeTransaction extends Context.Service()( "t3code-relay/environments/ManagedTunnelLimits.test/FakeTransaction", ) {} @@ -89,22 +93,23 @@ describe("ManagedTunnelLimits", () => { it.effect("counts each allocation by its tunnel and link state", () => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; const pg = yield* Effect.acquireRelease( Effect.promise(() => PGlite.create()), (client) => Effect.promise(() => client.close()), ); - yield* Effect.promise(() => - pg.exec(` - create table relay_managed_endpoint_allocations ( - user_id text not null, environment_id text not null, tunnel_id text, - tunnel_released_at text, updated_at text not null - ); - create table relay_environment_links ( - user_id text not null, environment_id text not null, revoked_at text, - managed_tunnels_enabled boolean not null, updated_at text not null - ); - `), - ); + const migrationsDir = new URL("../../migrations/postgres/", import.meta.url).pathname; + const migrations = (yield* fs.readDirectory(migrationsDir)).toSorted(); + const applyMigrations = (names: ReadonlyArray) => + Effect.forEach(names, (name) => + fs + .readFileString(`${migrationsDir}${name}/migration.sql`) + .pipe(Effect.flatMap((migration) => Effect.promise(() => pg.exec(migration)))), + ); + const backfill = migrations.indexOf(BACKFILL_MIGRATION); + expect(backfill).toBe(migrations.length - 1); + yield* applyMigrations(migrations.slice(0, backfill)); + const at = (minutes: number) => `2026-10-27T12:${String(minutes).padStart(2, "0")}:00.000Z`; const cases: ReadonlyArray<{ readonly environmentId: string; @@ -169,35 +174,75 @@ describe("ManagedTunnelLimits", () => { allocation: { updatedAt: 50 }, link: { revoked: true, updatedAt: 48 }, }, + { + environmentId: "withdrawn-at-provisioning-write", + counted: false, + allocation: { tunnelId: "t", updatedAt: 20 }, + link: { revoked: true, updatedAt: 20 }, + }, { environmentId: "self", counted: false, allocation: { tunnelId: "t", updatedAt: 10 } }, ]; - for (const entry of cases) { - yield* Effect.promise(() => - pg.query( - "insert into relay_managed_endpoint_allocations values ('user-1', $1, $2, $3, $4)", - [ - entry.environmentId, - entry.allocation.tunnelId ?? null, - entry.allocation.releasedAt === undefined ? null : at(entry.allocation.releasedAt), - at(entry.allocation.updatedAt), - ], - ), + // Written before the deploy: claimDeprovision bumped updatedAt past the revoke, then cleanup failed. + const legacyCases: typeof cases = [ + { + environmentId: "legacy-failed-deprovision", + counted: false, + allocation: { tunnelId: "t", updatedAt: 30 }, + link: { revoked: true, updatedAt: 20 }, + }, + { + environmentId: "legacy-failed-publish-only-deprovision", + counted: false, + allocation: { tunnelId: "t", updatedAt: 30 }, + link: { publishOnly: true, updatedAt: 20 }, + }, + { + environmentId: "legacy-live", + counted: true, + allocation: { tunnelId: "t", updatedAt: 30 }, + link: { updatedAt: 20 }, + }, + ]; + const insert = (entries: typeof cases) => + Effect.forEach(entries, (entry) => + Effect.promise(async () => { + await pg.query( + `insert into relay_managed_endpoint_allocations + (user_id, environment_id, hostname, tunnel_id, tunnel_name, tunnel_released_at, created_at, updated_at) + values ('user-1', $1::text, $1::text, $2, $1::text, $3, $4, $4)`, + [ + entry.environmentId, + entry.allocation.tunnelId ?? null, + entry.allocation.releasedAt === undefined ? null : at(entry.allocation.releasedAt), + at(entry.allocation.updatedAt), + ], + ); + const link = entry.link; + if (link !== undefined) { + await pg.query( + `insert into relay_environment_links + (user_id, environment_id, environment_public_key, endpoint_http_base_url, + endpoint_ws_base_url, endpoint_provider_kind, revoked_at, managed_tunnels_enabled, + created_at, updated_at) + values ('user-1', $1, 'key', 'https://e', 'wss://e', 'cloudflare_tunnel', $2, $3, $4, $4)`, + [ + entry.environmentId, + link.revoked === true ? at(link.updatedAt) : null, + link.publishOnly !== true, + at(link.updatedAt), + ], + ); + } + }), ); - if (entry.link !== undefined) { - const link = entry.link; - yield* Effect.promise(() => - pg.query("insert into relay_environment_links values ('user-1', $1, $2, $3, $4)", [ - entry.environmentId, - link.revoked === true ? at(link.updatedAt) : null, - link.publishOnly !== true, - at(link.updatedAt), - ]), - ); - } - } + yield* insert(legacyCases); + yield* applyMigrations(migrations.slice(backfill)); + yield* insert(cases); yield* Effect.promise(() => pg.query( - "insert into relay_managed_endpoint_allocations values ('user-2', 'other-user', 't', null, $1)", + `insert into relay_managed_endpoint_allocations + (user_id, environment_id, hostname, tunnel_id, tunnel_name, created_at, updated_at) + values ('user-2', 'other-user', 'other-user', 't', 'other-user', $1, $1)`, [at(10)], ), ); @@ -216,11 +261,11 @@ describe("ManagedTunnelLimits", () => { ); expect(counted.map((row) => row.environmentId).toSorted()).toEqual( - cases + [...legacyCases, ...cases] .filter((entry) => entry.counted) .map((entry) => entry.environmentId) .toSorted(), ); - }), + }).pipe(Effect.provide(NodeServices.layer)), ); }); From 3e10e2efa03c2c1c2cb73b741f77324541338d2c Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:36:33 -0700 Subject: [PATCH 3/6] test(relay): the backfill test allows later migrations Co-Authored-By: Claude Opus 5.5 --- infra/relay/src/environments/ManagedTunnelLimits.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/relay/src/environments/ManagedTunnelLimits.test.ts b/infra/relay/src/environments/ManagedTunnelLimits.test.ts index f08a422b0a183..c5a5ce2a581a0 100644 --- a/infra/relay/src/environments/ManagedTunnelLimits.test.ts +++ b/infra/relay/src/environments/ManagedTunnelLimits.test.ts @@ -175,7 +175,7 @@ describe("ManagedTunnelLimits", () => { .pipe(Effect.flatMap((migration) => Effect.promise(() => pg.exec(migration)))), ); const backfill = migrations.indexOf(BACKFILL_MIGRATION); - expect(backfill).toBe(migrations.length - 1); + expect(backfill).toBeGreaterThan(-1); yield* applyMigrations(migrations.slice(0, backfill)); const at = (minutes: number) => `2026-10-27T12:${String(minutes).padStart(2, "0")}:00.000Z`; From e8ab0a77ab8be62a4ff491e4b816dd3132c992a9 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:48:48 -0700 Subject: [PATCH 4/6] fix(relay): a deleted tunnel whose allocation removal failed stops counting Deprovision deletes the tunnel and then removes the allocation inside one transaction. If the removal or the commit fails, the rollback keeps the deleted tunnel's id, so a later stalled relink makes the tunnel limit count a tunnel that no longer exists. After a confirmed delete, a failed deprovision now marks the row released. That keeps tunnel reuse intact, and a tunnel that still exists keeps counting. Co-Authored-By: Claude Opus 5.5 --- .../ManagedEndpointProvider.test.ts | 40 +++++++++++++++++++ .../environments/ManagedEndpointProvider.ts | 16 ++++++++ .../environments/ManagedTunnelLimits.test.ts | 7 ++++ 3 files changed, 63 insertions(+) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 36ea69d5a2694..ac59bc9db80b4 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -1406,6 +1406,46 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("marks the tunnel released when removing the allocation fails after deletion", () => { + const allocationCalls: AllocationCall[] = []; + const allocations = makeAllocations(allocationCalls); + const layer = layerProvider(makePersistentTunnelClient(), makeDnsClient(), { + ...allocations, + removeClaimed: (input) => + Effect.suspend(() => { + allocationCalls.push({ operation: "removeClaimed", input }); + return Effect.fail( + new ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError({ + operation: "remove-claimed", + stage: "database-request", + userId: input.userId, + environmentId: input.environmentId, + }), + ); + }), + }); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + const result = yield* Effect.result(provider.deprovision(key)); + + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") { + expect(result.failure).toMatchObject({ stage: "remove-allocation" }); + } + // The tunnel is gone; without the mark the tunnel limit would keep counting it. + expect(allocationCalls.at(-1)).toMatchObject({ + operation: "claimRelease", + input: { tunnelId: "tunnel-id", markReleased: true }, + }); + }).pipe(Effect.provide(layer)); + }); + it.effect("treats already deleted remote resources as successfully deprovisioned", () => { const allocationCalls: AllocationCall[] = []; const notFound = { _tag: "NotFound" } as const; diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 5524b5117faaf..06901c8ae176f 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -882,6 +882,7 @@ const make = Effect.gen(function* () { return false; } const tunnelId = allocation.tunnelId; + let tunnelDeleted = false; const deprovision = Effect.gen(function* () { const dnsRecordId = allocation.dnsRecordId; if (dnsRecordId !== null) { @@ -909,6 +910,7 @@ const make = Effect.gen(function* () { }), ), ); + tunnelDeleted = true; } return yield* allocations .removeClaimed({ @@ -956,6 +958,20 @@ const make = Effect.gen(function* () { }), ), }), + // The rollback keeps the deleted tunnel's id; mark it released so the tunnel limit stops counting it. + Effect.tapError(() => + tunnelDeleted + ? allocations + .claimRelease({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + markReleased: true, + }) + .pipe(Effect.ignore) + : Effect.void, + ), ); return Option.getOrElse(removed, () => false); }), diff --git a/infra/relay/src/environments/ManagedTunnelLimits.test.ts b/infra/relay/src/environments/ManagedTunnelLimits.test.ts index c5a5ce2a581a0..5837ae3f0eb52 100644 --- a/infra/relay/src/environments/ManagedTunnelLimits.test.ts +++ b/infra/relay/src/environments/ManagedTunnelLimits.test.ts @@ -242,6 +242,13 @@ describe("ManagedTunnelLimits", () => { allocation: { updatedAt: 50 }, link: { revoked: true, updatedAt: 48 }, }, + // Tunnel deleted but the row survived; deprovision marked it released, then a relink stalled. + { + environmentId: "released-after-failed-removal-then-reserved", + counted: false, + allocation: { tunnelId: "t", releasedAt: 20, updatedAt: 30 }, + link: { revoked: true, updatedAt: 15 }, + }, { environmentId: "withdrawn-at-provisioning-write", counted: false, From 445bc3747653e78e9bd9dd77daa49e620018b69b Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:54:24 -0700 Subject: [PATCH 5/6] fix(relay): the deleted-tunnel mark also runs on interruption and logs its failure Co-Authored-By: Claude Opus 5.5 --- .../environments/ManagedEndpointProvider.test.ts | 7 ++++++- .../src/environments/ManagedEndpointProvider.ts | 16 +++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index ac59bc9db80b4..93ad7491bf7bf 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -1439,9 +1439,14 @@ describe("ManagedEndpointProvider", () => { expect(result.failure).toMatchObject({ stage: "remove-allocation" }); } // The tunnel is gone; without the mark the tunnel limit would keep counting it. + const claim = allocationCalls.find((call) => call.operation === "claimDeprovision"); expect(allocationCalls.at(-1)).toMatchObject({ operation: "claimRelease", - input: { tunnelId: "tunnel-id", markReleased: true }, + input: { + tunnelId: "tunnel-id", + generation: (claim?.input as { generation: number }).generation + 1, + markReleased: true, + }, }); }).pipe(Effect.provide(layer)); }); diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 06901c8ae176f..9a76b9c09981a 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -958,8 +958,9 @@ const make = Effect.gen(function* () { }), ), }), - // The rollback keeps the deleted tunnel's id; mark it released so the tunnel limit stops counting it. - Effect.tapError(() => + // The rollback keeps the deleted tunnel's id; mark it released so the tunnel limit stops + // counting it. onError also covers an interrupted unlink, which rolls back the same way. + Effect.onError(() => tunnelDeleted ? allocations .claimRelease({ @@ -969,7 +970,16 @@ const make = Effect.gen(function* () { generation: claimedGeneration, markReleased: true, }) - .pipe(Effect.ignore) + .pipe( + Effect.catch((error) => + Effect.logWarning("Failed to mark a deleted managed tunnel released", { + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + error: error.message, + }), + ), + ) : Effect.void, ), ); From 4cba6d8ec2472240c950402ee73392f13b530193 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:55:18 -0700 Subject: [PATCH 6/6] test(relay): deleted-tunnel mark test asserts the claim without optional chaining Co-Authored-By: Claude Opus 5.5 --- infra/relay/src/environments/ManagedEndpointProvider.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 93ad7491bf7bf..01230171be9ba 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -1440,11 +1440,12 @@ describe("ManagedEndpointProvider", () => { } // The tunnel is gone; without the mark the tunnel limit would keep counting it. const claim = allocationCalls.find((call) => call.operation === "claimDeprovision"); + if (claim === undefined) return expect.unreachable("deprovision never claimed the row"); expect(allocationCalls.at(-1)).toMatchObject({ operation: "claimRelease", input: { tunnelId: "tunnel-id", - generation: (claim?.input as { generation: number }).generation + 1, + generation: (claim.input as { generation: number }).generation + 1, markReleased: true, }, });