From 7f0f70c13ac8fadb95dfc6501071b594cb212378 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sun, 13 Sep 2026 13:56:14 +0000 Subject: [PATCH 1/6] fix(auth): contain destructive identity consolidation --- server/src/db/identity-db.ts | 71 ++- server/src/db/identity-mutation-policy.ts | 17 + server/src/db/user-merge-db.ts | 8 + server/src/http.ts | 7 + server/src/routes/account-linking.ts | 4 +- server/src/routes/admin/users.ts | 7 +- .../routes/identity-mutation-containment.ts | 8 + server/src/routes/workos-webhooks.ts | 53 +- .../integration/admin-bind-email.test.ts | 169 ++---- .../admin-link-unlink-credential.test.ts | 367 +++--------- .../admin-promote-credential.test.ts | 404 ++++--------- .../integration/authorization-epoch.test.ts | 72 ++- .../integration/merge-bind-not-delete.test.ts | 546 +++++------------- .../workos-user-deleted-primary.test.ts | 224 ++++--- .../identity-mutation-containment.test.ts | 86 +++ ...dentity-mutation-route-containment.test.ts | 159 +++++ .../unit/native-auth-http-wiring.test.ts | 60 +- server/tests/unit/set-primary-email.test.ts | 70 +-- ...workos-webhook-user-deleted-wiring.test.ts | 54 +- 19 files changed, 967 insertions(+), 1419 deletions(-) create mode 100644 server/src/db/identity-mutation-policy.ts create mode 100644 server/src/routes/identity-mutation-containment.ts create mode 100644 server/tests/unit/identity-mutation-containment.test.ts create mode 100644 server/tests/unit/identity-mutation-route-containment.test.ts diff --git a/server/src/db/identity-db.ts b/server/src/db/identity-db.ts index 9de63e54f0d..ca819fb8dcc 100644 --- a/server/src/db/identity-db.ts +++ b/server/src/db/identity-db.ts @@ -11,6 +11,7 @@ */ import { getPool } from './client.js'; +import { assertIdentityConsolidationAllowed } from './identity-mutation-policy.js'; import { bumpAuthorizationEpochs } from './authorization-epoch-db.js'; import { createLogger } from '../logger.js'; import { notifySystemError } from '../addie/error-notifier.js'; @@ -18,32 +19,58 @@ import { notifySystemError } from '../addie/error-notifier.js'; const logger = createLogger('identity-db'); /** - * When a WorkOS user that is the primary binding on a multi-credential - * identity is deleted (operator action, account closure, GDPR/CCPA erasure - * webhook), the CASCADE on `identity_workos_users.workos_user_id` drops the - * binding. The identity is left with zero primaries until an admin - * intervenes — `attachIdentityId` resolves `primary_workos_user_id` to NULL - * and skips the id-swap, so the surviving secondary signs in to an empty - * workspace (all their app-state was keyed on the now-deleted primary's - * workos_user_id). That's a denial-of-service against any non-primary user. - * - * Promote the longest-bound surviving secondary to primary in the same - * transaction, before the CASCADE fires. The id-swap in the auth middleware - * then routes both the dead binding and the surviving secondary to the new - * primary, keeping app-state reads intact. - * - * Returns the promoted credential's workos_user_id when a promotion ran, or - * null when there was no successor (single-credential identity — normal - * deletion, nothing to do) or the deleted user was not primary. - * - * On unexpected DB errors: log + ops alert, return null. The caller still - * returns 200 to WorkOS so the webhook doesn't retry-storm on a transient - * promotion failure; the binding is still deleted by the subsequent CASCADE, - * and admins can repair the identity manually. + * Apply an authoritative provider deletion without selecting a new primary. + * Revoke cached routing for every affected credential in the same transaction + * as the binding disappears. Return the IDs to evict from local caches after + * commit; their persisted epochs handle caches on other instances. + */ +export async function deleteIdentityCredential(workosUserId: string): Promise { + const client = await getPool().connect(); + + try { + await client.query('BEGIN'); + // Use a stable lock order when simultaneous webhooks delete siblings. + // Unlink also needs the binding row lock before it can change the identity. + const bound = await client.query<{ workos_user_id: string }>( + `SELECT workos_user_id FROM identity_workos_users + WHERE identity_id = ( + SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1 + ) + ORDER BY workos_user_id + FOR UPDATE`, + [workosUserId], + ); + const affectedUserIds = [...new Set([ + workosUserId, + ...bound.rows.map((row) => row.workos_user_id), + ])]; + + // Authentication checks the actual credential's epoch, so deleting the + // primary's epoch alone would leave each surviving credential's cache valid. + await bumpAuthorizationEpochs(client, affectedUserIds); + await client.query('DELETE FROM users WHERE workos_user_id = $1', [workosUserId]); + await client.query('DELETE FROM organization_memberships WHERE workos_user_id = $1', [workosUserId]); + await client.query('COMMIT'); + return affectedUserIds; + } catch (err) { + await client.query('ROLLBACK').catch(() => undefined); + throw err; + } finally { + client.release(); + } +} + +/** + * Legacy primary promotion is disabled for #6827: changing the canonical + * credential can transfer authority even without moving membership rows. + * Provider deletions use deleteIdentityCredential without inferring a successor. + * @throws IdentityMutationDisabledError before any database access. */ export async function promoteSecondaryIfPrimaryDeleted( workosUserId: string, ): Promise<{ promotedUserId: string } | null> { + // A primary flip changes canonical authority even without moving rows. + assertIdentityConsolidationAllowed(); const pool = getPool(); const client = await pool.connect(); diff --git a/server/src/db/identity-mutation-policy.ts b/server/src/db/identity-mutation-policy.ts new file mode 100644 index 00000000000..7cba02b1527 --- /dev/null +++ b/server/src/db/identity-mutation-policy.ts @@ -0,0 +1,17 @@ +/** + * Temporary containment for #6827. Generic consolidation cannot preserve + * credential authority or membership provenance, in either direction. + * Operator confirmation, empty previews and caller privilege are not bypasses. + */ +export class IdentityMutationDisabledError extends Error { + readonly code = 'identity_mutation_disabled'; + + constructor() { + super('Identity consolidation is disabled until authority and provenance can be preserved.'); + this.name = 'IdentityMutationDisabledError'; + } +} + +export function assertIdentityConsolidationAllowed(): void { + throw new IdentityMutationDisabledError(); +} diff --git a/server/src/db/user-merge-db.ts b/server/src/db/user-merge-db.ts index 6452b3752e5..fd0c477ed9e 100644 --- a/server/src/db/user-merge-db.ts +++ b/server/src/db/user-merge-db.ts @@ -9,6 +9,7 @@ */ import { getPool } from './client.js'; +import { assertIdentityConsolidationAllowed } from './identity-mutation-policy.js'; import { bumpAuthorizationEpochs } from './authorization-epoch-db.js'; import { createLogger } from '../logger.js'; const logger = createLogger('user-merge-db'); @@ -125,6 +126,10 @@ export interface MergeUsersOptions { } /** + * Disabled for #6827 containment; every caller is refused before database access. + * The legacy implementation below must not be re-enabled without preserving + * authority and provenance. Operator confirmation is not an exception. + * * Merge two user accounts. Moves all of the secondary user's app-state rows * to the primary, then binds the secondary's WorkOS user to the primary's * identity as a non-primary sign-in credential. The secondary WorkOS user @@ -145,6 +150,9 @@ export async function mergeUsers( mergedBy: string, options: MergeUsersOptions = {} ): Promise { + // No caller may move, copy or deduplicate authority through identity linkage. + // Keep this before pool access, including for promotion and automatic callers. + assertIdentityConsolidationAllowed(); const pool = getPool(); const client = await pool.connect(); diff --git a/server/src/http.ts b/server/src/http.ts index ba99ce7f519..56fdd272441 100644 --- a/server/src/http.ts +++ b/server/src/http.ts @@ -63,6 +63,7 @@ import { syncSlackUsers, getSyncStatus, tryAutoLinkWebsiteUserToSlack } from "./ import { isSlackConfigured, testSlackConnection } from "./slack/client.js"; import { handleSlashCommand } from "./slack/commands.js"; import { getCompanyDomain, getGoogleEmailAliases } from "./utils/email-domain.js"; +import { assertIdentityConsolidationAllowed } from "./db/identity-mutation-policy.js"; import { hasActiveSlackLink } from "./utils/slack-linkage.js"; import { isUuid } from "./utils/uuid.js"; import { resolveUserNameWithFallbacks, sanitizeName } from "./utils/resolve-user-name.js"; @@ -7816,6 +7817,9 @@ ${p.category ? `${p.category}\n` : ''}${publishedUrl}< const workosUsers = await workos.userManagement.listUsers({ email: aliasEmail }); const match = workosUsers.data.find(u => u.id !== user.id); if (match) { + duplicateAliasEmail = match.email; + // Containment before even creating a local duplicate credential. + assertIdentityConsolidationAllowed(); // Insert into local users table so mergeUsers can operate on it await pool.query( `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, workos_created_at, workos_updated_at, created_at, updated_at) @@ -7835,6 +7839,9 @@ ${p.category ? `${p.category}\n` : ''}${publishedUrl}< if (existing) { duplicateAliasEmail = existing.email; + // Refuse before alias claims or WorkOS membership copies. Mailbox + // equivalence cannot authorize consolidation (#6827). + assertIdentityConsolidationAllowed(); // Claim the alias atomically — UNIQUE(LOWER(email)) prevents // two users from claiming the same target concurrently. diff --git a/server/src/routes/account-linking.ts b/server/src/routes/account-linking.ts index 1e9853a3051..e1cfed0bb1f 100644 --- a/server/src/routes/account-linking.ts +++ b/server/src/routes/account-linking.ts @@ -4,7 +4,7 @@ import rateLimit from 'express-rate-limit'; import { createLogger } from '../logger.js'; import { requireAuth } from '../middleware/auth.js'; import { query, getPool } from '../db/client.js'; -import { mergeUsers } from '../db/user-merge-db.js'; +import { refuseIdentityConsolidation } from './identity-mutation-containment.js'; import { sendEmailLinkVerification } from '../notifications/email.js'; import { getWorkos } from '../auth/workos-client.js'; import { CachedPostgresStore } from '../middleware/pg-rate-limit-store.js'; @@ -215,7 +215,7 @@ export function createAccountLinkingRouter(): Router { }); // PUT /api/me/linked-emails/primary — swap a linked alias to be the primary email - router.put('/primary', requireAuth, verifyExecuteLimiter, async (req: Request, res: Response) => { + router.put('/primary', requireAuth, refuseIdentityConsolidation, verifyExecuteLimiter, async (req: Request, res: Response) => { try { const userId = req.user!.id; const { email } = req.body; diff --git a/server/src/routes/admin/users.ts b/server/src/routes/admin/users.ts index 639ecd74567..6ada65f7271 100644 --- a/server/src/routes/admin/users.ts +++ b/server/src/routes/admin/users.ts @@ -22,6 +22,7 @@ import { backfillOrganizationMemberships, backfillUsers, backfillOrganizationDom import { sendSlackInviteEmail, hasSlackInviteBeenSent } from '../../notifications/email.js'; import { getWorkos } from '../../auth/workos-client.js'; import { mergeUsers } from '../../db/user-merge-db.js'; +import { refuseIdentityConsolidation } from '../identity-mutation-containment.js'; import { resolveMembershipTier, type MembershipTierRow } from '../../db/organization-db.js'; import { buildCountryMembersCsv, @@ -856,7 +857,7 @@ export function createAdminUsersRouter(): Router { // // Trust model: admin is asserting the email belongs to the person. No // verification email is sent to the new address. Phase 3 may add one. - router.post('/:userId/linked-emails', ...requireGlobalAdmin, async (req, res) => { + router.post('/:userId/linked-emails', ...requireGlobalAdmin, refuseIdentityConsolidation, async (req, res) => { const adminEmail = req.user!.email; const adminUserId = req.user!.id; const existingUserId = req.params.userId; @@ -1041,7 +1042,7 @@ export function createAdminUsersRouter(): Router { // its own app-state, mergeUsers moves that data to this user — admin is // asserting the two represent the same person. The trust model and // confirmation UX live on the admin frontend. - router.post('/:userId/credentials', ...requireGlobalAdmin, async (req, res) => { + router.post('/:userId/credentials', ...requireGlobalAdmin, refuseIdentityConsolidation, async (req, res) => { const adminEmail = req.user!.email; const adminUserId = req.user!.id; const adminAuthCredentialId = req.user!.authWorkosUserId ?? req.user!.id; @@ -1375,7 +1376,7 @@ export function createAdminUsersRouter(): Router { // data — degraded but not broken. A failure of the follow-up UPDATE // would persist that degraded state; the audit row records the intent // and the recovery is a one-line UPDATE. - router.post('/:userId/credentials/:credentialId/promote', ...requireGlobalAdmin, async (req, res) => { + router.post('/:userId/credentials/:credentialId/promote', ...requireGlobalAdmin, refuseIdentityConsolidation, async (req, res) => { const adminEmail = req.user!.email; const adminUserId = req.user!.id; const adminIdentityId = req.user!.identityId; diff --git a/server/src/routes/identity-mutation-containment.ts b/server/src/routes/identity-mutation-containment.ts new file mode 100644 index 00000000000..67db3eb793b --- /dev/null +++ b/server/src/routes/identity-mutation-containment.ts @@ -0,0 +1,8 @@ +import type { RequestHandler } from 'express'; +import { IdentityMutationDisabledError } from '../db/identity-mutation-policy.js'; + +/** Refuse after authentication, before any provider or local mutation. */ +export const refuseIdentityConsolidation: RequestHandler = (_req, res) => { + const error = new IdentityMutationDisabledError(); + res.status(409).json({ error: error.code, message: error.message }); +}; diff --git a/server/src/routes/workos-webhooks.ts b/server/src/routes/workos-webhooks.ts index 3e35d58cc50..846029fdb72 100644 --- a/server/src/routes/workos-webhooks.ts +++ b/server/src/routes/workos-webhooks.ts @@ -22,7 +22,7 @@ import { Router, Request, Response } from 'express'; import { createLogger } from '../logger.js'; import { getPool } from '../db/client.js'; import { invalidateSessionsForUsers } from '../middleware/auth.js'; -import { promoteSecondaryIfPrimaryDeleted } from '../db/identity-db.js'; +import { deleteIdentityCredential } from '../db/identity-db.js'; import { upsertWorkosDomain, autoPromotePrimaryIfNone, @@ -495,21 +495,6 @@ async function upsertUser(user: UserData): Promise { logger.info({ userId: user.id, email: user.email }, 'Upserted user'); } -/** - * Delete user from local users table - * Called on user.deleted events - */ -async function deleteUser(userId: string): Promise { - const pool = getPool(); - - await pool.query( - `DELETE FROM users WHERE workos_user_id = $1`, - [userId] - ); - - logger.info({ userId }, 'Deleted user'); -} - /** * Update user details across all their memberships. * Reads the resolved name from the users table (which may have been enriched @@ -547,23 +532,6 @@ async function updateUserAcrossMemberships(user: UserData): Promise { }, 'Updated user details across memberships'); } -/** - * Delete all memberships for a user - */ -async function deleteUserMemberships(userId: string): Promise { - const pool = getPool(); - - const result = await pool.query( - `DELETE FROM organization_memberships WHERE workos_user_id = $1`, - [userId] - ); - - logger.info({ - userId, - deletedCount: result.rowCount, - }, 'Deleted all memberships for user'); -} - /** * Sync organization domains from WorkOS * This upserts domains and removes any that are no longer in WorkOS @@ -1154,21 +1122,10 @@ export function createWorkOSWebhooksRouter(): Router { case 'user.deleted': { const user = event.data as unknown as UserData; - // Promote a surviving secondary BEFORE the CASCADE on - // identity_workos_users.workos_user_id fires (via deleteUser). - // Without this, the identity is left with zero primaries and the - // surviving secondary signs in to an empty workspace — a DoS - // vector reachable via GDPR/CCPA-driven WorkOS deletions. - const promoted = await promoteSecondaryIfPrimaryDeleted(user.id); - await deleteUser(user.id); - await deleteUserMemberships(user.id); - // Close the 60-second session/JWT cache window where a cached - // pre-deletion swap would still route reads to the dead binding. - // Invalidate both the deleted user and the promoted successor so - // the next request re-resolves identity from the DB. - const sessionsToInvalidate = [user.id]; - if (promoted) sessionsToInvalidate.push(promoted.promotedUserId); - invalidateSessionsForUsers(sessionsToInvalidate); + // Provider deletion revokes this credential. Do not infer a new + // primary: that would redirect surviving credentials' authority. + const affectedUserIds = await deleteIdentityCredential(user.id); + invalidateSessionsForUsers(affectedUserIds); invalidateUnifiedUsersCache(); break; } diff --git a/server/tests/integration/admin-bind-email.test.ts b/server/tests/integration/admin-bind-email.test.ts index c82f7549f02..19b4182eeb8 100644 --- a/server/tests/integration/admin-bind-email.test.ts +++ b/server/tests/integration/admin-bind-email.test.ts @@ -1,15 +1,4 @@ -/** - * Admin "create + bind sign-in email" integration test (Phase 2b). - * - * Exercises POST /api/admin/users/:userId/linked-emails: - * - Creates a fresh WorkOS user for the new email (mocked). - * - Inserts into local users (trigger fires, creating a singleton identity). - * - mergeUsers re-points the new user's binding to the existing user's - * identity as is_primary = FALSE; drops the orphan singleton. - * - * After this, the existing user has two bound WorkOS users; the auth - * middleware will id-swap a non-primary login to the canonical id. - */ +/** Creation-and-bind is contained before any WorkOS mutation or local insert. */ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; @@ -22,7 +11,7 @@ vi.hoisted(() => { process.env.WORKOS_COOKIE_PASSWORD ??= 'test-cookie-password-at-least-32-chars-long'; }); import request from 'supertest'; -import { initializeDatabase, closeDatabase, getPool } from '../../src/db/client.js'; +import { initializeDatabase, closeDatabase } from '../../src/db/client.js'; import { runMigrations } from '../../src/db/migrate.js'; import type { Pool } from 'pg'; @@ -80,12 +69,13 @@ vi.mock('../../src/auth/workos-client.js', () => { }; }); -import { HTTPServer } from '../../src/http.js'; +import express from 'express'; +import { createAdminUsersRouter } from '../../src/routes/admin/users.js'; +import { stopAuthTimers } from '../../src/middleware/auth.js'; const EXISTING_USER_ID = 'user_test_bind_existing'; describe('POST /api/admin/users/:userId/linked-emails (admin bind)', () => { - let server: HTTPServer; let app: any; let pool: Pool; @@ -94,14 +84,14 @@ describe('POST /api/admin/users/:userId/linked-emails (admin bind)', () => { connectionString: process.env.DATABASE_URL || 'postgresql://adcp:localdev@localhost:5432/adcp_test', }); await runMigrations(); - server = new HTTPServer(); - await server.start(0); - app = server.app; + app = express(); + app.use(express.json()); + app.use('/api/admin/users', createAdminUsersRouter()); }, 60000); afterAll(async () => { await cleanup(); - await server?.stop(); + stopAuthTimers(); await closeDatabase(); }); @@ -134,114 +124,59 @@ describe('POST /api/admin/users/:userId/linked-emails (admin bind)', () => { ]); } - it('binds a new sign-in email to the existing user\'s identity', async () => { - const response = await request(app) - .post(`/api/admin/users/${EXISTING_USER_ID}/linked-emails`) - .send({ email: 'newalias@test.example' }) - .expect(201); - - expect(response.body).toMatchObject({ - bound: true, - existing_user_id: EXISTING_USER_ID, - new_email: 'newalias@test.example', - new_workos_user_id: MOCK_NEW_WORKOS_USER_ID, - }); - - // Both WorkOS users now bound to one identity. Existing is primary. - const bindings = await pool.query<{ workos_user_id: string; identity_id: string; is_primary: boolean }>( - `SELECT workos_user_id, identity_id, is_primary - FROM identity_workos_users - WHERE workos_user_id IN ($1, $2) - ORDER BY is_primary DESC`, - [EXISTING_USER_ID, MOCK_NEW_WORKOS_USER_ID] + it.each([ + { email: 'newalias@test.example' }, + { email: 'newalias@test.example', consolidate: true }, + { email: 'existing@test.example' }, + { email: 'not-an-email' }, + {}, + ])('refuses create-and-bind before provider or local writes: %j', async (body) => { + const before = await pool.query( + `SELECT to_jsonb(u) AS state FROM users u WHERE workos_user_id = $1`, + [EXISTING_USER_ID], ); - - expect(bindings.rows).toHaveLength(2); - expect(bindings.rows[0].workos_user_id).toBe(EXISTING_USER_ID); - expect(bindings.rows[0].is_primary).toBe(true); - expect(bindings.rows[1].workos_user_id).toBe(MOCK_NEW_WORKOS_USER_ID); - expect(bindings.rows[1].is_primary).toBe(false); - expect(bindings.rows[0].identity_id).toBe(bindings.rows[1].identity_id); - }); - - it('400s on invalid email', async () => { - const response = await request(app) - .post(`/api/admin/users/${EXISTING_USER_ID}/linked-emails`) - .send({ email: 'not-an-email' }) - .expect(400); - expect(response.body.error).toMatch(/invalid email/i); - }); - - it('409s when the new email matches the user\'s current primary email', async () => { - const response = await request(app) - .post(`/api/admin/users/${EXISTING_USER_ID}/linked-emails`) - .send({ email: 'existing@test.example' }) - .expect(409); - expect(response.body.error).toMatch(/already.*primary/i); - }); - - it('409s when the new email already has an existing AAO account', async () => { - await pool.query( - `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, - workos_created_at, workos_updated_at, created_at, updated_at) - VALUES ('user_test_bind_other', 'taken@test.example', 'Other', 'User', true, NOW(), NOW(), NOW(), NOW())` + const bindingsBefore = await pool.query( + `SELECT * FROM identity_workos_users WHERE workos_user_id = $1`, + [EXISTING_USER_ID], ); - try { - const response = await request(app) - .post(`/api/admin/users/${EXISTING_USER_ID}/linked-emails`) - .send({ email: 'taken@test.example' }) - .expect(409); - expect(response.body.error).toMatch(/already.*account/i); - } finally { - await pool.query(`DELETE FROM users WHERE workos_user_id = 'user_test_bind_other'`); - } - }); - - it('404s when the existing user does not exist', async () => { - const response = await request(app) - .post(`/api/admin/users/user_does_not_exist/linked-emails`) - .send({ email: 'fresh@test.example' }) - .expect(404); - expect(response.body.error).toMatch(/not found/i); - }); - - it('translates WorkOS 422 (email already in use upstream) to a 409', async () => { - mockCreateUser.mockImplementationOnce(async () => { - const err: any = new Error('Email already in use'); - err.status = 422; - throw err; - }); const response = await request(app) .post(`/api/admin/users/${EXISTING_USER_ID}/linked-emails`) - .send({ email: 'taken-upstream@test.example' }) + .send(body) .expect(409); - // The error wording surfaces the WorkOS status + message and points - // admins at the "Link existing WorkOS user" tool. - expect(response.body.message).toMatch(/already in use upstream|Link existing WorkOS user/i); - // No local users row should have been created. - const localCheck = await pool.query( - `SELECT 1 FROM users WHERE LOWER(email) = 'taken-upstream@test.example'` - ); - expect(localCheck.rows).toEqual([]); + expect(response.body).toEqual({ + error: 'identity_mutation_disabled', + message: 'Identity consolidation is disabled until authority and provenance can be preserved.', + }); + expect(mockCreateUser).not.toHaveBeenCalled(); + expect(mockDeleteUser).not.toHaveBeenCalled(); + expect((await pool.query( + `SELECT to_jsonb(u) AS state FROM users u WHERE workos_user_id = $1`, + [EXISTING_USER_ID], + )).rows).toEqual(before.rows); + expect((await pool.query( + `SELECT * FROM identity_workos_users WHERE workos_user_id = $1`, + [EXISTING_USER_ID], + )).rows).toEqual(bindingsBefore.rows); + expect((await pool.query( + `SELECT 1 FROM users WHERE workos_user_id = $1`, + [MOCK_NEW_WORKOS_USER_ID], + )).rows).toEqual([]); }); - it('rolls back the WorkOS user when local bind fails after createUser succeeded', async () => { - // Pre-create the new WorkOS user id locally so the post-createUser INSERT - // would succeed, but force mergeUsers to fail by deleting the trigger- - // created identity binding for the existing user (Phase 1 trigger - // guarantees normally — we break the invariant to simulate a partial - // failure path). + it('refuses when the existing identity has no primary binding', async () => { await pool.query(`DELETE FROM identity_workos_users WHERE workos_user_id = $1`, [EXISTING_USER_ID]); - - const response = await request(app) + await request(app) .post(`/api/admin/users/${EXISTING_USER_ID}/linked-emails`) - .send({ email: 'rollback@test.example' }) - .expect(500); - - expect(response.body.error).toMatch(/failed to bind/i); - expect(response.body.message).toMatch(/rolled back|retry/i); - expect(mockDeleteUser).toHaveBeenCalledWith(MOCK_NEW_WORKOS_USER_ID); + .send({ email: 'missing-primary@test.example', consolidate: true }) + .expect(409) + .expect(({ body }) => expect(body.error).toBe('identity_mutation_disabled')); + expect(mockCreateUser).not.toHaveBeenCalled(); + expect(mockDeleteUser).not.toHaveBeenCalled(); + expect((await pool.query( + `SELECT 1 FROM identity_workos_users WHERE workos_user_id = $1`, + [EXISTING_USER_ID], + )).rows).toEqual([]); }); }); diff --git a/server/tests/integration/admin-link-unlink-credential.test.ts b/server/tests/integration/admin-link-unlink-credential.test.ts index 45cedc66bbc..758d5767a05 100644 --- a/server/tests/integration/admin-link-unlink-credential.test.ts +++ b/server/tests/integration/admin-link-unlink-credential.test.ts @@ -11,7 +11,6 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; import request from 'supertest'; import type { Pool } from 'pg'; -import { readFileSync } from 'node:fs'; vi.hoisted(() => { process.env.WORKOS_API_KEY ??= 'sk_test_mock_key'; @@ -58,15 +57,17 @@ vi.mock('../../src/middleware/csrf.js', () => ({ csrfProtection: (_req: any, _res: any, next: any) => next(), })); -import { initializeDatabase, closeDatabase, getPool } from '../../src/db/client.js'; +import { initializeDatabase, closeDatabase } from '../../src/db/client.js'; import { runMigrations } from '../../src/db/migrate.js'; -import { HTTPServer } from '../../src/http.js'; +import express from 'express'; +import { createAdminUsersRouter } from '../../src/routes/admin/users.js'; +import { stopAuthTimers } from '../../src/middleware/auth.js'; const HOST_USER_ID = 'user_test_link_host'; const TARGET_USER_ID = 'user_test_link_target'; +const AUTHORITY_ORG_ID = 'org_test_unlink_authority'; describe('admin link / unlink credential', () => { - let server: HTTPServer; let app: any; let pool: Pool; @@ -75,14 +76,14 @@ describe('admin link / unlink credential', () => { connectionString: process.env.DATABASE_URL || 'postgresql://adcp:localdev@localhost:5432/adcp_test', }); await runMigrations(); - server = new HTTPServer(); - await server.start(0); - app = server.app; + app = express(); + app.use(express.json()); + app.use('/api/admin/users', createAdminUsersRouter()); }, 60000); afterAll(async () => { await cleanup(); - await server?.stop(); + stopAuthTimers(); await closeDatabase(); }); @@ -98,307 +99,60 @@ describe('admin link / unlink credential', () => { }); async function cleanup() { + await pool.query(`DELETE FROM organization_memberships WHERE workos_organization_id = $1`, [AUTHORITY_ORG_ID]); + await pool.query(`DELETE FROM organizations WHERE workos_organization_id = $1`, [AUTHORITY_ORG_ID]); await pool.query(`DELETE FROM users WHERE workos_user_id IN ($1, $2)`, [HOST_USER_ID, TARGET_USER_ID]); } - describe('POST /credentials (bind existing)', () => { - it('keeps the declared global-admin and people-UI confirmation contracts', () => { - const routeSource = readFileSync( - new URL('../../src/routes/admin/users.ts', import.meta.url), - 'utf8', - ); - const bindRoute = routeSource.match( - /router\.post\('\/:userId\/credentials', \.\.\.requireGlobalAdmin, async \(req, res\) => \{[\s\S]*?\n \}\);/ - )?.[0]; - expect(bindRoute).toContain('...requireGlobalAdmin'); - - const peopleUi = readFileSync( - new URL('../../public/admin-people.html', import.meta.url), - 'utf8', - ); - expect(peopleUi).toContain('data.consolidate_confirmation_required'); - expect(peopleUi).toContain('JSON.stringify({ workos_user_id: trimmed, consolidate: true })'); - }); - - it('binds an existing WorkOS user (already in local users) under host\'s identity', async () => { - // Insert the target locally so the trigger creates its singleton identity - await pool.query( - `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, - workos_created_at, workos_updated_at, created_at, updated_at) - VALUES ($1, 'target@test.example', 'Target', 'User', true, NOW(), NOW(), NOW(), NOW())`, - [TARGET_USER_ID] - ); - - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID }) - .expect(201); - - expect(response.body).toMatchObject({ - linked: true, - existing_user_id: HOST_USER_ID, - bound_workos_user_id: TARGET_USER_ID, - }); - - // Both bound to one identity; target is non-primary. - const result = await pool.query( - `SELECT iwu.workos_user_id, iwu.is_primary, iwu.identity_id - FROM identity_workos_users iwu - WHERE iwu.workos_user_id IN ($1, $2) - ORDER BY iwu.is_primary DESC`, - [HOST_USER_ID, TARGET_USER_ID] - ); - expect(result.rows).toHaveLength(2); - expect(result.rows[0].workos_user_id).toBe(HOST_USER_ID); - expect(result.rows[0].is_primary).toBe(true); - expect(result.rows[1].workos_user_id).toBe(TARGET_USER_ID); - expect(result.rows[1].is_primary).toBe(false); - expect(result.rows[0].identity_id).toBe(result.rows[1].identity_id); - }); - - it('fetches the WorkOS user and upserts when not in local users yet', async () => { - mockGetUser.mockResolvedValueOnce({ - id: TARGET_USER_ID, - email: 'fetched@test.example', - firstName: 'Fetched', - lastName: 'User', - emailVerified: true, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }); - - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID }) - .expect(201); - - expect(mockGetUser).toHaveBeenCalledWith(TARGET_USER_ID); - - const local = await pool.query(`SELECT email FROM users WHERE workos_user_id = $1`, [TARGET_USER_ID]); - expect(local.rows[0].email).toBe('fetched@test.example'); - }); - - it('400s on missing or malformed workos_user_id', async () => { - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({}) - .expect(400); - - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: 'not-a-workos-id' }) - .expect(400); - }); - - it('400s on self-bind', async () => { - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: HOST_USER_ID }) - .expect(400); - }); - - it('idempotent: returns linked: true with no change when already bound', async () => { - await pool.query( - `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, - workos_created_at, workos_updated_at, created_at, updated_at) - VALUES ($1, 'target@test.example', 'Target', 'User', true, NOW(), NOW(), NOW(), NOW())`, - [TARGET_USER_ID] - ); - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID }) - .expect(201); - - // Second call is a no-op. - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID }) - .expect(200); - expect(response.body.linked).toBe(true); - }); - - it('404s when host user does not exist', async () => { - await request(app) - .post(`/api/admin/users/user_does_not_exist/credentials`) - .send({ workos_user_id: TARGET_USER_ID }) - .expect(404); - }); - - it('404s when WorkOS getUser returns 404 for the credential id', async () => { - const err: any = new Error('Not found'); - err.status = 404; - mockGetUser.mockRejectedValueOnce(err); - - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID }) - .expect(404); - expect(response.body.error).toMatch(/not found/i); - }); - - it('refuses to consolidate a cred with existing app-state without explicit opt-in', async () => { - // Insert cred locally + give it an organization_membership (real AAO data) + describe('POST /credentials containment', () => { + it.each([false, true])('refuses existing-account consolidation even with consolidate=%s', async (consolidate) => { await pool.query( `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, workos_created_at, workos_updated_at, created_at, updated_at) VALUES ($1, 'target@test.example', 'Target', 'User', true, NOW(), NOW(), NOW(), NOW())`, - [TARGET_USER_ID] - ); - const orgId = 'org_test_link_consolidate'; - await pool.query( - `INSERT INTO organizations (workos_organization_id, name, created_at, updated_at) - VALUES ($1, 'Test Org', NOW(), NOW()) - ON CONFLICT (workos_organization_id) DO NOTHING`, - [orgId] + [TARGET_USER_ID], ); - await pool.query( - `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role, created_at, updated_at) - VALUES ($1, $2, 'target@test.example', 'member', NOW(), NOW())`, - [TARGET_USER_ID, orgId] + const before = await pool.query( + `SELECT * FROM identity_workos_users WHERE workos_user_id IN ($1, $2) ORDER BY workos_user_id`, + [HOST_USER_ID, TARGET_USER_ID], ); - - try { - const refused = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID }) - .expect(409); - expect(refused.body.consolidate_confirmation_required).toBe(true); - expect(refused.body.message).toMatch(/consolidate/i); - - // A static global-admin key has no person-level principal, so it may - // observe the preflight but cannot acknowledge a destructive merge. + for (const accessMechanism of ['sso', 'static_admin_api_key']) { await request(app) .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .set('X-Test-Admin-Access-Mechanism', 'static_admin_api_key') - .send({ workos_user_id: TARGET_USER_ID, consolidate: true }) - .expect(403) - .expect(({ body }) => { - expect(body.error).toBe('identity_bearing_admin_required'); - }); - - // Same call with consolidate: true succeeds and moves the membership - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID, consolidate: true }) - .expect(201); - - const moved = await pool.query( - `SELECT workos_user_id FROM organization_memberships WHERE workos_organization_id = $1`, - [orgId] - ); - expect(moved.rows).toHaveLength(1); - expect(moved.rows[0].workos_user_id).toBe(HOST_USER_ID); - - // The destructive, server-confirmed consolidation is attributed to - // the authenticated global admin by mergeUsers' audit record. - const audit = await pool.query<{ workos_user_id: string; details: any }>( - `SELECT workos_user_id, details - FROM registry_audit_log - WHERE action = 'merge_user' AND resource_id = $1 - ORDER BY created_at DESC LIMIT 1`, - [TARGET_USER_ID], - ); - expect(audit.rows).toHaveLength(1); - expect(audit.rows[0].workos_user_id).toBe('user_test_admin_link'); - expect(audit.rows[0].details).toMatchObject({ - primary_user_id: HOST_USER_ID, - secondary_user_id: TARGET_USER_ID, - audit_context: { - acting_workos_user_id: 'user_test_admin_link_credential', - consolidation_confirmed: true, - }, - }); - } finally { - await pool.query(`DELETE FROM organization_memberships WHERE workos_organization_id = $1`, [orgId]); - await pool.query(`DELETE FROM organizations WHERE workos_organization_id = $1`, [orgId]); + .set('X-Test-Admin-Access-Mechanism', accessMechanism) + .send({ workos_user_id: TARGET_USER_ID, consolidate }) + .expect(409) + .expect(({ body }) => expect(body.error).toBe('identity_mutation_disabled')); } + expect((await pool.query( + `SELECT * FROM identity_workos_users WHERE workos_user_id IN ($1, $2) ORDER BY workos_user_id`, + [HOST_USER_ID, TARGET_USER_ID], + )).rows).toEqual(before.rows); + expect(mockGetUser).not.toHaveBeenCalled(); }); - it('blocks a corporate-tier credential from being absorbed by an individual-tier host', async () => { - await pool.query( - `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, - workos_created_at, workos_updated_at, created_at, updated_at) - VALUES ($1, 'corporate@test.example', 'Corporate', 'User', true, NOW(), NOW(), NOW(), NOW())`, - [TARGET_USER_ID], - ); - const personalOrgId = 'org_test_link_personal_tier'; - const corporateOrgId = 'org_test_link_corporate_tier'; - await pool.query( - `INSERT INTO organizations - (workos_organization_id, name, is_personal, membership_tier, subscription_status, created_at, updated_at) - VALUES - ($1, 'Personal test workspace', true, 'individual_professional', 'active', NOW(), NOW()), - ($2, 'Corporate test workspace', false, NULL, 'active', NOW(), NOW())`, - [personalOrgId, corporateOrgId], - ); - await pool.query( - `UPDATE organizations - SET subscription_price_lookup_key = 'aao_membership_member' - WHERE workos_organization_id = $1`, - [corporateOrgId], - ); - await pool.query( - `INSERT INTO organization_memberships - (workos_user_id, workos_organization_id, email, role, created_at, updated_at) - VALUES - ($1, $2, 'host@test.example', 'member', NOW(), NOW()), - ($3, $4, 'corporate@test.example', 'member', NOW(), NOW())`, - [HOST_USER_ID, personalOrgId, TARGET_USER_ID, corporateOrgId], - ); - - try { - for (const body of [ - { workos_user_id: TARGET_USER_ID }, - { workos_user_id: TARGET_USER_ID, consolidate: true }, - ]) { - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send(body) - .expect(409); - expect(response.body).toMatchObject({ - error: 'corporate_membership_personal_host_conflict', - corporate_memberships: [{ - workos_organization_id: corporateOrgId, - membership_tier: 'company_icl', - }], - personal_memberships: [{ - workos_organization_id: personalOrgId, - membership_tier: 'individual_professional', - }], - }); - } - - const [membership, bindings] = await Promise.all([ - pool.query( - `SELECT workos_user_id FROM organization_memberships - WHERE workos_organization_id = $1`, - [corporateOrgId], - ), - pool.query<{ identity_id: string }>( - `SELECT identity_id FROM identity_workos_users - WHERE workos_user_id IN ($1, $2) - ORDER BY workos_user_id`, - [HOST_USER_ID, TARGET_USER_ID], - ), - ]); - expect(membership.rows).toEqual([{ workos_user_id: TARGET_USER_ID }]); - expect(bindings.rows).toHaveLength(2); - expect(bindings.rows[0].identity_id).not.toBe(bindings.rows[1].identity_id); - } finally { - await pool.query( - `DELETE FROM organization_memberships - WHERE workos_organization_id IN ($1, $2)`, - [personalOrgId, corporateOrgId], - ); - await pool.query( - `DELETE FROM organizations - WHERE workos_organization_id IN ($1, $2)`, - [personalOrgId, corporateOrgId], - ); - } + it('refuses before fetching or upserting a missing WorkOS credential', async () => { + await request(app) + .post(`/api/admin/users/${HOST_USER_ID}/credentials`) + .send({ workos_user_id: TARGET_USER_ID, consolidate: true }) + .expect(409) + .expect(({ body }) => expect(body.error).toBe('identity_mutation_disabled')); + expect(mockGetUser).not.toHaveBeenCalled(); + expect((await pool.query(`SELECT 1 FROM users WHERE workos_user_id = $1`, [TARGET_USER_ID])).rows).toEqual([]); }); }); + // Fixtures model existing bindings; the now-disabled merge route must never + // be used to prepare read-only listing or binding-only revocation tests. + async function bindExistingFixture() { + await pool.query( + `UPDATE identity_workos_users SET identity_id = ( + SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1 + ), is_primary = FALSE WHERE workos_user_id = $2`, + [HOST_USER_ID, TARGET_USER_ID], + ); + } + describe('GET /credentials', () => { it('lists bound credentials for the host\'s identity', async () => { await pool.query( @@ -407,10 +161,7 @@ describe('admin link / unlink credential', () => { VALUES ($1, 'target@test.example', 'Target', 'User', true, NOW(), NOW(), NOW(), NOW())`, [TARGET_USER_ID] ); - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID }) - .expect(201); + await bindExistingFixture(); const response = await request(app) .get(`/api/admin/users/${HOST_USER_ID}/credentials`) @@ -441,13 +192,23 @@ describe('admin link / unlink credential', () => { VALUES ($1, 'target@test.example', 'Target', 'User', true, NOW(), NOW(), NOW(), NOW())`, [TARGET_USER_ID] ); - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID }) - .expect(201); + await bindExistingFixture(); }); - it('unbinds a non-primary credential and gives it a fresh singleton identity', async () => { + it("unbinds a non-primary credential without moving either credential's organization authority", async () => { + await pool.query( + `INSERT INTO organizations (workos_organization_id, name) VALUES ($1, 'Unlink authority fixture')`, + [AUTHORITY_ORG_ID], + ); + await pool.query( + `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role) + VALUES ($1, $3, 'host@test.example', 'admin'), ($2, $3, 'target@test.example', 'member')`, + [HOST_USER_ID, TARGET_USER_ID, AUTHORITY_ORG_ID], + ); + const authorityBefore = await pool.query( + `SELECT * FROM organization_memberships WHERE workos_organization_id = $1 ORDER BY workos_user_id`, + [AUTHORITY_ORG_ID], + ); const beforeIdentity = await pool.query<{ identity_id: string }>( `SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1`, [HOST_USER_ID] @@ -458,6 +219,10 @@ describe('admin link / unlink credential', () => { .delete(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}`) .expect(200); expect(response.body.removed).toBe(true); + expect((await pool.query( + `SELECT * FROM organization_memberships WHERE workos_organization_id = $1 ORDER BY workos_user_id`, + [AUTHORITY_ORG_ID], + )).rows).toEqual(authorityBefore.rows); const after = await pool.query<{ identity_id: string; is_primary: boolean }>( `SELECT identity_id, is_primary FROM identity_workos_users WHERE workos_user_id = $1`, diff --git a/server/tests/integration/admin-promote-credential.test.ts b/server/tests/integration/admin-promote-credential.test.ts index 302da4ed7b0..e2b3b0dcd44 100644 --- a/server/tests/integration/admin-promote-credential.test.ts +++ b/server/tests/integration/admin-promote-credential.test.ts @@ -1,12 +1,4 @@ -/** - * Admin "promote credential to primary" integration test. - * - * Exercises POST /api/admin/users/:userId/credentials/:credentialId/promote. - * The new primary should: - * - hold all app-state previously on the old primary (org_memberships, etc.) - * - have is_primary = TRUE; the old primary, FALSE - * - record an audit row - */ +/** Identity promotion containment preserves both privilege directions. */ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; import request from 'supertest'; @@ -18,8 +10,10 @@ vi.hoisted(() => { process.env.WORKOS_COOKIE_PASSWORD ??= 'test-cookie-password-at-least-32-chars-long'; }); +const provider = vi.hoisted(() => ({ getUser: vi.fn(), createUser: vi.fn(), updateUser: vi.fn(), deleteUser: vi.fn() })); + vi.mock('../../src/auth/workos-client.js', () => { - const mockUserManagement = { getUser: vi.fn(), createUser: vi.fn(), deleteUser: vi.fn() }; + const mockUserManagement = provider; const mockWorkos = { userManagement: mockUserManagement }; return { workos: mockWorkos, getWorkos: () => mockWorkos }; }); @@ -34,10 +28,6 @@ vi.mock('../../src/middleware/auth.js', async (importOriginal) => { emailVerified: true, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), - // Tests can set X-Test-Admin-Identity header to simulate an admin - // who is bound to the same identity as the target — exercises the - // self-promote guard. - identityId: req.headers['x-test-admin-identity'] || undefined, }; next(); }; @@ -59,9 +49,12 @@ vi.mock('../../src/middleware/csrf.js', () => ({ csrfProtection: (_req: any, _res: any, next: any) => next(), })); -import { initializeDatabase, closeDatabase, getPool } from '../../src/db/client.js'; +import { initializeDatabase, closeDatabase } from '../../src/db/client.js'; import { runMigrations } from '../../src/db/migrate.js'; -import { HTTPServer } from '../../src/http.js'; +import express from 'express'; +import { handleEmailLinkVerification } from '../../src/routes/account-linking.js'; +import { createAdminUsersRouter } from '../../src/routes/admin/users.js'; +import { stopAuthTimers } from '../../src/middleware/auth.js'; const HOST_USER_ID = 'user_test_promote_host'; const TARGET_USER_ID = 'user_test_promote_target'; @@ -69,7 +62,6 @@ const HOST_ORG_ID = 'org_test_promote_host'; const TARGET_ORG_ID = 'org_test_promote_target'; describe('admin promote credential to primary', () => { - let server: HTTPServer; let app: any; let pool: Pool; @@ -78,19 +70,23 @@ describe('admin promote credential to primary', () => { connectionString: process.env.DATABASE_URL || 'postgresql://adcp:localdev@localhost:5432/adcp_test', }); await runMigrations(); - server = new HTTPServer(); - await server.start(0); - app = server.app; + // Isolate the verification route's durable limiter across repeated runs. + await pool.query(`DELETE FROM rate_limit_hits WHERE key LIKE 'verify-email-exec:%'`); + app = express(); + app.use(express.json()); + app.use('/api/admin/users', createAdminUsersRouter()); + handleEmailLinkVerification(app); }, 60000); afterAll(async () => { await cleanup(); - await server?.stop(); + stopAuthTimers(); await closeDatabase(); }); beforeEach(async () => { await cleanup(); + vi.clearAllMocks(); // Insert two users; trigger creates a singleton identity for each await pool.query( `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, @@ -123,299 +119,117 @@ describe('admin promote credential to primary', () => { ); } - /** - * Replicates the Ahmed shape: a host with one org, a bound target with - * its own org, target gets promoted to primary; both orgs end up on - * target's workos_user_id. - */ - async function setupBoundPair() { - // Host has Host Org + async function snapshotAuthority() { + const snapshot = await Promise.all([ + pool.query(`SELECT * FROM users WHERE workos_user_id IN ($1, $2) ORDER BY workos_user_id`, [HOST_USER_ID, TARGET_USER_ID]), + pool.query(`SELECT * FROM identity_workos_users WHERE workos_user_id IN ($1, $2) ORDER BY workos_user_id`, [HOST_USER_ID, TARGET_USER_ID]), + pool.query(`SELECT * FROM organization_memberships WHERE workos_user_id IN ($1, $2) ORDER BY workos_user_id, workos_organization_id`, [HOST_USER_ID, TARGET_USER_ID]), + pool.query(`SELECT * FROM organizations WHERE workos_organization_id IN ($1, $2) ORDER BY workos_organization_id`, [HOST_ORG_ID, TARGET_ORG_ID]), + ]); + return snapshot.map(result => result.rows); + } + + // Seed an existing linked identity directly. Containment must not require a + // destructive merge to succeed as part of test preparation. + async function setupBoundPair(hostRole: string, targetRole: string) { await pool.query( `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role, created_at, updated_at) - VALUES ($1, $2, 'host@test.example', 'admin', NOW(), NOW())`, - [HOST_USER_ID, HOST_ORG_ID] + VALUES ($1, $3, 'host@test.example', $4, NOW(), NOW()), + ($2, $3, 'target@test.example', $5, NOW(), NOW())`, + [HOST_USER_ID, TARGET_USER_ID, HOST_ORG_ID, hostRole, targetRole], ); - // Target has Target Org await pool.query( - `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role, created_at, updated_at) - VALUES ($1, $2, 'target@test.example', 'admin', NOW(), NOW())`, - [TARGET_USER_ID, TARGET_ORG_ID] + `UPDATE users SET primary_organization_id = $1 WHERE workos_user_id = $2`, + [HOST_ORG_ID, HOST_USER_ID], ); - // Bind target as non-primary under host's identity (mergeUsers does this); - // mergeUsers also moves target's data to host, so we set it up by hand - // post-bind to keep target_org membership on the target's workos_user_id. - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID, consolidate: true }) - .expect(201); - // After bind, target_org_membership was moved to HOST_USER_ID. Move it - // back to TARGET_USER_ID to simulate the post-resync Ahmed state where - // an org membership lands on the non-primary credential (because WorkOS - // org_membership webhook routed it to that user_id). await pool.query( - `UPDATE organization_memberships SET workos_user_id = $1 - WHERE workos_organization_id = $2`, - [TARGET_USER_ID, TARGET_ORG_ID] - ); - } - - it('promotes the target credential and moves the old primary\'s app-state forward', async () => { - await setupBoundPair(); - - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .expect(200); - - expect(response.body).toMatchObject({ - promoted: true, - previous_primary_id: HOST_USER_ID, - new_primary_id: TARGET_USER_ID, - }); - - // is_primary swapped - const bindings = await pool.query<{ workos_user_id: string; is_primary: boolean }>( - `SELECT workos_user_id, is_primary FROM identity_workos_users - WHERE workos_user_id IN ($1, $2) - ORDER BY is_primary DESC`, - [HOST_USER_ID, TARGET_USER_ID] - ); - expect(bindings.rows).toHaveLength(2); - expect(bindings.rows.find(r => r.workos_user_id === TARGET_USER_ID)?.is_primary).toBe(true); - expect(bindings.rows.find(r => r.workos_user_id === HOST_USER_ID)?.is_primary).toBe(false); - - // Both orgs now on TARGET_USER_ID (host's app-state moved forward; - // target's stayed since it was already there) - const memberships = await pool.query<{ workos_user_id: string }>( - `SELECT workos_user_id FROM organization_memberships - WHERE workos_organization_id IN ($1, $2) - ORDER BY workos_organization_id`, - [HOST_ORG_ID, TARGET_ORG_ID] + `UPDATE users SET primary_organization_id = $1 WHERE workos_user_id = $2`, + [TARGET_ORG_ID, TARGET_USER_ID], ); - expect(memberships.rows).toHaveLength(2); - expect(memberships.rows.every(r => r.workos_user_id === TARGET_USER_ID)).toBe(true); - }); - - /** - * Both credentials hold a membership in the same organization. The - * consolidation deletes the outgoing primary's row, so the endpoint must - * refuse without stated intent. - */ - async function setupOverlappingPair() { - await setupBoundPair(); - // Give the target its own membership in the host's org, so promoting - // would delete the host's row for that org rather than move it. await pool.query( - `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role, created_at, updated_at) - VALUES ($1, $2, 'target@test.example', 'member', NOW(), NOW())`, - [TARGET_USER_ID, HOST_ORG_ID] + `UPDATE identity_workos_users SET identity_id = ( + SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1 + ), is_primary = FALSE WHERE workos_user_id = $2`, + [HOST_USER_ID, TARGET_USER_ID], ); } - it('409s rather than silently deleting a membership the target already holds', async () => { - await setupOverlappingPair(); - - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .expect(409); - - expect(response.body).toMatchObject({ - consolidate_confirmation_required: true, - superseded_organization_ids: [HOST_ORG_ID], - superseded_working_group_ids: [], - }); - - // Nothing moved: the host is still primary and still holds its row. - const bindings = await pool.query<{ is_primary: boolean }>( - `SELECT is_primary FROM identity_workos_users WHERE workos_user_id = $1`, - [HOST_USER_ID] - ); - expect(bindings.rows[0].is_primary).toBe(true); - }); - - it('promotes once the caller confirms the consolidation', async () => { - await setupOverlappingPair(); - - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .send({ consolidate: true }) - .expect(200); - - const bindings = await pool.query<{ workos_user_id: string; is_primary: boolean }>( - `SELECT workos_user_id, is_primary FROM identity_workos_users - WHERE workos_user_id IN ($1, $2)`, - [HOST_USER_ID, TARGET_USER_ID] - ); - expect(bindings.rows.find(r => r.workos_user_id === TARGET_USER_ID)?.is_primary).toBe(true); - }); - - it('requires an identity-bearing admin to confirm destructive promotion', async () => { - await setupOverlappingPair(); - - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .set('X-Test-Admin-Access-Mechanism', 'static_admin_api_key') - .send({ consolidate: true }) - .expect(403); - expect(response.body.error).toBe('identity_bearing_admin_required'); - }); - - it('promotes without confirmation when the memberships do not overlap', async () => { - await setupBoundPair(); - - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .expect(200); + it.each([ + ['admin', 'member'], + ['member', 'admin'], + ])('preserves %s primary and %s target authority through concurrent promotion, consolidation and replay', async (hostRole, targetRole) => { + await setupBoundPair(hostRole, targetRole); + const before = await snapshotAuthority(); + const calls = [ + `/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`, + `/api/admin/users/${TARGET_USER_ID}/credentials/${HOST_USER_ID}/promote`, + `/api/admin/users/${HOST_USER_ID}/credentials`, + `/api/admin/users/${TARGET_USER_ID}/credentials`, + ]; + const invoke = (path: string, consolidate: boolean) => request(app).post(path) + .send({ workos_user_id: path.includes(HOST_USER_ID) ? TARGET_USER_ID : HOST_USER_ID, consolidate }) + .expect(409) + .expect(({ body }) => expect(body.error).toBe('identity_mutation_disabled')); + await Promise.all(calls.flatMap(path => [invoke(path, false), invoke(path, true)])); + for (const path of calls) await invoke(path, true); + expect(await snapshotAuthority()).toEqual(before); + for (const mock of Object.values(provider)) expect(mock).not.toHaveBeenCalled(); }); - it('blocks promotion away from a corporate primary after a personal credential was bound', async () => { + it.each([ + [HOST_USER_ID, TARGET_USER_ID], + [TARGET_USER_ID, HOST_USER_ID], + ])('refuses an in-flight member merge token from %s to %s and its replay', async (primaryId, targetId) => { + await setupBoundPair('admin', 'member'); + const token = `containment-${primaryId}-${targetId}`; await pool.query( - `UPDATE organizations - SET is_personal = false, membership_tier = 'company_standard', subscription_status = 'active' - WHERE workos_organization_id = $1`, - [HOST_ORG_ID], - ); - await pool.query( - `UPDATE organizations - SET is_personal = true, membership_tier = 'individual_professional', subscription_status = 'active' - WHERE workos_organization_id = $1`, - [TARGET_ORG_ID], - ); - await pool.query( - `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role, created_at, updated_at) - VALUES ($1, $2, 'host@test.example', 'member', NOW(), NOW()), - ($3, $4, 'target@test.example', 'member', NOW(), NOW())`, - [HOST_USER_ID, HOST_ORG_ID, TARGET_USER_ID, TARGET_ORG_ID], - ); - - // This is the supported direction: the corporate credential remains - // primary when the personal credential is bound. - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials`) - .send({ workos_user_id: TARGET_USER_ID, consolidate: true }) - .expect(201); - - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .expect(409); - expect(response.body).toMatchObject({ - error: 'corporate_membership_primary_required', - corporate_memberships: [{ - workos_organization_id: HOST_ORG_ID, - membership_tier: 'company_standard', - }], - }); - - const [corporateMembership, bindings] = await Promise.all([ - pool.query( - `SELECT workos_user_id FROM organization_memberships - WHERE workos_organization_id = $1`, - [HOST_ORG_ID], - ), - pool.query<{ workos_user_id: string; is_primary: boolean }>( - `SELECT workos_user_id, is_primary FROM identity_workos_users - WHERE workos_user_id IN ($1, $2)`, - [HOST_USER_ID, TARGET_USER_ID], - ), - ]); - expect(corporateMembership.rows).toEqual([{ workos_user_id: HOST_USER_ID }]); - expect(bindings.rows.find((row) => row.workos_user_id === HOST_USER_ID)?.is_primary).toBe(true); - expect(bindings.rows.find((row) => row.workos_user_id === TARGET_USER_ID)?.is_primary).toBe(false); + `INSERT INTO email_link_tokens (token, primary_workos_user_id, target_email, target_workos_user_id, expires_at) + VALUES ($1, $2, 'target@test.example', $3, NOW() + INTERVAL '1 hour')`, + [token, primaryId, targetId], + ); + const before = await snapshotAuthority(); + for (let attempt = 0; attempt < 2; attempt++) { + const response = await request(app).post('/verify-email-link') + .type('form').send({ token, consolidate: true }).expect(200); + expect(response.text).toContain('Verification Failed'); + expect(await snapshotAuthority()).toEqual(before); + } + expect((await pool.query(`SELECT status FROM email_link_tokens WHERE token = $1`, [token])).rows) + .toEqual([{ status: 'revoked' }]); + expect((await pool.query(`SELECT 1 FROM user_email_aliases WHERE workos_user_id IN ($1, $2)`, [HOST_USER_ID, TARGET_USER_ID])).rows) + .toEqual([]); + for (const mock of Object.values(provider)) expect(mock).not.toHaveBeenCalled(); }); - it('writes a promote_credential_to_primary audit row', async () => { - await setupBoundPair(); - await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .expect(200); - - const audit = await pool.query<{ details: any }>( - `SELECT details FROM registry_audit_log - WHERE action = 'promote_credential_to_primary' AND resource_id = $1 - ORDER BY created_at DESC LIMIT 1`, - [TARGET_USER_ID] - ); - expect(audit.rows).toHaveLength(1); - expect(audit.rows[0].details).toMatchObject({ - previous_primary_id: HOST_USER_ID, - new_primary_id: TARGET_USER_ID, - }); + it('preserves bare alias verification without moving authority or calling WorkOS', async () => { + await setupBoundPair('admin', 'member'); + const token = 'containment-bare-alias'; + await pool.query( + `INSERT INTO email_link_tokens (token, primary_workos_user_id, target_email, expires_at) + VALUES ($1, $2, 'alias@test.example', NOW() + INTERVAL '1 hour')`, + [token, HOST_USER_ID], + ); + const before = await snapshotAuthority(); + const response = await request(app).post('/verify-email-link') + .type('form').send({ token }).expect(200); + expect(response.text).toContain('Email Linked'); + expect(await snapshotAuthority()).toEqual(before); + expect((await pool.query(`SELECT email FROM user_email_aliases WHERE workos_user_id = $1`, [HOST_USER_ID])).rows) + .toEqual([{ email: 'alias@test.example' }]); + for (const mock of Object.values(provider)) expect(mock).not.toHaveBeenCalled(); }); - it('is idempotent: promoting an already-primary credential returns 200 with no change', async () => { - // Promote target so it's the primary - await setupBoundPair(); + it('refuses promotion without a current primary even for a static admin API key', async () => { + await setupBoundPair('admin', 'member'); + await pool.query(`UPDATE identity_workos_users SET is_primary = FALSE WHERE workos_user_id = $1`, [HOST_USER_ID]); + const before = await snapshotAuthority(); await request(app) .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .expect(200); - - // Calling again on the same credential should be a no-op - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .expect(200); - expect(response.body.promoted).toBe(true); - expect(response.body.message).toMatch(/already primary/i); - - // Bindings unchanged - const bindings = await pool.query<{ workos_user_id: string; is_primary: boolean }>( - `SELECT workos_user_id, is_primary FROM identity_workos_users - WHERE workos_user_id IN ($1, $2)`, - [HOST_USER_ID, TARGET_USER_ID] - ); - expect(bindings.rows.find(r => r.workos_user_id === TARGET_USER_ID)?.is_primary).toBe(true); - }); - - it('400s when the credentialId in the URL matches the host id', async () => { - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${HOST_USER_ID}/promote`) - .expect(400); - expect(response.body.error).toMatch(/must differ/i); - }); - - it('404s when the credential is not bound to the host\'s identity', async () => { - // Target is its own singleton identity, not bound to host - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .expect(404); - expect(response.body.error).toMatch(/not bound/i); - }); - - it('refuses when the admin is signed in as a member of the target identity', async () => { - await setupBoundPair(); - - // Find the host's identity id - const identityRow = await pool.query<{ identity_id: string }>( - `SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1`, - [HOST_USER_ID] - ); - const hostIdentityId = identityRow.rows[0].identity_id; - - // Simulate an admin whose identityId == host's identityId via the test - // header the auth mock reads. - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .set('X-Test-Admin-Identity', hostIdentityId) - .expect(409); - expect(response.body.error).toMatch(/own credential/i); - }); - - it('repairs an orphan-no-primary state by setting the target as primary directly', async () => { - await setupBoundPair(); - // Manually break the primary so the identity has no current primary - await pool.query( - `UPDATE identity_workos_users SET is_primary = FALSE WHERE workos_user_id = $1`, - [HOST_USER_ID] - ); - - const response = await request(app) - .post(`/api/admin/users/${HOST_USER_ID}/credentials/${TARGET_USER_ID}/promote`) - .expect(200); - expect(response.body.message).toMatch(/no current primary|invariant repaired/i); - - const bindings = await pool.query<{ workos_user_id: string; is_primary: boolean }>( - `SELECT workos_user_id, is_primary FROM identity_workos_users - WHERE workos_user_id IN ($1, $2)`, - [HOST_USER_ID, TARGET_USER_ID] - ); - expect(bindings.rows.find(r => r.workos_user_id === TARGET_USER_ID)?.is_primary).toBe(true); + .set('X-Test-Admin-Access-Mechanism', 'static_admin_api_key') + .send({ consolidate: true }) + .expect(409) + .expect(({ body }) => expect(body.error).toBe('identity_mutation_disabled')); + expect(await snapshotAuthority()).toEqual(before); + for (const mock of Object.values(provider)) expect(mock).not.toHaveBeenCalled(); }); }); diff --git a/server/tests/integration/authorization-epoch.test.ts b/server/tests/integration/authorization-epoch.test.ts index c6d8fc2f5a8..ba98a9f4b7f 100644 --- a/server/tests/integration/authorization-epoch.test.ts +++ b/server/tests/integration/authorization-epoch.test.ts @@ -7,8 +7,7 @@ * - bumping is monotonic per credential * - a CASCADE delete moves the fingerprint (so callers must compare for * inequality, not ordering) - * - mergeUsers bumps both credentials inside its own transaction - * - a failed mergeUsers leaves the fingerprint untouched + * - refused generic merges and promotions leave fingerprints untouched */ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; @@ -94,54 +93,53 @@ describe('Authorization epoch (migration 565)', () => { expect(after).toBe(''); }); - it('bumps both credentials when mergeUsers binds them to one identity', async () => { + it('preserves both fingerprints when generic merging is refused', async () => { const primaryId = await insertUser('merge_primary'); const secondaryId = await insertUser('merge_secondary'); + await bumpAuthorizationEpochs(pool, [primaryId, secondaryId]); + const before = await getAuthorizationFingerprint([primaryId, secondaryId]); - await mergeUsers(primaryId, secondaryId, primaryId); - - expect(await getAuthorizationFingerprint([primaryId])).toBe(`${primaryId}:1`); - expect(await getAuthorizationFingerprint([secondaryId])).toBe(`${secondaryId}:1`); - }); - - it('bumps both credentials when the primary is deleted and a secondary is promoted', async () => { - const primaryId = await insertUser('promote_primary'); - const secondaryId = await insertUser('promote_secondary'); - await mergeUsers(primaryId, secondaryId, primaryId); - - const promoted = await promoteSecondaryIfPrimaryDeleted(primaryId); + await expect(mergeUsers(primaryId, secondaryId, primaryId)).rejects.toMatchObject({ + code: 'identity_mutation_disabled', + }); - expect(promoted).toEqual({ promotedUserId: secondaryId }); - expect(await getAuthorizationFingerprint([primaryId])).toBe(`${primaryId}:2`); - expect(await getAuthorizationFingerprint([secondaryId])).toBe(`${secondaryId}:2`); + expect(await getAuthorizationFingerprint([primaryId, secondaryId])).toBe(before); }); - it('bumps every bound credential on promotion, not just the successor', async () => { - // Three credentials on one identity. Deleting the primary promotes the - // longest-bound secondary; the third credential's canonical routing moves - // to that successor too, so its sessions must be revalidated as well. + it('preserves every bound fingerprint when automatic promotion is refused', async () => { const primaryId = await insertUser('fanout_primary'); const successorId = await insertUser('fanout_successor'); const bystanderId = await insertUser('fanout_bystander'); - await mergeUsers(primaryId, successorId, primaryId); - await mergeUsers(primaryId, bystanderId, primaryId); - - const bystanderBefore = await getAuthorizationFingerprint([bystanderId]); + const credentials = [primaryId, successorId, bystanderId]; + // Model a historical multi-credential identity without invoking the + // disabled generic merge as test setup. + const original = await pool.query<{ identity_id: string }>( + `SELECT identity_id FROM identity_workos_users WHERE workos_user_id = ANY($1)`, + [[successorId, bystanderId]], + ); + await pool.query( + `UPDATE identity_workos_users + SET identity_id = (SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1), + is_primary = FALSE + WHERE workos_user_id = ANY($2)`, + [primaryId, [successorId, bystanderId]], + ); + await pool.query(`DELETE FROM identities WHERE id = ANY($1)`, [original.rows.map((row) => row.identity_id)]); + await bumpAuthorizationEpochs(pool, credentials); + const before = await getAuthorizationFingerprint(credentials); - const promoted = await promoteSecondaryIfPrimaryDeleted(primaryId); + await expect(promoteSecondaryIfPrimaryDeleted(primaryId)).rejects.toMatchObject({ + code: 'identity_mutation_disabled', + }); - expect(promoted).toEqual({ promotedUserId: successorId }); - expect(await getAuthorizationFingerprint([bystanderId])).not.toBe(bystanderBefore); + expect(await getAuthorizationFingerprint(credentials)).toBe(before); }); - it('leaves the fingerprint untouched when mergeUsers rolls back', async () => { - const secondaryId = await insertUser('rollback_secondary'); - const missingPrimaryId = `${TEST_USER_PREFIX}rollback_missing_primary`; - - await expect(mergeUsers(missingPrimaryId, secondaryId, secondaryId)).rejects.toThrow( - 'Primary user does not exist' - ); - + it('refuses before checking for a missing primary or changing the fingerprint', async () => { + const secondaryId = await insertUser('missing_secondary'); + await expect(mergeUsers(`${TEST_USER_PREFIX}missing_primary`, secondaryId, secondaryId)).rejects.toMatchObject({ + code: 'identity_mutation_disabled', + }); expect(await getAuthorizationFingerprint([secondaryId])).toBe(''); }); }); diff --git a/server/tests/integration/merge-bind-not-delete.test.ts b/server/tests/integration/merge-bind-not-delete.test.ts index 083632b173f..de818823f60 100644 --- a/server/tests/integration/merge-bind-not-delete.test.ts +++ b/server/tests/integration/merge-bind-not-delete.test.ts @@ -1,34 +1,38 @@ /** - * mergeUsers — bind, don't delete (Phase 2a) integration tests. - * - * The new contract: - * - All of the secondary user's app-state rows move to the primary (same - * as before). - * - The secondary WorkOS user stays alive in `users`. Each linked email is - * a real sign-in credential. - * - Both WorkOS users end up bound to the primary's identity. The - * secondary's binding is `is_primary = FALSE`. - * - The secondary's orphaned singleton identity gets dropped. + * Generic identity consolidation containment for #6827. The former merge + * success contract must stay disabled until authority/provenance are preserved. + * These fixtures deliberately include both conflicting and exclusive grants. */ - -import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; -import { initializeDatabase, closeDatabase, getPool } from '../../src/db/client.js'; -import { runMigrations } from '../../src/db/migrate.js'; -import { mergeUsers } from '../../src/db/user-merge-db.js'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import type { Pool } from 'pg'; - -const PRIMARY_ID = 'user_bind_test_primary'; -const SECONDARY_ID = 'user_bind_test_secondary'; -const ORG_ID = 'org_bind_test'; - -describe('mergeUsers (bind, don\'t delete)', () => { +import { closeDatabase, initializeDatabase } from '../../src/db/client.js'; +import { runMigrations } from '../../src/db/migrate.js'; +import { mergeUsers, previewUserMerge } from '../../src/db/user-merge-db.js'; +import { promoteSecondaryIfPrimaryDeleted } from '../../src/db/identity-db.js'; +import { bumpAuthorizationEpochs } from '../../src/db/authorization-epoch-db.js'; + +const JORDAN = 'user_containment_jordan'; +const SAM = 'user_containment_sam'; +const USERS = [JORDAN, SAM]; +const PINNACLE = 'org_containment_pinnacle'; +const PERSONAL = 'org_containment_sam'; +const ORGS = [PINNACLE, PERSONAL]; +const GROUP_SLUG = 'containment-governance'; +const refusal = { code: 'identity_mutation_disabled' }; + +describe('generic identity mutation containment preserves authority', () => { let pool: Pool; + let groupId: string; + let adminGroupId: string; beforeAll(async () => { pool = initializeDatabase({ connectionString: process.env.DATABASE_URL || 'postgresql://adcp:localdev@localhost:5432/adcp_test', }); await runMigrations(); + const adminGroup = await pool.query<{ id: string }>(`SELECT id FROM working_groups WHERE slug = 'aao-admin'`); + expect(adminGroup.rows).toHaveLength(1); + adminGroupId = adminGroup.rows[0].id; }, 60000); afterAll(async () => { @@ -39,420 +43,166 @@ describe('mergeUsers (bind, don\'t delete)', () => { beforeEach(async () => { await cleanup(); await pool.query( - `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, - workos_created_at, workos_updated_at, created_at, updated_at) - VALUES ($1, 'primary@test.example', 'Primary', 'User', true, NOW(), NOW(), NOW(), NOW()), - ($2, 'secondary@test.example', 'Secondary', 'User', true, NOW(), NOW(), NOW(), NOW())`, - [PRIMARY_ID, SECONDARY_ID] + `INSERT INTO organizations (workos_organization_id, name, subscription_status, stripe_subscription_id) + VALUES ($1, 'Pinnacle Agency', 'active', 'sub_containment_pinnacle'), + ($2, 'Sam personal workspace', NULL, NULL)`, ORGS, ); await pool.query( - `INSERT INTO organizations (workos_organization_id, name, created_at, updated_at) - VALUES ($1, 'Test Org', NOW(), NOW()) - ON CONFLICT (workos_organization_id) DO NOTHING`, - [ORG_ID] + `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, + primary_organization_id, workos_created_at, workos_updated_at) + VALUES ($1, 'jordan@pinnacle.example', 'Jordan', 'Ochoa', true, $3, NOW(), NOW()), + ($2, 'sam@pinnacle.example', 'Sam', 'Adeyemi', true, $4, NOW(), NOW())`, + [JORDAN, SAM, PINNACLE, PERSONAL], ); - }); - - async function cleanup() { - // addie_threads.person_id has no CASCADE, and person_relationships has no - // FK to users(workos_user_id), so deleting users wouldn't reach either. - // Drop the threads first so the person_relationships DELETE can succeed. await pool.query( - `DELETE FROM addie_threads - WHERE person_id IN ( - SELECT id FROM person_relationships WHERE workos_user_id IN ($1, $2) - )`, - [PRIMARY_ID, SECONDARY_ID] + `INSERT INTO organization_memberships + (workos_user_id, workos_organization_id, workos_membership_id, email, role) + VALUES ($1, $3, 'om_containment_jordan', 'jordan@pinnacle.example', 'owner'), + ($2, $3, 'om_containment_sam_member', 'sam@pinnacle.example', 'member'), + ($2, $4, 'om_containment_sam_personal', 'sam@pinnacle.example', 'owner')`, + [JORDAN, SAM, PINNACLE, PERSONAL], ); - await pool.query( - `DELETE FROM person_relationships WHERE workos_user_id IN ($1, $2)`, - [PRIMARY_ID, SECONDARY_ID] + const group = await pool.query<{ id: string }>( + `INSERT INTO working_groups (name, slug, is_private, committee_type) + VALUES ('Governance containment fixture', $1, true, 'governance') RETURNING id`, [GROUP_SLUG], ); + groupId = group.rows[0].id; + // The active aao-admin membership is the actual platform-admin grant. + // Sam's inactive duplicate must never replace or absorb Jordan's grant. await pool.query( - `DELETE FROM addie_prompt_telemetry WHERE workos_user_id IN ($1, $2)`, - [PRIMARY_ID, SECONDARY_ID] + `INSERT INTO working_group_memberships (working_group_id, workos_user_id, status) + VALUES ($1, $3, 'active'), ($1, $4, 'inactive'), + ($2, $3, 'active'), ($2, $4, 'inactive')`, + [adminGroupId, groupId, JORDAN, SAM], ); await pool.query( - `DELETE FROM admin_module_completions WHERE workos_user_id IN ($1, $2)`, - [PRIMARY_ID, SECONDARY_ID] + `INSERT INTO aao_admin_access_events + (event_type, actor_user_id, target_user_id, mechanism, actor_authorization_mechanism, reason) + VALUES ('granted', $1, $1, 'aao_admin_working_group', 'development', 'Containment fixture grant')`, + [JORDAN], ); + await pool.query(`INSERT INTO working_group_leaders (working_group_id, user_id) VALUES ($1, $2)`, [groupId, JORDAN]); await pool.query( - `DELETE FROM learner_progress WHERE workos_user_id IN ($1, $2)`, - [PRIMARY_ID, SECONDARY_ID] + `INSERT INTO committee_interest (working_group_id, workos_user_id, interest_level) + VALUES ($1, $2, 'leader'), ($1, $3, 'participant')`, [groupId, JORDAN, SAM], ); await pool.query( - `DELETE FROM organization_memberships WHERE workos_user_id IN ($1, $2)`, - [PRIMARY_ID, SECONDARY_ID] - ); - await pool.query(`DELETE FROM users WHERE workos_user_id IN ($1, $2)`, [PRIMARY_ID, SECONDARY_ID]); - await pool.query(`DELETE FROM organizations WHERE workos_organization_id = $1`, [ORG_ID]); - } - - it('keeps the secondary WorkOS user alive after merge', async () => { - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); - - const result = await pool.query( - `SELECT workos_user_id FROM users WHERE workos_user_id IN ($1, $2)`, - [PRIMARY_ID, SECONDARY_ID] + `INSERT INTO working_group_topic_subscriptions (working_group_id, workos_user_id, topic_slugs) + VALUES ($1, $2, ARRAY['governance']), ($1, $3, ARRAY['learning'])`, [groupId, JORDAN, SAM], ); - expect(result.rows.map(r => r.workos_user_id).sort()).toEqual([PRIMARY_ID, SECONDARY_ID].sort()); - }); - - it('binds both WorkOS users to one identity, with the secondary marked non-primary', async () => { - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); - - const result = await pool.query<{ workos_user_id: string; identity_id: string; is_primary: boolean }>( - `SELECT workos_user_id, identity_id, is_primary - FROM identity_workos_users - WHERE workos_user_id IN ($1, $2) - ORDER BY workos_user_id`, - [PRIMARY_ID, SECONDARY_ID] - ); - - expect(result.rows).toHaveLength(2); - const [primary, secondary] = result.rows.sort((a, b) => - a.workos_user_id === PRIMARY_ID ? -1 : 1 - ); - expect(primary.workos_user_id).toBe(PRIMARY_ID); - expect(primary.is_primary).toBe(true); - expect(secondary.workos_user_id).toBe(SECONDARY_ID); - expect(secondary.is_primary).toBe(false); - expect(primary.identity_id).toBe(secondary.identity_id); - }); - - it('drops the secondary user\'s orphaned singleton identity', async () => { - // Capture the secondary's pre-merge identity id, then verify it's gone after. - const before = await pool.query<{ identity_id: string }>( - `SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1`, - [SECONDARY_ID] - ); - const secondaryIdentityBefore = before.rows[0].identity_id; - - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); - - const after = await pool.query( - `SELECT 1 FROM identities WHERE id = $1`, - [secondaryIdentityBefore] - ); - expect(after.rows).toEqual([]); - }); - - it('moves organization memberships from secondary to primary (existing read paths still work)', async () => { await pool.query( - `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role, created_at, updated_at) - VALUES ($1, $2, 'secondary@test.example', 'member', NOW(), NOW())`, - [SECONDARY_ID, ORG_ID] - ); - - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); - - const result = await pool.query<{ workos_user_id: string }>( - `SELECT workos_user_id FROM organization_memberships WHERE workos_organization_id = $1`, - [ORG_ID] - ); - expect(result.rows).toHaveLength(1); - expect(result.rows[0].workos_user_id).toBe(PRIMARY_ID); - }); - - it('preserves completed certification progress when the primary has a lower-state duplicate', async () => { - const primaryProgress = await pool.query<{ id: string }>( - `INSERT INTO learner_progress - (workos_user_id, module_id, status, started_at, attempts, addie_thread_id) - VALUES ($1, 'B1', 'in_progress', NOW() - INTERVAL '1 day', 1, 'primary-thread') - RETURNING id`, - [PRIMARY_ID] - ); - const secondaryProgress = await pool.query<{ id: string }>( - `INSERT INTO learner_progress - (workos_user_id, module_id, status, started_at, completed_at, score, - attempts, addie_thread_id) - VALUES ($1, 'B1', 'completed', NOW() - INTERVAL '2 days', - NOW() - INTERVAL '1 hour', '{"knowledge": 92}', 2, - 'secondary-thread') - RETURNING id`, - [SECONDARY_ID] + `INSERT INTO subscription_line_items + (workos_organization_id, stripe_subscription_id, stripe_subscription_item_id, price_id, quantity) + VALUES ($1, 'sub_containment_pinnacle', 'si_containment_pinnacle', 'price_containment', 10)`, [PINNACLE], ); await pool.query( - `INSERT INTO learner_progress - (workos_user_id, module_id, status, started_at, attempts) - VALUES ($1, 'B2', 'in_progress', NOW(), 1)`, - [SECONDARY_ID] + `INSERT INTO user_email_aliases (workos_user_id, email) + VALUES ($1, 'jordan-alias@pinnacle.example'), ($2, 'sam-alias@pinnacle.example')`, USERS, ); await pool.query( `INSERT INTO learner_progress (workos_user_id, module_id, status) - VALUES ($1, 'B3', 'not_started'), ($2, 'B3', 'failed')`, - [PRIMARY_ID, SECONDARY_ID] - ); - await pool.query( - `INSERT INTO admin_module_completions - (workos_user_id, module_id, admin_user_id, addie_thread_id, score, - learner_progress_id) - VALUES ($1, 'B1', 'admin_test', 'secondary-thread', - '{"knowledge": 92}', $2)`, - [SECONDARY_ID, secondaryProgress.rows[0].id] - ); - - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); - - const progress = await pool.query<{ - id: string; - module_id: string; - status: string; - score: Record<string, number> | null; - attempts: number; - addie_thread_id: string | null; - }>( - `SELECT id, module_id, status, score, attempts, addie_thread_id - FROM learner_progress - WHERE workos_user_id = $1 - ORDER BY module_id`, - [PRIMARY_ID] - ); - expect(progress.rows).toHaveLength(3); - expect(progress.rows[0]).toMatchObject({ - id: primaryProgress.rows[0].id, - module_id: 'B1', - status: 'completed', - score: { knowledge: 92 }, - attempts: 3, - addie_thread_id: 'secondary-thread', - }); - expect(progress.rows[1]).toMatchObject({ - module_id: 'B2', - status: 'in_progress', - }); - expect(progress.rows[2]).toMatchObject({ - module_id: 'B3', - status: 'failed', - }); - - const secondaryRows = await pool.query( - `SELECT 1 FROM learner_progress WHERE workos_user_id = $1`, - [SECONDARY_ID] - ); - expect(secondaryRows.rows).toHaveLength(0); - - const audit = await pool.query<{ - workos_user_id: string; - learner_progress_id: string; - }>( - `SELECT workos_user_id, learner_progress_id - FROM admin_module_completions - WHERE module_id = 'B1' AND workos_user_id = $1`, - [PRIMARY_ID] - ); - expect(audit.rows).toEqual([{ - workos_user_id: PRIMARY_ID, - learner_progress_id: primaryProgress.rows[0].id, - }]); - }); - - it('records a merge_user audit row', async () => { - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); - - const result = await pool.query<{ resource_id: string; details: { primary_user_id: string; secondary_user_id: string } }>( - `SELECT resource_id, details - FROM registry_audit_log - WHERE action = 'merge_user' AND resource_id = $1 - ORDER BY created_at DESC LIMIT 1`, - [SECONDARY_ID] - ); - - expect(result.rows).toHaveLength(1); - expect(result.rows[0].details.primary_user_id).toBe(PRIMARY_ID); - expect(result.rows[0].details.secondary_user_id).toBe(SECONDARY_ID); - }); - - it('throws if the primary lacks an identity binding (Phase 1 trigger should make this impossible)', async () => { - // Manually break the invariant to confirm we fail loud rather than - // silently producing partial state. The AFTER INSERT trigger normally - // guarantees this can't happen. - await pool.query(`DELETE FROM identity_workos_users WHERE workos_user_id = $1`, [PRIMARY_ID]); - - await expect(mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID)).rejects.toThrow( - /no identity binding/ - ); - - // ROLLBACK should leave the secondary intact (no partial bind). - const secondaryStill = await pool.query( - `SELECT 1 FROM users WHERE workos_user_id = $1`, - [SECONDARY_ID] - ); - expect(secondaryStill.rows).toHaveLength(1); - }); - - it('repoints addie_threads and person_events to the primary\'s person_relationship before deleting the secondary\'s', async () => { - // Both users have a person_relationships row (the path that previously - // failed with FK 23503 on addie_threads_person_id_fkey). - const primaryRel = await pool.query<{ id: string }>( - `INSERT INTO person_relationships (workos_user_id, email, stage) - VALUES ($1, 'primary@test.example', 'exploring') RETURNING id`, - [PRIMARY_ID] - ); - const secondaryRel = await pool.query<{ id: string }>( - `INSERT INTO person_relationships (workos_user_id, email, stage) - VALUES ($1, 'secondary@test.example', 'exploring') RETURNING id`, - [SECONDARY_ID] - ); - const primaryRelId = primaryRel.rows[0].id; - const secondaryRelId = secondaryRel.rows[0].id; - - // Thread keyed to secondary's person_relationship — the row that hit FK - // violations under the old code path. - const thread = await pool.query<{ thread_id: string }>( - `INSERT INTO addie_threads (channel, external_id, user_type, person_id) - VALUES ('slack', 'C_test_merge_pr', 'slack', $1) RETURNING thread_id`, - [secondaryRelId] - ); - await pool.query( - `INSERT INTO person_events (person_id, event_type, data) - VALUES ($1, 'message_received', '{}'::jsonb)`, - [secondaryRelId] + VALUES ($1, 'B1', 'completed'), ($2, 'B1', 'not_started')`, USERS, ); - - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); - - const threadAfter = await pool.query<{ person_id: string }>( - `SELECT person_id FROM addie_threads WHERE thread_id = $1`, - [thread.rows[0].thread_id] - ); - expect(threadAfter.rows[0].person_id).toBe(primaryRelId); - - const eventAfter = await pool.query<{ count: string }>( - `SELECT COUNT(*) as count FROM person_events WHERE person_id = $1 AND event_type = 'message_received'`, - [primaryRelId] - ); - expect(parseInt(eventAfter.rows[0].count, 10)).toBe(1); - - const secondaryRelAfter = await pool.query( - `SELECT 1 FROM person_relationships WHERE id = $1`, - [secondaryRelId] - ); - expect(secondaryRelAfter.rows).toHaveLength(0); + await bumpAuthorizationEpochs(pool, USERS); }); - it('refreshes person_relationships.email from users.email for the surviving primary row (promote case)', async () => { - // Promote shape: primary is the freshly-added credential with no - // person_relationships row of its own; secondary owns the row with the - // old email. After mergeUsers, the row is re-pointed to primary and its - // email column should reflect the primary's current users.email. - await pool.query( - `INSERT INTO person_relationships (workos_user_id, email, stage) - VALUES ($1, 'secondary@test.example', 'exploring')`, - [SECONDARY_ID] - ); + async function cleanup() { + await pool.query(`DELETE FROM aao_admin_access_events WHERE target_user_id = ANY($1)`, [USERS]); + await pool.query(`DELETE FROM working_group_memberships WHERE workos_user_id = ANY($1)`, [USERS]); + await pool.query(`DELETE FROM working_groups WHERE slug = $1`, [GROUP_SLUG]); + await pool.query(`DELETE FROM organization_memberships WHERE workos_user_id = ANY($1)`, [USERS]); + await pool.query(`DELETE FROM users WHERE workos_user_id = ANY($1)`, [USERS]); + await pool.query(`DELETE FROM organizations WHERE workos_organization_id = ANY($1)`, [ORGS]); + } - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); + async function snapshot() { + const tables: Record<string, unknown> = {}; + const selections: [string, string, string[]][] = [ + ['users', 'workos_user_id = ANY($1)', USERS], + ['identities', 'id IN (SELECT identity_id FROM identity_workos_users WHERE workos_user_id = ANY($1))', USERS], + ['identity_workos_users', 'workos_user_id = ANY($1)', USERS], + ['authorization_epochs', 'workos_user_id = ANY($1)', USERS], + ['organization_memberships', 'workos_user_id = ANY($1)', USERS], + ['organizations', 'workos_organization_id = ANY($1)', ORGS], + ['working_group_memberships', 'workos_user_id = ANY($1)', USERS], + ['working_group_leaders', 'user_id = ANY($1)', USERS], + ['committee_interest', 'workos_user_id = ANY($1)', USERS], + ['working_group_topic_subscriptions', 'workos_user_id = ANY($1)', USERS], + ['subscription_line_items', 'workos_organization_id = ANY($1)', ORGS], + ['aao_admin_access_events', 'target_user_id = ANY($1)', USERS], + ['user_email_aliases', 'workos_user_id = ANY($1)', USERS], + ['learner_progress', 'workos_user_id = ANY($1)', USERS], + ]; + // Compare complete rows, including timestamps, ownership, roles and IDs. + for (const [table, where, ids] of selections) { + const result = await pool.query( + `SELECT row_to_json(row) AS row FROM (SELECT * FROM ${table} WHERE ${where}) row + ORDER BY row_to_json(row)::text`, [ids], + ); + tables[table] = result.rows.map(({ row }) => row); + expect(result.rows.length, `${table} fixture must exercise real rows`).toBeGreaterThan(0); + } + return tables; + } - const rel = await pool.query<{ workos_user_id: string; email: string }>( - `SELECT workos_user_id, email FROM person_relationships WHERE workos_user_id = $1`, - [PRIMARY_ID] - ); - expect(rel.rows).toHaveLength(1); - expect(rel.rows[0].email).toBe('primary@test.example'); + it.each([ + ['privilege gain', SAM, JORDAN], + ['privilege loss', JORDAN, SAM], + ])('refuses the %s direction without moving or deleting any authority', async (_direction, primary, secondary) => { + const before = await snapshot(); + await expect(mergeUsers(primary, secondary, JORDAN)).rejects.toMatchObject(refusal); + expect(await snapshot()).toEqual(before); }); - it('refreshes person_relationships.email for primary\'s pre-existing row when its email is stale (merge case)', async () => { - // Both users have a person_relationships row, and primary's row carries a - // stale email that pre-dates the credential swap. After mergeUsers, the - // secondary's row is deleted and the primary's row keeps its id but its - // email column gets re-derived from users.email. - const primaryRel = await pool.query<{ id: string }>( - `INSERT INTO person_relationships (workos_user_id, email, stage) - VALUES ($1, 'stale-primary@test.example', 'exploring') RETURNING id`, - [PRIMARY_ID] + it.each([ + ['privileged to ordinary', JORDAN, SAM], + ['ordinary to privileged', SAM, JORDAN], + ])('refuses automatic promotion from %s with historical bindings', async (_direction, primary, secondary) => { + const original = await pool.query<{ identity_id: string }>( + `SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1`, [secondary], ); + // Historical bindings are fixture input, not permission to transfer grants. await pool.query( - `INSERT INTO person_relationships (workos_user_id, email, stage) - VALUES ($1, 'secondary@test.example', 'exploring')`, - [SECONDARY_ID] + `UPDATE identity_workos_users + SET identity_id = (SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1), + is_primary = FALSE + WHERE workos_user_id = $2`, [primary, secondary], ); + await pool.query(`DELETE FROM identities WHERE id = $1`, [original.rows[0].identity_id]); + const before = await snapshot(); - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); + await expect(promoteSecondaryIfPrimaryDeleted(primary)).rejects.toMatchObject(refusal); - const rel = await pool.query<{ id: string; email: string }>( - `SELECT id, email FROM person_relationships WHERE workos_user_id = $1`, - [PRIMARY_ID] - ); - expect(rel.rows).toHaveLength(1); - expect(rel.rows[0].id).toBe(primaryRel.rows[0].id); - expect(rel.rows[0].email).toBe('primary@test.example'); + expect(await snapshot()).toEqual(before); }); - it('refreshes organization_memberships.email from users.email when it has drifted', async () => { - await pool.query( - `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role, created_at, updated_at) - VALUES ($1, $2, 'stale-primary@test.example', 'member', NOW(), NOW())`, - [PRIMARY_ID, ORG_ID] - ); - - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); - - const row = await pool.query<{ email: string }>( - `SELECT email FROM organization_memberships WHERE workos_user_id = $1 AND workos_organization_id = $2`, - [PRIMARY_ID, ORG_ID] - ); - expect(row.rows).toHaveLength(1); - expect(row.rows[0].email).toBe('primary@test.example'); + it('does not permit explicit consolidation or primary-promotion options to bypass refusal', async () => { + const before = await snapshot(); + await expect(mergeUsers(SAM, JORDAN, JORDAN, { + ensurePrimaryFlag: true, + auditContext: { consolidation_confirmed: true, consolidate: true }, + })).rejects.toMatchObject(refusal); + expect(await snapshot()).toEqual(before); }); - it('refreshes email even on the consolidate path when secondary has app-state to move', async () => { - // Consolidate shape: secondary owns app-state (telemetry rows + a stale - // relationship row) and primary is the newly-bound credential. The merge - // moves data forward AND refreshes the denormalized email on the - // surviving primary row. This proves the refresh isn't short-circuited - // by any of the other merge branches. - await pool.query( - `INSERT INTO addie_prompt_telemetry (workos_user_id, rule_id, shown_count, clicked_count) - VALUES ($1, 'secondary_rule', 3, 1)`, - [SECONDARY_ID] - ); - await pool.query( - `INSERT INTO person_relationships (workos_user_id, email, stage) - VALUES ($1, 'secondary@test.example', 'exploring')`, - [SECONDARY_ID] - ); - - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); - - const rel = await pool.query<{ email: string }>( - `SELECT email FROM person_relationships WHERE workos_user_id = $1`, - [PRIMARY_ID] - ); - expect(rel.rows[0].email).toBe('primary@test.example'); - - const telemetry = await pool.query<{ rule_id: string }>( - `SELECT rule_id FROM addie_prompt_telemetry WHERE workos_user_id = $1`, - [PRIMARY_ID] - ); - expect(telemetry.rows.map(r => r.rule_id)).toEqual(['secondary_rule']); + it('preserves all authority under concurrent, reversed and replayed calls', async () => { + const before = await snapshot(); + const attempts = Array.from({ length: 12 }, (_, index) => index % 3 === 0 + ? promoteSecondaryIfPrimaryDeleted(index % 2 ? SAM : JORDAN) + : mergeUsers(index % 2 ? SAM : JORDAN, index % 2 ? JORDAN : SAM, JORDAN, { ensurePrimaryFlag: true })); + for (const result of await Promise.allSettled(attempts)) { + expect(result.status).toBe('rejected'); + if (result.status === 'rejected') expect(result.reason).toMatchObject(refusal); + } + await expect(mergeUsers(SAM, JORDAN, JORDAN)).rejects.toMatchObject(refusal); + expect(await snapshot()).toEqual(before); }); - it('merges addie_prompt_telemetry on (workos_user_id, rule_id), keeping primary\'s row when both have one for the same rule', async () => { - // Same rule on both — primary's counters survive, secondary's are dropped. - await pool.query( - `INSERT INTO addie_prompt_telemetry (workos_user_id, rule_id, shown_count, clicked_count) - VALUES ($1, 'shared_rule', 5, 2), - ($2, 'shared_rule', 9, 4), - ($2, 'unique_rule', 1, 0)`, - [PRIMARY_ID, SECONDARY_ID] - ); - - await mergeUsers(PRIMARY_ID, SECONDARY_ID, PRIMARY_ID); - - const after = await pool.query<{ rule_id: string; shown_count: number; clicked_count: number }>( - `SELECT rule_id, shown_count, clicked_count FROM addie_prompt_telemetry - WHERE workos_user_id = $1 ORDER BY rule_id`, - [PRIMARY_ID] - ); - expect(after.rows).toHaveLength(2); - expect(after.rows[0]).toMatchObject({ rule_id: 'shared_rule', shown_count: 5, clicked_count: 2 }); - expect(after.rows[1]).toMatchObject({ rule_id: 'unique_rule', shown_count: 1, clicked_count: 0 }); - - const secondaryAfter = await pool.query( - `SELECT 1 FROM addie_prompt_telemetry WHERE workos_user_id = $1`, - [SECONDARY_ID] - ); - expect(secondaryAfter.rows).toHaveLength(0); + it('keeps read-only detection available without changing authority', async () => { + const before = await snapshot(); + const preview = await previewUserMerge(SAM, JORDAN); + expect(preview.tables).toContainEqual({ table_name: 'organization_memberships', row_count: 1 }); + expect(preview.tables).toContainEqual({ table_name: 'working_group_memberships', row_count: 2 }); + expect(await snapshot()).toEqual(before); }); }); diff --git a/server/tests/integration/workos-user-deleted-primary.test.ts b/server/tests/integration/workos-user-deleted-primary.test.ts index 2e68d0276f0..f00ccc9aa28 100644 --- a/server/tests/integration/workos-user-deleted-primary.test.ts +++ b/server/tests/integration/workos-user-deleted-primary.test.ts @@ -1,28 +1,20 @@ /** - * WorkOS user.deleted webhook — primary-bound user safety. - * - * Issue adcontextprotocol/adcp#3718: deleting the primary binding on a - * multi-credential identity (e.g. GDPR/CCPA erasure via WorkOS dashboard) - * leaves the identity with zero primaries after the CASCADE on - * identity_workos_users.workos_user_id fires. `attachIdentityId` then - * resolves the surviving secondary to a NULL primary and skips the id-swap, - * so they sign in to an empty workspace. - * - * Mitigation: promote the longest-bound surviving secondary to primary in - * the same transaction, BEFORE the CASCADE runs. This test exercises the - * `promoteSecondaryIfPrimaryDeleted` helper end-to-end against a real DB - * and asserts the wiring in the user.deleted handler. + * Automatic promotion containment: historical multi-credential bindings must + * not change primary routing when a provider deletes a credential. Ordinary + * deletion remains valid and must not silently promote a surviving credential. */ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; import { initializeDatabase, closeDatabase } from '../../src/db/client.js'; import { runMigrations } from '../../src/db/migrate.js'; -import { promoteSecondaryIfPrimaryDeleted } from '../../src/db/identity-db.js'; +import { deleteIdentityCredential, promoteSecondaryIfPrimaryDeleted } from '../../src/db/identity-db.js'; +import { bumpAuthorizationEpochs, getAuthorizationFingerprint } from '../../src/db/authorization-epoch-db.js'; import type { Pool } from 'pg'; const TEST_USER_PREFIX = 'user_wh_deleted_test_'; +const TEST_ORG = 'org_wh_deleted_test_pinnacle'; -describe('user.deleted: promote secondary before CASCADE (#3718)', () => { +describe('user.deleted: automatic promotion containment (#6827)', () => { let pool: Pool; beforeAll(async () => { @@ -34,14 +26,22 @@ describe('user.deleted: promote secondary before CASCADE (#3718)', () => { }, 60000); afterAll(async () => { - await pool.query(`DELETE FROM users WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); + await cleanup(); await closeDatabase(); }); beforeEach(async () => { - await pool.query(`DELETE FROM users WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); + await cleanup(); }); + async function cleanup() { + await pool.query(`DELETE FROM organization_memberships WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); + await pool.query(`DELETE FROM working_group_memberships WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); + await pool.query(`DELETE FROM working_group_leaders WHERE user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); + await pool.query(`DELETE FROM users WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); + await pool.query(`DELETE FROM organizations WHERE workos_organization_id = $1`, [TEST_ORG]); + } + async function insertUser(suffix: string, email: string): Promise<string> { const userId = `${TEST_USER_PREFIX}${suffix}`; await pool.query( @@ -90,88 +90,152 @@ describe('user.deleted: promote secondary before CASCADE (#3718)', () => { return identityId; } - it('promotes the longest-bound secondary to primary when the primary is deleted', async () => { - const primary = await insertUser('promote_p', 'primary@test.example'); - const olderSecondary = await insertUser('promote_s1', 'older-secondary@test.example'); - const newerSecondary = await insertUser('promote_s2', 'newer-secondary@test.example'); - - // older binds first (smaller bound_at offset), newer binds later + it('refuses to choose among historical secondary credentials without changing bindings', async () => { + const primary = await insertUser('primary', 'jordan@pinnacle.example'); + const olderSecondary = await insertUser('older', 'sam@pinnacle.example'); + const newerSecondary = await insertUser('newer', 'maya@pinnacle.example'); const identityId = await bindAsSecondary(primary, olderSecondary, 1); await bindAsSecondary(primary, newerSecondary, 2); + const before = await pool.query( + `SELECT * FROM identity_workos_users WHERE identity_id = $1 ORDER BY workos_user_id`, [identityId], + ); - const result = await promoteSecondaryIfPrimaryDeleted(primary); - - expect(result).not.toBeNull(); - expect(result!.promotedUserId).toBe(olderSecondary); + await expect(promoteSecondaryIfPrimaryDeleted(primary)).rejects.toMatchObject({ + code: 'identity_mutation_disabled', + }); - // The deleted user is no longer primary; the older secondary is. - const primaries = await pool.query<{ workos_user_id: string; is_primary: boolean }>( - `SELECT workos_user_id, is_primary FROM identity_workos_users - WHERE identity_id = $1 - ORDER BY workos_user_id`, - [identityId] + const after = await pool.query( + `SELECT * FROM identity_workos_users WHERE identity_id = $1 ORDER BY workos_user_id`, [identityId], ); - const byUser = Object.fromEntries(primaries.rows.map(r => [r.workos_user_id, r.is_primary])); - expect(byUser[primary]).toBe(false); - expect(byUser[olderSecondary]).toBe(true); - expect(byUser[newerSecondary]).toBe(false); - - // Exactly one primary on the identity — the partial unique index holds. - const primaryCount = primaries.rows.filter(r => r.is_primary).length; - expect(primaryCount).toBe(1); + expect(after.rows).toEqual(before.rows); }); - it('returns null and leaves state untouched when the identity has no secondaries', async () => { - const onlyUser = await insertUser('solo', 'solo@test.example'); - - const result = await promoteSecondaryIfPrimaryDeleted(onlyUser); - - expect(result).toBeNull(); - - // The original primary binding is unchanged. - const row = await pool.query<{ is_primary: boolean }>( - `SELECT is_primary FROM identity_workos_users WHERE workos_user_id = $1`, - [onlyUser] + it.each(['primary', 'secondary'])('refuses %s promotion calls, including replays', async (target) => { + const primary = await insertUser('replay_primary', 'jordan-replay@pinnacle.example'); + const secondary = await insertUser('replay_secondary', 'sam-replay@pinnacle.example'); + const identityId = await bindAsSecondary(primary, secondary, 1); + const before = await pool.query( + `SELECT * FROM identity_workos_users WHERE identity_id = $1 ORDER BY workos_user_id`, [identityId], + ); + const userId = target === 'primary' ? primary : secondary; + for (const result of await Promise.allSettled(Array.from({ length: 8 }, () => promoteSecondaryIfPrimaryDeleted(userId)))) { + expect(result.status).toBe('rejected'); + if (result.status === 'rejected') expect(result.reason).toMatchObject({ code: 'identity_mutation_disabled' }); + } + await expect(promoteSecondaryIfPrimaryDeleted(userId)).rejects.toMatchObject({ code: 'identity_mutation_disabled' }); + const after = await pool.query( + `SELECT * FROM identity_workos_users WHERE identity_id = $1 ORDER BY workos_user_id`, [identityId], ); - expect(row.rows[0].is_primary).toBe(true); + expect(after.rows).toEqual(before.rows); }); - it('returns null when the deleted user is not a primary binding', async () => { - const primary = await insertUser('np_p', 'primary2@test.example'); - const secondary = await insertUser('np_s', 'secondary2@test.example'); - await bindAsSecondary(primary, secondary, 1); - - // Delete the secondary (not the primary). No promotion is needed. - const result = await promoteSecondaryIfPrimaryDeleted(secondary); - - expect(result).toBeNull(); + it('allows ordinary deletion without promoting a historical secondary', async () => { + const primary = await insertUser('cascade_primary', 'jordan-cascade@pinnacle.example'); + const secondary = await insertUser('cascade_secondary', 'sam-cascade@pinnacle.example'); + const identityId = await bindAsSecondary(primary, secondary, 1); + await pool.query(`DELETE FROM users WHERE workos_user_id = $1`, [primary]); - const primaryRow = await pool.query<{ is_primary: boolean }>( - `SELECT is_primary FROM identity_workos_users WHERE workos_user_id = $1`, - [primary] + const survivors = await pool.query<{ workos_user_id: string; is_primary: boolean }>( + `SELECT workos_user_id, is_primary FROM identity_workos_users WHERE identity_id = $1`, [identityId], ); - expect(primaryRow.rows[0].is_primary).toBe(true); + expect(survivors.rows).toEqual([{ workos_user_id: secondary, is_primary: false }]); }); - it('CASCADE on the deleted user removes the binding after promotion', async () => { - const primary = await insertUser('cascade_p', 'cascadeprimary@test.example'); - const secondary = await insertUser('cascade_s', 'cascadesecondary@test.example'); + async function seedDeletionAuthority() { + const primary = await insertUser('revoke_primary', 'jordan-revoke@pinnacle.example'); + const secondary = await insertUser('revoke_secondary', 'sam-revoke@pinnacle.example'); + const bystander = await insertUser('revoke_bystander', 'maya-revoke@pinnacle.example'); const identityId = await bindAsSecondary(primary, secondary, 1); + await bindAsSecondary(primary, bystander, 2); + await pool.query( + `INSERT INTO organizations (workos_organization_id, name) VALUES ($1, 'Pinnacle Agency')`, [TEST_ORG], + ); + await pool.query( + `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role) + VALUES ($1, $4, 'jordan-revoke@pinnacle.example', 'owner'), + ($2, $4, 'sam-revoke@pinnacle.example', 'member'), + ($3, $4, 'maya-revoke@pinnacle.example', 'admin')`, [primary, secondary, bystander, TEST_ORG], + ); + await pool.query( + `UPDATE users SET primary_organization_id = $1 WHERE workos_user_id = ANY($2)`, + [TEST_ORG, [primary, secondary, bystander]], + ); + await pool.query( + `INSERT INTO working_group_memberships (working_group_id, workos_user_id, status) + SELECT id, $1, 'active' FROM working_groups WHERE slug = 'aao-admin' + UNION ALL SELECT id, $2, 'inactive' FROM working_groups WHERE slug = 'aao-admin'`, [bystander, secondary], + ); + await pool.query( + `INSERT INTO working_group_leaders (working_group_id, user_id) + SELECT id, $1 FROM working_groups WHERE slug = 'aao-admin'`, [bystander], + ); + await bumpAuthorizationEpochs(pool, [primary, secondary, bystander]); + return { primary, secondary, bystander, identityId }; + } - const result = await promoteSecondaryIfPrimaryDeleted(primary); - expect(result?.promotedUserId).toBe(secondary); + async function survivorAuthority(ids: string[]) { + const tables = [ + ['users', 'workos_user_id'], + ['identity_workos_users', 'workos_user_id'], + ['organization_memberships', 'workos_user_id'], + ['working_group_memberships', 'workos_user_id'], + ['working_group_leaders', 'user_id'], + ]; + const state: Record<string, unknown> = {}; + for (const [table, column] of tables) { + const result = await pool.query( + `SELECT row_to_json(row) AS row FROM (SELECT * FROM ${table} WHERE ${column} = ANY($1)) row + ORDER BY row_to_json(row)::text`, [ids], + ); + state[table] = result.rows.map(({ row }) => row); + } + return state; + } - // The webhook handler runs deleteUser() next, which CASCADE-drops the - // (now non-primary) binding. Surviving primary remains. - await pool.query(`DELETE FROM users WHERE workos_user_id = $1`, [primary]); + it('atomically revokes former sibling caches without promoting or changing their authority', async () => { + const { primary, secondary, bystander, identityId } = await seedDeletionAuthority(); + const survivors = [secondary, bystander]; + const authorityBefore = await survivorAuthority(survivors); + const fingerprintsBefore = await Promise.all(survivors.map((id) => getAuthorizationFingerprint([id]))); - const survivors = await pool.query<{ workos_user_id: string; is_primary: boolean }>( - `SELECT workos_user_id, is_primary FROM identity_workos_users WHERE identity_id = $1`, - [identityId] + const results = await Promise.all([deleteIdentityCredential(primary), deleteIdentityCredential(primary)]); + + expect(results.flat()).toEqual(expect.arrayContaining([primary, secondary, bystander])); + expect(await survivorAuthority(survivors)).toEqual(authorityBefore); + for (const [index, id] of survivors.entries()) { + expect(await getAuthorizationFingerprint([id])).not.toBe(fingerprintsBefore[index]); + } + expect((await pool.query(`SELECT 1 FROM users WHERE workos_user_id = $1`, [primary])).rows).toEqual([]); + expect((await pool.query(`SELECT 1 FROM organization_memberships WHERE workos_user_id = $1`, [primary])).rows).toEqual([]); + expect((await pool.query( + `SELECT workos_user_id FROM identity_workos_users WHERE identity_id = $1 AND is_primary = TRUE`, [identityId], + )).rows).toEqual([]); + + const fingerprintAfter = await getAuthorizationFingerprint(survivors); + await deleteIdentityCredential(primary); + expect(await getAuthorizationFingerprint(survivors)).toBe(fingerprintAfter); + expect(await survivorAuthority(survivors)).toEqual(authorityBefore); + }); + + it('rolls back epoch changes and deletion together when the database refuses deletion', async () => { + const { primary, secondary, bystander } = await seedDeletionAuthority(); + const credentials = [primary, secondary, bystander]; + const before = await survivorAuthority(credentials); + const fingerprintBefore = await getAuthorizationFingerprint(credentials); + // A test-only referencing row forces a failure after the epoch bump. + await pool.query( + `CREATE TABLE identity_containment_delete_guard ( + workos_user_id TEXT REFERENCES users(workos_user_id) ON DELETE RESTRICT + )`, ); - expect(survivors.rows).toHaveLength(1); - expect(survivors.rows[0].workos_user_id).toBe(secondary); - expect(survivors.rows[0].is_primary).toBe(true); + try { + await pool.query(`INSERT INTO identity_containment_delete_guard VALUES ($1)`, [primary]); + await expect(deleteIdentityCredential(primary)).rejects.toThrow(); + expect(await survivorAuthority(credentials)).toEqual(before); + expect(await getAuthorizationFingerprint(credentials)).toBe(fingerprintBefore); + } finally { + await pool.query(`DROP TABLE identity_containment_delete_guard`); + } }); it('deletes an account with accumulated community data', async () => { diff --git a/server/tests/unit/identity-mutation-containment.test.ts b/server/tests/unit/identity-mutation-containment.test.ts new file mode 100644 index 00000000000..39a825ef937 --- /dev/null +++ b/server/tests/unit/identity-mutation-containment.test.ts @@ -0,0 +1,86 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { getPool } = vi.hoisted(() => ({ getPool: vi.fn() })); +vi.mock('../../src/db/client.js', () => ({ getPool })); +vi.mock('../../src/logger.js', () => ({ + createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }), +})); +vi.mock('../../src/addie/error-notifier.js', () => ({ notifySystemError: vi.fn() })); + +import { mergeUsers, previewUserMerge } from '../../src/db/user-merge-db.js'; +import { promoteSecondaryIfPrimaryDeleted } from '../../src/db/identity-db.js'; +import { IdentityMutationDisabledError } from '../../src/db/identity-mutation-policy.js'; + +const refusal = { + code: 'identity_mutation_disabled', + message: 'Identity consolidation is disabled until authority and provenance can be preserved.', +}; + +describe('identity mutation containment at the database seam', () => { + beforeEach(() => { + getPool.mockReset(); + getPool.mockImplementation(() => { + throw new Error('The containment guard must refuse before accessing the database'); + }); + }); + + afterEach(() => { + expect(getPool).not.toHaveBeenCalled(); + }); + + it.each([ + ['automatic', 'user_sam', 'user_jordan', 'auto'], + ['member', 'user_sam', 'user_jordan', 'user_sam'], + ['admin', 'user_jordan', 'user_sam', 'user_admin'], + ['same credential', 'user_sam', 'user_sam', 'user_sam'], + ['missing credential', '', 'user_jordan', 'user_admin'], + ])('refuses %s merges before opening a connection', async (_caller, primary, secondary, actor) => { + await expect(mergeUsers(primary, secondary, actor)).rejects.toMatchObject(refusal); + }); + + it('rejects promotion and operator confirmation options without a bypass', async () => { + await expect(mergeUsers('user_sam', 'user_jordan', 'user_admin', { + ensurePrimaryFlag: true, + auditContext: { consolidation_confirmed: true, consolidate: true }, + })).rejects.toBeInstanceOf(IdentityMutationDisabledError); + }); + + it.each(['user_sam', 'user_jordan', ''])('refuses automatic primary promotion for %j', async (userId) => { + await expect(promoteSecondaryIfPrimaryDeleted(userId)).rejects.toMatchObject(refusal); + }); + + it('keeps simultaneous, reversed, and replayed operations harmless', async () => { + const attempts = Array.from({ length: 12 }, (_, index) => index % 3 === 0 + ? promoteSecondaryIfPrimaryDeleted('user_jordan') + : mergeUsers( + index % 2 ? 'user_sam' : 'user_jordan', + index % 2 ? 'user_jordan' : 'user_sam', + 'user_admin', + { ensurePrimaryFlag: true, auditContext: { consolidation_confirmed: true } }, + )); + for (const result of await Promise.allSettled(attempts)) { + expect(result.status).toBe('rejected'); + if (result.status === 'rejected') expect(result.reason).toMatchObject(refusal); + } + await expect(mergeUsers('user_sam', 'user_jordan', 'user_admin')).rejects.toMatchObject(refusal); + }); +}); + +describe('read-only merge detection', () => { + it('keeps preview available and issues only reads', async () => { + const query = vi.fn().mockResolvedValue({ rows: [{ count: '1' }] }); + const release = vi.fn(); + getPool.mockReset(); + getPool.mockReturnValue({ connect: vi.fn().mockResolvedValue({ query, release }) }); + + const preview = await previewUserMerge('user_sam', 'user_jordan'); + + expect(preview.tables).toContainEqual({ table_name: 'organization_memberships', row_count: 1 }); + expect(query).toHaveBeenCalled(); + for (const [sql, parameters] of query.mock.calls) { + expect(sql).toMatch(/^SELECT COUNT\(\*\)/); + expect(parameters).toEqual(['user_jordan']); + } + expect(release).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/tests/unit/identity-mutation-route-containment.test.ts b/server/tests/unit/identity-mutation-route-containment.test.ts new file mode 100644 index 00000000000..2d2ecd1cbd0 --- /dev/null +++ b/server/tests/unit/identity-mutation-route-containment.test.ts @@ -0,0 +1,159 @@ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import express, { type NextFunction, type Request, type Response } from 'express'; +import request from 'supertest'; + +const mocks = vi.hoisted(() => ({ + getPool: vi.fn(), + getWorkos: vi.fn(), + query: vi.fn(), + createUser: vi.fn(), + updateUser: vi.fn(), + deleteUser: vi.fn(), + getUser: vi.fn(), + sendEmailLinkVerification: vi.fn(), +})); + +vi.hoisted(() => { + process.env.WORKOS_API_KEY ??= 'sk_test_identity_containment'; + process.env.WORKOS_CLIENT_ID ??= 'client_test_identity_containment'; + process.env.WORKOS_COOKIE_PASSWORD ??= 'test-cookie-password-at-least-32-chars-long'; +}); + +// Exercise the registered routers after the authentication boundary, without a +// database. Any handler that reaches a provider or local data seam fails here. +vi.mock('../../src/middleware/auth.js', async (importOriginal) => { + const original = await importOriginal<typeof import('../../src/middleware/auth.js')>(); + const authenticate = (req: Request, res: Response, next: NextFunction) => { + if (req.headers['x-test-unauthenticated']) return res.sendStatus(401); + req.user = { + id: String(req.headers['x-test-user'] || 'user_member'), + authWorkosUserId: 'user_signed_in_credential', + email: 'member@example.test', + emailVerified: true, + createdAt: new Date(0).toISOString(), + updatedAt: new Date(0).toISOString(), + }; + req.adminAccessMechanism = req.headers['x-test-admin-access-mechanism'] as typeof req.adminAccessMechanism; + next(); + }; + return { ...original, requireAuth: authenticate, requireGlobalAdmin: [authenticate] }; +}); + +vi.mock('../../src/db/client.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/db/client.js')>()), + getPool: mocks.getPool, + query: mocks.query, +})); +vi.mock('../../src/auth/workos-client.js', () => { + const workos = { userManagement: { + createUser: mocks.createUser, + updateUser: mocks.updateUser, + deleteUser: mocks.deleteUser, + getUser: mocks.getUser, + } }; + return { workos, getWorkos: () => { mocks.getWorkos(); return workos; } }; +}); +vi.mock('../../src/routes/workos-webhooks.js', () => ({ + backfillOrganizationMemberships: vi.fn(), + backfillUsers: vi.fn(), + backfillOrganizationDomains: vi.fn(), +})); +vi.mock('../../src/notifications/email.js', () => ({ + sendEmailLinkVerification: mocks.sendEmailLinkVerification, + sendSlackInviteEmail: vi.fn(), + hasSlackInviteBeenSent: vi.fn(), +})); + +import { createAdminUsersRouter } from '../../src/routes/admin/users.js'; +import { createAccountLinkingRouter } from '../../src/routes/account-linking.js'; +import { stopAuthTimers } from '../../src/middleware/auth.js'; + +const app = express(); +app.use(express.json()); +app.use('/api/admin/users', createAdminUsersRouter()); +app.use('/api/me/linked-emails', createAccountLinkingRouter()); + +const refusal = { + error: 'identity_mutation_disabled', + message: 'Identity consolidation is disabled until authority and provenance can be preserved.', +}; +const operations = [ + { method: 'post', path: '/api/admin/users/user_member/linked-emails', body: { email: 'admin@example.test' } }, + { method: 'post', path: '/api/admin/users/user_member/credentials', body: { workos_user_id: 'user_admin' } }, + { method: 'post', path: '/api/admin/users/user_admin/credentials', body: { workos_user_id: 'user_member' } }, + { method: 'post', path: '/api/admin/users/user_member/credentials/user_admin/promote', body: {} }, + { method: 'post', path: '/api/admin/users/user_admin/credentials/user_member/promote', body: {} }, + { method: 'post', path: '/api/admin/users/user_missing/credentials/user_missing/promote', body: {} }, + { method: 'put', path: '/api/me/linked-emails/primary', body: { email: 'admin@example.test' } }, +] as const; + +function expectNoDataAccess() { + for (const mock of Object.values(mocks)) expect(mock).not.toHaveBeenCalled(); +} + +describe('identity mutation route containment', () => { + beforeEach(() => { + vi.resetAllMocks(); + mocks.getPool.mockImplementation(() => { throw new Error('Unexpected database pool access'); }); + mocks.query.mockImplementation(() => { throw new Error('Unexpected database query'); }); + }); + + it.each(operations)('refuses $method $path before provider or local access', async ({ method, path, body }) => { + for (const consolidate of [false, true]) { + for (const mechanism of ['sso', 'static_admin_api_key']) { + const response = await request(app)[method](path) + .set('X-Test-Admin-Access-Mechanism', mechanism) + .send({ ...body, consolidate }) + .expect(409); + expect(response.body).toEqual(refusal); + expectNoDataAccess(); + } + } + }); + + it.each(operations)('still authenticates $method $path', async ({ method, path, body }) => { + await request(app)[method](path) + .set('X-Test-Unauthenticated', 'true') + .send(body) + .expect(401); + expectNoDataAccess(); + }); + + it('refuses concurrent and replayed calls in both identity directions', async () => { + const invoke = ({ method, path, body }: typeof operations[number]) => request(app)[method](path) + .send({ ...body, consolidate: true }) + .expect(409) + .expect(({ body }) => expect(body).toEqual(refusal)); + await Promise.all([...operations, ...operations].map(invoke)); + for (const operation of operations) await invoke(operation); + expectNoDataAccess(); + }); + + it.each([{}, { email: null }, { email: 'member@example.test' }, { workos_user_id: 'invalid' }])( + 'cannot bypass refusal through missing, same-identity, or invalid input: %j', async (body) => { + for (const { method, path } of operations) { + await request(app)[method](path).send(body).expect(409) + .expect(({ body }) => expect(body).toEqual(refusal)); + } + expectNoDataAccess(); + }, + ); + + it('refuses a member attempt to combine existing accounts without issuing a token or email', async () => { + mocks.query.mockImplementation(async (sql: string) => { + if (sql.includes('COUNT(*)')) return { rows: [{ count: '0' }] }; + if (sql.includes('FROM users')) return { rows: [{ workos_user_id: 'user_existing', email: 'existing@example.test' }] }; + if (sql.trimStart().startsWith('SELECT')) return { rows: [] }; + throw new Error(`Unexpected mutation: ${sql}`); + }); + const response = await request(app).post('/api/me/linked-emails') + .send({ email: 'existing@example.test', consolidate: true }).expect(409); + expect(response.body.error).toBe('This email already has an AAO account'); + expect(mocks.query.mock.calls.every(([sql]) => sql.trimStart().startsWith('SELECT'))).toBe(true); + expect(mocks.sendEmailLinkVerification).not.toHaveBeenCalled(); + expect(mocks.getPool).not.toHaveBeenCalled(); + for (const method of [mocks.createUser, mocks.updateUser, mocks.deleteUser]) expect(method).not.toHaveBeenCalled(); + }); +}); + +afterAll(() => stopAuthTimers()); diff --git a/server/tests/unit/native-auth-http-wiring.test.ts b/server/tests/unit/native-auth-http-wiring.test.ts index c1f4b4b55d5..29642c6e0e3 100644 --- a/server/tests/unit/native-auth-http-wiring.test.ts +++ b/server/tests/unit/native-auth-http-wiring.test.ts @@ -2,6 +2,14 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import request from 'supertest'; const mocks = vi.hoisted(() => ({ + query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }), + listUsers: vi.fn(), + createUser: vi.fn(), + updateUser: vi.fn(), + deleteUser: vi.fn(), + createOrganizationMembership: vi.fn(), + updateOrganizationMembership: vi.fn(), + deleteOrganizationMembership: vi.fn(), getAuthorizationUrl: vi.fn(), authenticateWithCode: vi.fn(), listOrganizationMemberships: vi.fn(), @@ -27,6 +35,13 @@ vi.mock('@workos-inc/node', () => ({ DomainDataState: { Verified: 'verified' }, WorkOS: class WorkOS { userManagement = { + listUsers: mocks.listUsers, + createUser: mocks.createUser, + updateUser: mocks.updateUser, + deleteUser: mocks.deleteUser, + createOrganizationMembership: mocks.createOrganizationMembership, + updateOrganizationMembership: mocks.updateOrganizationMembership, + deleteOrganizationMembership: mocks.deleteOrganizationMembership, getAuthorizationUrl: mocks.getAuthorizationUrl, authenticateWithCode: mocks.authenticateWithCode, listOrganizationMemberships: mocks.listOrganizationMemberships, @@ -49,12 +64,12 @@ vi.mock('../../src/config.js', async () => { vi.mock('../../src/db/client.js', () => ({ initializeDatabase: vi.fn(), getPool: vi.fn().mockReturnValue({ - query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }), + query: mocks.query, }), isDatabaseInitialized: vi.fn().mockReturnValue(false), closeDatabase: vi.fn(), healthCheck: vi.fn().mockResolvedValue(undefined), - query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }), + query: mocks.query, })); vi.mock('../../src/db/migrate.js', () => ({ @@ -404,4 +419,45 @@ describe('native OAuth HTTPServer wiring', () => { expect(response.status).toBe(302); expect(mocks.sendAccountLinkedMessage).toHaveBeenCalledWith(origin, 'Person'); }); + + it.each([ + ['local', 'user_sam', 'sam.adeyemi@gmail.com', 'user_jordan', 'sam.adeyemi@googlemail.com'], + ['provider', 'user_jordan', 'sam.adeyemi@googlemail.com', 'user_sam', 'sam.adeyemi@gmail.com'], + ])('only detects aliases during concurrent/replayed OAuth callbacks (%s)', async (location, id, email, otherId, otherEmail) => { + mocks.authenticateWithCode.mockResolvedValue({ + sealedSession: 'test-sealed-session', + user: { id, email, firstName: 'Sam', lastName: 'Adeyemi', emailVerified: true, + createdAt: '2026-01-01T00:00:00.000Z', updatedAt: '2026-01-01T00:00:00.000Z' }, + }); + mocks.listOrganizationMemberships.mockResolvedValue({ data: [ + { organizationId: 'org_pinnacle', userId: id, status: 'active' }, + ] }); + mocks.listUsers.mockResolvedValue({ data: [{ id: otherId, email: otherEmail }] }); + mocks.query.mockImplementation(async (sql: string) => ({ + rows: sql.includes('FROM users u') && sql.includes('user_email_aliases') && location === 'local' + ? [{ workos_user_id: otherId, email: otherEmail }] : [], + rowCount: 0, + })); + server = new HTTPServer(); + const callback = () => request(appFor(server!)).get('/auth/callback').query({ + code: 'test-code', state: JSON.stringify({ return_to: '/member-hub', consolidate: true }), + }); + const responses = await Promise.all([callback(), callback(), callback()]); + responses.push(await callback()); + for (const response of responses) { + expect(response.status).toBe(302); + expect(response.headers.location).toBe(`/member-hub?duplicate_email=${encodeURIComponent(otherEmail)}`); + } + for (const write of [mocks.createUser, mocks.updateUser, mocks.deleteUser, + mocks.createOrganizationMembership, mocks.updateOrganizationMembership, mocks.deleteOrganizationMembership]) { + expect(write).not.toHaveBeenCalled(); + } + expect(mocks.listOrganizationMemberships.mock.calls.every(([input]) => input.userId === id)).toBe(true); + const writes = mocks.query.mock.calls.filter(([sql]) => /^\s*(INSERT|UPDATE|DELETE)\b/i.test(sql)); + // Ordinary login may upsert the authenticating user. Alias detection must + // never synthesize the duplicate or mutate either credential's authority. + expect(writes.every(([sql, params]) => sql.trimStart().startsWith('INSERT INTO users ') && params[0] === id)).toBe(true); + expect(mocks.query.mock.calls.some(([sql]) => sql.includes('FROM users u') && sql.includes('user_email_aliases'))).toBe(true); + }); + }); diff --git a/server/tests/unit/set-primary-email.test.ts b/server/tests/unit/set-primary-email.test.ts index ec783dfc3b5..e25d5016c68 100644 --- a/server/tests/unit/set-primary-email.test.ts +++ b/server/tests/unit/set-primary-email.test.ts @@ -3,79 +3,13 @@ import * as fs from 'fs'; import * as path from 'path'; import { isEmailUnavailable } from '../../src/routes/account-linking-errors.js'; -/** - * Static analysis test for the set-primary-email endpoint. - * Verifies the route handler exists and follows the correct swap pattern. - */ - -const ACCOUNT_LINKING_FILE = path.resolve( - __dirname, - '../../src/routes/account-linking.ts' -); - const DASHBOARD_SETTINGS_FILE = path.resolve( __dirname, '../../public/dashboard-settings.html' ); -describe('Set primary email endpoint', () => { - const source = fs.readFileSync(ACCOUNT_LINKING_FILE, 'utf-8'); - - it('registers a PUT /primary route on the router', () => { - expect(source).toMatch(/router\.put\(\s*['"]\/primary['"]/); - }); - - it('requires authentication and rate limiting', () => { - expect(source).toMatch(/router\.put\(\s*['"]\/primary['"],\s*requireAuth,\s*verifyExecuteLimiter/); - }); - - it('validates email is a verified alias before swapping', () => { - expect(source).toMatch(/user_email_aliases/); - expect(source).toMatch(/workos_user_id = \$1 AND LOWER\(email\) = \$2 AND verified_at IS NOT NULL/); - }); - - it('updates WorkOS as source of truth', () => { - expect(source).toMatch(/getWorkos\(\)\.userManagement\.updateUser/); - }); - - it('swaps emails in a transaction', () => { - // Must use BEGIN/COMMIT for atomicity - expect(source).toMatch(/BEGIN/); - expect(source).toMatch(/COMMIT/); - // Must update users table - expect(source).toMatch(/UPDATE users SET email/); - // Must delete alias for new primary - expect(source).toMatch(/DELETE FROM user_email_aliases/); - // Must insert old primary as alias - expect(source).toMatch(/INSERT INTO user_email_aliases[\s\S]*?VALUES/); - }); - - it('locks alias row with FOR UPDATE to prevent races', () => { - expect(source).toMatch(/FOR UPDATE/); - }); - - it('updates WorkOS before the DB swap so a WorkOS rejection leaves DB state untouched', () => { - const beginIdx = source.indexOf("'BEGIN'"); - const workosIdx = source.indexOf('getWorkos().userManagement.updateUser'); - expect(workosIdx).toBeGreaterThan(-1); - expect(beginIdx).toBeGreaterThan(-1); - expect(workosIdx).toBeLessThan(beginIdx); - }); - - it('classifies WorkOS rejections via isEmailUnavailable and returns a friendly 409', () => { - expect(source).toMatch(/isEmailUnavailable\(workosError\)/); - expect(source).toMatch(/already associated with another account/); - }); - - it('rolls back on error', () => { - expect(source).toMatch(/ROLLBACK/); - }); - - it('refreshes denormalized email on organization_memberships and person_relationships', () => { - expect(source).toMatch(/UPDATE organization_memberships SET email/); - expect(source).toMatch(/UPDATE person_relationships SET email/); - }); -}); +// The executable identity-mutation-route-containment suite covers this route's +// refusal before provider and database access. Keep helper/UI coverage here. describe('isEmailUnavailable', () => { it('matches WorkOS GenericServerException with "This email is not available" message', () => { diff --git a/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts b/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts index f2cb6659273..03d56407d57 100644 --- a/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts +++ b/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts @@ -3,24 +3,14 @@ import * as fs from 'fs'; import * as path from 'path'; /** - * Static-analysis wiring assertions on the user.deleted webhook handler. - * - * Issue adcontextprotocol/adcp#3718: WorkOS-side deletion of a primary - * credential leaves multi-credential identities with zero primaries. The - * handler must promote a surviving secondary before the CASCADE fires, - * invalidate the 60-second session cache, and never return a 5xx on - * promotion failure (which would trigger WorkOS retry storms). - * - * These checks are static so a refactor that re-orders the calls or drops - * the invalidation fails fast without spinning up the full HTTP stack. + * Keep provider revocation wired without choosing a successor identity. + * Runtime DB-seam tests separately prove that promotion refuses before writes. */ const WEBHOOK_FILE = path.resolve(__dirname, '../../src/routes/workos-webhooks.ts'); -const IDENTITY_DB_FILE = path.resolve(__dirname, '../../src/db/identity-db.ts'); const webhookSource = fs.readFileSync(WEBHOOK_FILE, 'utf-8'); -const identitySource = fs.readFileSync(IDENTITY_DB_FILE, 'utf-8'); -describe('user.deleted handler wiring (#3718)', () => { +describe('user.deleted containment wiring (#6827)', () => { const userDeletedBlockMatch = webhookSource.match( /case 'user\.deleted':\s*\{([\s\S]*?)\n\s{10}\}/, ); @@ -30,41 +20,13 @@ describe('user.deleted handler wiring (#3718)', () => { expect(block).not.toEqual(''); }); - it('calls promoteSecondaryIfPrimaryDeleted before deleteUser', () => { - const promoteIdx = block.indexOf('promoteSecondaryIfPrimaryDeleted'); - const deleteUserIdx = block.indexOf('deleteUser('); - expect(promoteIdx).toBeGreaterThanOrEqual(0); - expect(deleteUserIdx).toBeGreaterThan(promoteIdx); + it('preserves credential revocation without promoting a successor', () => { + expect(block).not.toContain('promoteSecondaryIfPrimaryDeleted'); + expect(block).toContain('await deleteIdentityCredential(user.id)'); }); it('invalidates session caches for the deleted user', () => { - expect(block).toMatch(/invalidateSessionsForUsers\s*\(/); - }); -}); - -describe('promoteSecondaryIfPrimaryDeleted error containment (#3718)', () => { - // The handler awaits the helper directly without a try/catch — a thrown - // error would land in the outer catch and produce a 500, triggering WorkOS - // retry storms. The helper must catch its own errors and return null so - // the handler falls through to the 200 path. - const helperSource = identitySource.match( - /export async function promoteSecondaryIfPrimaryDeleted[\s\S]*?\n\}\n/, - )?.[0]; - - it('matched the helper source', () => { - expect(helperSource).toBeDefined(); - }); - - it('swallows DB errors (catch + return null) so the webhook returns 200', () => { - expect(helperSource).toMatch(/catch\s*\(/); - expect(helperSource).toMatch(/return null/); - }); - - it('emits an explicit ops alert on promotion failure', () => { - // logger.warn auto-posts to #admin-errors (posthog.ts:201-205); plus an - // explicit notifySystemError so the alert doesn't get lost in the warn - // stream during noisy hours. - expect(helperSource).toMatch(/logger\.warn/); - expect(helperSource).toMatch(/notifySystemError/); + expect(block).toContain('invalidateSessionsForUsers(affectedUserIds)'); + expect(block.indexOf('await deleteIdentityCredential')).toBeLessThan(block.indexOf('invalidateSessionsForUsers')); }); }); From 2c652e3698ec03b0934c9a04b4126b8c72355ebe Mon Sep 17 00:00:00 2001 From: Brian O'Kelley <bokelley@scope3.com> Date: Sun, 13 Sep 2026 14:20:22 +0000 Subject: [PATCH 2/6] fix(auth): signal identities left without a primary after deletion --- server/src/db/identity-db.ts | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/server/src/db/identity-db.ts b/server/src/db/identity-db.ts index ca819fb8dcc..705e654e5ec 100644 --- a/server/src/db/identity-db.ts +++ b/server/src/db/identity-db.ts @@ -26,13 +26,15 @@ const logger = createLogger('identity-db'); */ export async function deleteIdentityCredential(workosUserId: string): Promise<string[]> { const client = await getPool().connect(); + let affectedUserIds: string[] = [workosUserId]; + let orphanedIdentity: { id: string; survivingCredentials: number } | undefined; try { await client.query('BEGIN'); // Use a stable lock order when simultaneous webhooks delete siblings. // Unlink also needs the binding row lock before it can change the identity. - const bound = await client.query<{ workos_user_id: string }>( - `SELECT workos_user_id FROM identity_workos_users + const bound = await client.query<{ workos_user_id: string; identity_id: string; is_primary: boolean }>( + `SELECT workos_user_id, identity_id, is_primary FROM identity_workos_users WHERE identity_id = ( SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1 ) @@ -40,10 +42,15 @@ export async function deleteIdentityCredential(workosUserId: string): Promise<st FOR UPDATE`, [workosUserId], ); - const affectedUserIds = [...new Set([ + affectedUserIds = [...new Set([ workosUserId, ...bound.rows.map((row) => row.workos_user_id), ])]; + const deletedBinding = bound.rows.find((row) => row.workos_user_id === workosUserId); + const survivors = bound.rows.filter((row) => row.workos_user_id !== workosUserId); + if (deletedBinding && survivors.length > 0 && !survivors.some((row) => row.is_primary)) { + orphanedIdentity = { id: deletedBinding.identity_id, survivingCredentials: survivors.length }; + } // Authentication checks the actual credential's epoch, so deleting the // primary's epoch alone would leave each surviving credential's cache valid. @@ -51,13 +58,32 @@ export async function deleteIdentityCredential(workosUserId: string): Promise<st await client.query('DELETE FROM users WHERE workos_user_id = $1', [workosUserId]); await client.query('DELETE FROM organization_memberships WHERE workos_user_id = $1', [workosUserId]); await client.query('COMMIT'); - return affectedUserIds; } catch (err) { await client.query('ROLLBACK').catch(() => undefined); throw err; } finally { client.release(); } + + // Signal the committed access disruption without choosing a successor or + // inferring lost provenance. Rollbacks and replays without a binding do not + // report it. Notification failure must not prevent post-commit invalidation. + if (orphanedIdentity) { + logger.warn({ + deletedUserId: workosUserId, + identityId: orphanedIdentity.id, + survivingCredentialCount: orphanedIdentity.survivingCredentials, + }, 'Provider deletion left surviving credentials without a primary; operator review required'); + try { + notifySystemError({ + source: 'identity-primary-missing', + errorMessage: `Identity ${orphanedIdentity.id} has ${orphanedIdentity.survivingCredentials} surviving credential(s) and no primary after WorkOS user.deleted for ${workosUserId}. Operator review is required; automatic promotion and historical restoration remain disabled.`, + }); + } catch (err) { + logger.error({ err, identityId: orphanedIdentity.id }, 'Failed to report identity without a primary'); + } + } + return affectedUserIds; } /** From 47c6953d5f215dd99a06402a4ba4f00d52dde9f0 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley <bokelley@scope3.com> Date: Sun, 13 Sep 2026 14:21:39 +0000 Subject: [PATCH 3/6] test(auth): cover committed missing-primary alerts --- .../workos-user-deleted-primary.test.ts | 130 +++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/server/tests/integration/workos-user-deleted-primary.test.ts b/server/tests/integration/workos-user-deleted-primary.test.ts index f00ccc9aa28..7a62c1b040b 100644 --- a/server/tests/integration/workos-user-deleted-primary.test.ts +++ b/server/tests/integration/workos-user-deleted-primary.test.ts @@ -4,13 +4,20 @@ * deletion remains valid and must not silently promote a surviving credential. */ -import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; import { initializeDatabase, closeDatabase } from '../../src/db/client.js'; import { runMigrations } from '../../src/db/migrate.js'; import { deleteIdentityCredential, promoteSecondaryIfPrimaryDeleted } from '../../src/db/identity-db.js'; import { bumpAuthorizationEpochs, getAuthorizationFingerprint } from '../../src/db/authorization-epoch-db.js'; import type { Pool } from 'pg'; +const { notifySystemError, identityLogger } = vi.hoisted(() => ({ + notifySystemError: vi.fn(), + identityLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); +vi.mock('../../src/addie/error-notifier.js', () => ({ notifySystemError })); +vi.mock('../../src/logger.js', () => ({ createLogger: () => identityLogger })); + const TEST_USER_PREFIX = 'user_wh_deleted_test_'; const TEST_ORG = 'org_wh_deleted_test_pinnacle'; @@ -32,6 +39,8 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { beforeEach(async () => { await cleanup(); + vi.clearAllMocks(); + notifySystemError.mockReset(); }); async function cleanup() { @@ -211,10 +220,25 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { `SELECT workos_user_id FROM identity_workos_users WHERE identity_id = $1 AND is_primary = TRUE`, [identityId], )).rows).toEqual([]); + expect(notifySystemError).toHaveBeenCalledOnce(); + expect(identityLogger.warn).toHaveBeenCalledOnce(); + expect(identityLogger.warn.mock.calls[0][0]).toEqual({ + deletedUserId: primary, + identityId, + survivingCredentialCount: 2, + }); + expect(notifySystemError).toHaveBeenCalledWith({ + source: 'identity-primary-missing', + errorMessage: expect.stringContaining(identityId), + }); + expect(JSON.stringify(notifySystemError.mock.calls)).not.toContain('@'); + const fingerprintAfter = await getAuthorizationFingerprint(survivors); await deleteIdentityCredential(primary); expect(await getAuthorizationFingerprint(survivors)).toBe(fingerprintAfter); expect(await survivorAuthority(survivors)).toEqual(authorityBefore); + expect(notifySystemError).toHaveBeenCalledOnce(); + expect(identityLogger.warn).toHaveBeenCalledOnce(); }); it('rolls back epoch changes and deletion together when the database refuses deletion', async () => { @@ -233,11 +257,115 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { await expect(deleteIdentityCredential(primary)).rejects.toThrow(); expect(await survivorAuthority(credentials)).toEqual(before); expect(await getAuthorizationFingerprint(credentials)).toBe(fingerprintBefore); + expect(notifySystemError).not.toHaveBeenCalled(); + expect(identityLogger.warn).not.toHaveBeenCalled(); } finally { await pool.query(`DROP TABLE identity_containment_delete_guard`); } }); + it('signals only after commit and connection release, with committed deletion visible to another connection', async () => { + const { primary, secondary, bystander, identityId } = await seedDeletionAuthority(); + const observer = await pool.connect(); + try { + // Keep an independent connection for visibility checks and leave another + // idle connection for deletion, so release ordering is observable. + await pool.query('SELECT 1'); + const idleBeforeDeletion = pool.idleCount; + expect(idleBeforeDeletion).toBeGreaterThan(0); + let idleAtSignal = -1; + let stateAtSignal: Promise<{ rows: unknown[] }> | undefined; + notifySystemError.mockImplementation(() => { + idleAtSignal = pool.idleCount; + stateAtSignal = observer.query( + `SELECT + EXISTS (SELECT 1 FROM users WHERE workos_user_id = $1) AS deleted_user_exists, + EXISTS (SELECT 1 FROM organization_memberships WHERE workos_user_id = $1) AS deleted_membership_exists, + (SELECT COUNT(*)::int FROM identity_workos_users WHERE identity_id = $2) AS survivors, + (SELECT COUNT(*)::int FROM identity_workos_users WHERE identity_id = $2 AND is_primary) AS primaries, + (SELECT MIN(epoch)::int FROM authorization_epochs WHERE workos_user_id = ANY($3)) AS survivor_epoch`, + [primary, identityId, [secondary, bystander]], + ); + }); + + await deleteIdentityCredential(primary); + + expect(notifySystemError).toHaveBeenCalledOnce(); + expect(idleAtSignal).toBe(idleBeforeDeletion); + expect((await stateAtSignal)?.rows).toEqual([{ + deleted_user_exists: false, + deleted_membership_exists: false, + survivors: 2, + primaries: 0, + survivor_epoch: 2, + }]); + } finally { + observer.release(); + } + }); + + it.each([true, false])('does not signal when deleting the last binding (is_primary=%s) or replaying it', async (isPrimary) => { + const userId = await insertUser('signal_singleton', 'jordan-singleton@pinnacle.example'); + if (!isPrimary) { + await pool.query(`UPDATE identity_workos_users SET is_primary = FALSE WHERE workos_user_id = $1`, [userId]); + } + + expect(await deleteIdentityCredential(userId)).toEqual([userId]); + expect(await deleteIdentityCredential(userId)).toEqual([userId]); + + expect(notifySystemError).not.toHaveBeenCalled(); + expect(identityLogger.warn).not.toHaveBeenCalled(); + }); + + it('does not signal secondary deletion while the original primary survives', async () => { + const { primary, secondary, bystander } = await seedDeletionAuthority(); + const before = await survivorAuthority([primary, bystander]); + + await deleteIdentityCredential(secondary); + + expect(await survivorAuthority([primary, bystander])).toEqual(before); + expect(notifySystemError).not.toHaveBeenCalled(); + expect(identityLogger.warn).not.toHaveBeenCalled(); + }); + + it('signals an existing identity without a primary when another credential is deleted', async () => { + const { primary, secondary, bystander, identityId } = await seedDeletionAuthority(); + // Historical invalid state is an input, never permission to choose a primary. + await pool.query(`UPDATE identity_workos_users SET is_primary = FALSE WHERE workos_user_id = $1`, [primary]); + const before = await survivorAuthority([primary, bystander]); + + await deleteIdentityCredential(secondary); + + expect(await survivorAuthority([primary, bystander])).toEqual(before); + expect(notifySystemError).toHaveBeenCalledWith({ + source: 'identity-primary-missing', + errorMessage: expect.stringContaining(identityId), + }); + expect(identityLogger.warn.mock.calls[0][0]).toEqual({ + deletedUserId: secondary, + identityId, + survivingCredentialCount: 2, + }); + }); + + it('returns cache invalidation IDs and preserves committed revocation if notifying throws', async () => { + const { primary, secondary, bystander } = await seedDeletionAuthority(); + const before = await survivorAuthority([secondary, bystander]); + const notificationFailure = new Error('Notification transport unavailable'); + notifySystemError.mockImplementation(() => { throw notificationFailure; }); + + await expect(deleteIdentityCredential(primary)).resolves.toEqual(expect.arrayContaining([primary, secondary, bystander])); + + expect((await pool.query(`SELECT 1 FROM users WHERE workos_user_id = $1`, [primary])).rows).toEqual([]); + expect(await survivorAuthority([secondary, bystander])).toEqual(before); + expect(await getAuthorizationFingerprint([secondary])).toBe(`${secondary}:2`); + expect(await getAuthorizationFingerprint([bystander])).toBe(`${bystander}:2`); + expect(identityLogger.error).toHaveBeenCalledWith( + expect.objectContaining({ err: notificationFailure }), + expect.any(String), + ); + }); + it('deletes an account with accumulated community data', async () => { const userId = await insertUser('community', 'community@test.example'); await pool.query( From 7596daa2c4bf69cc2bb7b8bf6ab5a628ec1993d1 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley <bokelley@scope3.com> Date: Sun, 13 Sep 2026 14:41:19 +0000 Subject: [PATCH 4/6] fix(auth): serialize confirmed credential deletion --- server/src/db/identity-db.ts | 371 ++++++++++++++++-- server/src/routes/workos-webhooks.ts | 111 ++---- .../services/identity-credential-deletion.ts | 71 ++++ .../workos-user-deleted-primary.test.ts | 332 ++++++++++------ .../workos-user-deletion-race.test.ts | 368 +++++++++++++++++ ...workos-webhook-user-deleted-wiring.test.ts | 28 +- 6 files changed, 1043 insertions(+), 238 deletions(-) create mode 100644 server/src/services/identity-credential-deletion.ts create mode 100644 server/tests/integration/workos-user-deletion-race.test.ts diff --git a/server/src/db/identity-db.ts b/server/src/db/identity-db.ts index 705e654e5ec..33dfb41c8be 100644 --- a/server/src/db/identity-db.ts +++ b/server/src/db/identity-db.ts @@ -18,46 +18,354 @@ import { notifySystemError } from '../addie/error-notifier.js'; const logger = createLogger('identity-db'); +export const IDENTITY_RECOVERY_STATE = 'manual_primary_selection_required' as const; + +export type IdentityCredentialDeletionSource = 'workos_webhook' | 'sync_users_backfill'; + +export interface IdentityBeforeGraph { + identity: Record<string, unknown>; + identity_workos_users: Record<string, unknown>[]; + users: Record<string, unknown>[]; + organization_memberships: Record<string, unknown>[]; + working_group_memberships: Record<string, unknown>[]; + working_group_leaders: Record<string, unknown>[]; + slack_user_mappings: Record<string, unknown>[]; + authorization_epochs: Record<string, unknown>[]; +} + +export interface IdentityRecoveryQuarantine { + audit_id: string; + identity_id: string; + deleted_workos_user_id: string; + deletion_source: IdentityCredentialDeletionSource; + actor: { + type: 'workos_provider'; + source: IdentityCredentialDeletionSource; + workos_user_id: string; + }; + recovery_state: typeof IDENTITY_RECOVERY_STATE; + before_graph: IdentityBeforeGraph; +} + +export interface IdentityCredentialDeletionResult { + deleted: boolean; + affectedUserIds: string[]; + affectedSlackUserIds: string[]; + quarantine: IdentityRecoveryQuarantine | null; +} + +export interface WorkosUserUpsert { + id: string; + email: string; + firstName: string | null; + lastName: string | null; + emailVerified: boolean; + createdAt: string; + updatedAt: string; +} + +interface JsonRow { + row: Record<string, unknown>; +} + +function rowsAsJson(result: { rows: JsonRow[] }): Record<string, unknown>[] { + return result.rows.map(({ row }) => row); +} + +async function lockCredentialMutation( + client: { query: (text: string, params?: unknown[]) => Promise<unknown> }, + workosUserId: string, +): Promise<void> { + await client.query(`SELECT pg_advisory_xact_lock(hashtextextended($1, 6827))`, [workosUserId]); +} + +/** + * Serialize provider upserts with confirmed deletion and refuse resurrection + * after a durable provider-deletion audit exists for the credential. + */ +export async function upsertWorkosUserUnlessConfirmedDeleted( + user: WorkosUserUpsert, +): Promise<boolean> { + const client = await getPool().connect(); + try { + await client.query('BEGIN'); + await lockCredentialMutation(client, user.id); + const tombstone = await client.query( + `SELECT 1 FROM registry_audit_log + WHERE workos_user_id = $1 + AND action IN ('identity_credential_deleted', 'identity_primary_deletion_quarantined') + LIMIT 1`, + [user.id], + ); + if (tombstone.rowCount) { + await client.query('COMMIT'); + return false; + } + await client.query( + `INSERT INTO users ( + workos_user_id, email, first_name, last_name, + email_verified, workos_created_at, workos_updated_at, + created_at, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) + ON CONFLICT (workos_user_id) DO UPDATE SET + email = EXCLUDED.email, + first_name = COALESCE(NULLIF(TRIM(EXCLUDED.first_name), ''), users.first_name), + last_name = COALESCE(NULLIF(TRIM(EXCLUDED.last_name), ''), users.last_name), + email_verified = EXCLUDED.email_verified, + workos_updated_at = EXCLUDED.workos_updated_at, + updated_at = NOW()`, + [user.id, user.email, user.firstName, user.lastName, user.emailVerified, user.createdAt, user.updatedAt], + ); + await client.query('COMMIT'); + return true; + } catch (err) { + await client.query('ROLLBACK').catch(() => undefined); + throw err; + } finally { + client.release(); + } +} + +/** Retrieve the durable, unresolved recovery state for an identity. */ +export async function getIdentityRecoveryQuarantine( + identityId: string, +): Promise<IdentityRecoveryQuarantine | null> { + const result = await getPool().query<{ id: string; details: IdentityRecoveryQuarantine }>( + `SELECT id, details + FROM registry_audit_log + WHERE action = 'identity_primary_deletion_quarantined' + AND resource_type = 'identity_recovery' + AND resource_id = $1 + AND details->>'recovery_state' = $2 + ORDER BY created_at DESC, id DESC + LIMIT 1`, + [identityId, IDENTITY_RECOVERY_STATE], + ); + if (result.rows.length === 0) return null; + return { + ...result.rows[0].details, + audit_id: result.rows[0].id, + }; +} + /** * Apply an authoritative provider deletion without selecting a new primary. * Revoke cached routing for every affected credential in the same transaction * as the binding disappears. Return the IDs to evict from local caches after * commit; their persisted epochs handle caches on other instances. */ -export async function deleteIdentityCredential(workosUserId: string): Promise<string[]> { +export async function deleteIdentityCredentialTransaction( + workosUserId: string, + deletionSource: IdentityCredentialDeletionSource, +): Promise<IdentityCredentialDeletionResult> { const client = await getPool().connect(); - let affectedUserIds: string[] = [workosUserId]; - let orphanedIdentity: { id: string; survivingCredentials: number } | undefined; + let result: IdentityCredentialDeletionResult | undefined; try { await client.query('BEGIN'); - // Use a stable lock order when simultaneous webhooks delete siblings. - // Unlink also needs the binding row lock before it can change the identity. - const bound = await client.query<{ workos_user_id: string; identity_id: string; is_primary: boolean }>( - `SELECT workos_user_id, identity_id, is_primary FROM identity_workos_users - WHERE identity_id = ( - SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1 - ) - ORDER BY workos_user_id - FOR UPDATE`, + const identityLookup = await client.query<{ identity_id: string }>( + `SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1`, [workosUserId], ); - affectedUserIds = [...new Set([ + const identityId = identityLookup.rows[0]?.identity_id; + + // The identity row is the serialization point. Lock it before child rows + // so webhook/backfill deletion of siblings cannot deadlock by first + // holding different credential rows. + const identity = identityId + ? await client.query<JsonRow>( + `SELECT to_jsonb(i) AS row FROM identities i WHERE id = $1 FOR UPDATE OF i`, + [identityId], + ) + : { rows: [] as JsonRow[] }; + // Serialize this credential with WorkOS create/update/backfill upserts. + // Identity comes first whenever it exists, preserving the shared lock order. + await lockCredentialMutation(client, workosUserId); + const bound = identityId + ? await client.query<JsonRow>( + `SELECT to_jsonb(iwu) AS row + FROM identity_workos_users iwu + WHERE iwu.identity_id = $1 + ORDER BY iwu.workos_user_id + FOR UPDATE OF iwu`, + [identityId], + ) + : { rows: [] as JsonRow[] }; + const bindingRows = rowsAsJson(bound); + const targetBinding = bindingRows.find((row) => row.workos_user_id === workosUserId); + if (identityId && !targetBinding) { + // Another confirmed deletion won the identity lock. Do not mutate epochs + // or emit a second audit on the replaying caller. + await client.query('COMMIT'); + return { + deleted: false, + affectedUserIds: [workosUserId], + affectedSlackUserIds: [], + quarantine: null, + }; + } + const affectedUserIds = [...new Set([ workosUserId, - ...bound.rows.map((row) => row.workos_user_id), - ])]; - const deletedBinding = bound.rows.find((row) => row.workos_user_id === workosUserId); - const survivors = bound.rows.filter((row) => row.workos_user_id !== workosUserId); - if (deletedBinding && survivors.length > 0 && !survivors.some((row) => row.is_primary)) { - orphanedIdentity = { id: deletedBinding.identity_id, survivingCredentials: survivors.length }; + ...bindingRows.map((row) => String(row.workos_user_id)), + ])].sort(); + + // Lock and capture the complete authority-bearing before graph in stable + // orders. The graph becomes the durable recovery evidence if this is the + // primary-deletion fault path. + const users = await client.query<JsonRow>( + `SELECT to_jsonb(u) AS row FROM users u + WHERE u.workos_user_id = ANY($1) + ORDER BY u.workos_user_id + FOR UPDATE OF u`, + [affectedUserIds], + ); + const organizationMemberships = await client.query<JsonRow>( + `SELECT to_jsonb(om) AS row FROM organization_memberships om + WHERE om.workos_user_id = ANY($1) + ORDER BY om.workos_user_id, om.workos_organization_id + FOR UPDATE OF om`, + [affectedUserIds], + ); + const workingGroupMemberships = await client.query<JsonRow>( + `SELECT to_jsonb(wgm) AS row FROM working_group_memberships wgm + WHERE wgm.workos_user_id = ANY($1) + ORDER BY wgm.workos_user_id, wgm.working_group_id + FOR UPDATE OF wgm`, + [affectedUserIds], + ); + const workingGroupLeaders = await client.query<JsonRow>( + `SELECT to_jsonb(wgl) AS row FROM working_group_leaders wgl + WHERE wgl.user_id = ANY($1) + ORDER BY wgl.user_id, wgl.working_group_id + FOR UPDATE OF wgl`, + [affectedUserIds], + ); + const slackUserMappings = await client.query<JsonRow>( + `SELECT to_jsonb(sm) AS row FROM slack_user_mappings sm + WHERE sm.workos_user_id = ANY($1) + ORDER BY sm.workos_user_id, sm.slack_user_id + FOR UPDATE OF sm`, + [affectedUserIds], + ); + const authorizationEpochs = await client.query<JsonRow>( + `SELECT to_jsonb(ae) AS row FROM authorization_epochs ae + WHERE ae.workos_user_id = ANY($1) + ORDER BY ae.workos_user_id + FOR UPDATE OF ae`, + [affectedUserIds], + ); + + const userRows = rowsAsJson(users); + const targetUser = userRows.find((row) => row.workos_user_id === workosUserId); + if (!targetUser) { + await client.query('COMMIT'); + return { + deleted: false, + affectedUserIds: [workosUserId], + affectedSlackUserIds: [], + quarantine: null, + }; + } + const survivingBindings = bindingRows.filter((row) => row.workos_user_id !== workosUserId); + const needsQuarantine = targetBinding?.is_primary === true + && survivingBindings.length > 0 + && survivingBindings.every((row) => row.is_primary !== true); + let quarantine: IdentityRecoveryQuarantine | null = null; + const beforeGraph: IdentityBeforeGraph = { + identity: identity.rows[0]?.row ?? (identityId ? { id: identityId } : {}), + identity_workos_users: bindingRows, + users: userRows, + organization_memberships: rowsAsJson(organizationMemberships), + working_group_memberships: rowsAsJson(workingGroupMemberships), + working_group_leaders: rowsAsJson(workingGroupLeaders), + slack_user_mappings: rowsAsJson(slackUserMappings), + authorization_epochs: rowsAsJson(authorizationEpochs), + }; + const targetMembership = beforeGraph.organization_memberships.find( + (row) => row.workos_user_id === workosUserId, + ); + const auditOrganizationId = String( + targetUser.primary_organization_id + ?? targetMembership?.workos_organization_id + ?? 'identity-recovery-unscoped', + ); + const actor = { + type: 'workos_provider' as const, + source: deletionSource, + workos_user_id: workosUserId, + }; + const baseDetails = { + identity_id: identityId ?? null, + deleted_workos_user_id: workosUserId, + deletion_source: deletionSource, + actor, + before_graph: beforeGraph, + }; + const details = needsQuarantine && identityId + ? { ...baseDetails, identity_id: identityId, recovery_state: IDENTITY_RECOVERY_STATE } + : baseDetails; + const auditAction = needsQuarantine + ? 'identity_primary_deletion_quarantined' + : 'identity_credential_deleted'; + const auditResourceType = needsQuarantine ? 'identity_recovery' : 'identity_credential'; + const auditResourceId = identityId ?? workosUserId; + const audit = await client.query<{ id: string }>( + `INSERT INTO registry_audit_log ( + workos_organization_id, workos_user_id, action, + resource_type, resource_id, details + ) VALUES ($1, $2, $3, $4, $5, $6::jsonb) + RETURNING id`, + [auditOrganizationId, workosUserId, auditAction, auditResourceType, auditResourceId, JSON.stringify(details)], + ); + if (audit.rowCount !== 1 || audit.rows.length !== 1 || !audit.rows[0]?.id) { + throw new Error('Confirmed identity credential deletion audit was not durably recorded'); + } + if (needsQuarantine && identityId && 'recovery_state' in details) { + quarantine = { audit_id: audit.rows[0].id, ...details }; } // Authentication checks the actual credential's epoch, so deleting the // primary's epoch alone would leave each surviving credential's cache valid. await bumpAuthorizationEpochs(client, affectedUserIds); - await client.query('DELETE FROM users WHERE workos_user_id = $1', [workosUserId]); await client.query('DELETE FROM organization_memberships WHERE workos_user_id = $1', [workosUserId]); + // Preserve membership history while revoking live group authority. + await client.query( + `UPDATE working_group_memberships + SET status = 'inactive', updated_at = NOW() + WHERE workos_user_id = $1 AND status <> 'inactive'`, + [workosUserId], + ); + // Leadership rows have no inactive state; the complete rows are retained + // in before_graph before live authority is removed. + await client.query('DELETE FROM working_group_leaders WHERE user_id = $1', [workosUserId]); + // Preserve the Slack contact row while severing its WorkOS authority link. + await client.query( + `UPDATE slack_user_mappings + SET workos_user_id = NULL, + mapping_status = 'unmapped', + mapping_source = NULL, + mapped_at = NULL, + mapped_by_user_id = NULL, + updated_at = NOW() + WHERE workos_user_id = $1`, + [workosUserId], + ); + const deleted = await client.query<{ workos_user_id: string }>( + 'DELETE FROM users WHERE workos_user_id = $1 RETURNING workos_user_id', + [workosUserId], + ); + if (quarantine && deleted.rowCount !== 1) { + throw new Error('Quarantined primary credential deletion did not delete exactly one user'); + } await client.query('COMMIT'); + result = { + deleted: deleted.rowCount === 1, + affectedUserIds, + affectedSlackUserIds: beforeGraph.slack_user_mappings + .filter((row) => row.workos_user_id === workosUserId) + .map((row) => String(row.slack_user_id)), + quarantine, + }; } catch (err) { await client.query('ROLLBACK').catch(() => undefined); throw err; @@ -65,31 +373,14 @@ export async function deleteIdentityCredential(workosUserId: string): Promise<st client.release(); } - // Signal the committed access disruption without choosing a successor or - // inferring lost provenance. Rollbacks and replays without a binding do not - // report it. Notification failure must not prevent post-commit invalidation. - if (orphanedIdentity) { - logger.warn({ - deletedUserId: workosUserId, - identityId: orphanedIdentity.id, - survivingCredentialCount: orphanedIdentity.survivingCredentials, - }, 'Provider deletion left surviving credentials without a primary; operator review required'); - try { - notifySystemError({ - source: 'identity-primary-missing', - errorMessage: `Identity ${orphanedIdentity.id} has ${orphanedIdentity.survivingCredentials} surviving credential(s) and no primary after WorkOS user.deleted for ${workosUserId}. Operator review is required; automatic promotion and historical restoration remain disabled.`, - }); - } catch (err) { - logger.error({ err, identityId: orphanedIdentity.id }, 'Failed to report identity without a primary'); - } - } - return affectedUserIds; + return result; } /** * Legacy primary promotion is disabled for #6827: changing the canonical * credential can transfer authority even without moving membership rows. - * Provider deletions use deleteIdentityCredential without inferring a successor. + * Provider deletions use the identity-credential-deletion service without + * inferring a successor. * @throws IdentityMutationDisabledError before any database access. */ export async function promoteSecondaryIfPrimaryDeleted( diff --git a/server/src/routes/workos-webhooks.ts b/server/src/routes/workos-webhooks.ts index 846029fdb72..5671e7da3f6 100644 --- a/server/src/routes/workos-webhooks.ts +++ b/server/src/routes/workos-webhooks.ts @@ -21,8 +21,8 @@ import { Router, Request, Response } from 'express'; import { createLogger } from '../logger.js'; import { getPool } from '../db/client.js'; -import { invalidateSessionsForUsers } from '../middleware/auth.js'; -import { deleteIdentityCredential } from '../db/identity-db.js'; +import { deleteIdentityCredential } from '../services/identity-credential-deletion.js'; +import { upsertWorkosUserUnlessConfirmedDeleted } from '../db/identity-db.js'; import { upsertWorkosDomain, autoPromotePrimaryIfNone, @@ -437,42 +437,26 @@ async function deleteMembership(membership: OrganizationMembershipData): Promise * Upsert user to local users table * Called on user.created and user.updated events */ -async function upsertUser(user: UserData): Promise<void> { +async function upsertUser(user: UserData): Promise<boolean> { const pool = getPool(); const { firstName, lastName } = await resolveUserNameWithFallbacks( pool, user.id, user.first_name, user.last_name, ); - await pool.query( - `INSERT INTO users ( - workos_user_id, - email, - first_name, - last_name, - email_verified, - workos_created_at, - workos_updated_at, - created_at, - updated_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) - ON CONFLICT (workos_user_id) DO UPDATE SET - email = EXCLUDED.email, - first_name = COALESCE(NULLIF(TRIM(EXCLUDED.first_name), ''), users.first_name), - last_name = COALESCE(NULLIF(TRIM(EXCLUDED.last_name), ''), users.last_name), - email_verified = EXCLUDED.email_verified, - workos_updated_at = EXCLUDED.workos_updated_at, - updated_at = NOW()`, - [ - user.id, - user.email, - firstName, - lastName, - user.email_verified, - user.created_at, - user.updated_at, - ] - ); + const upserted = await upsertWorkosUserUnlessConfirmedDeleted({ + id: user.id, + email: user.email, + firstName, + lastName, + emailVerified: user.email_verified, + createdAt: user.created_at, + updatedAt: user.updated_at, + }); + if (!upserted) { + logger.warn({ userId: user.id }, 'Ignored stale WorkOS user event after confirmed deletion'); + return false; + } // Close the user.created vs organization_membership.created race: if a // membership webhook fired first, its backfill UPDATE was a no-op because @@ -493,6 +477,7 @@ async function upsertUser(user: UserData): Promise<void> { } logger.info({ userId: user.id, email: user.email }, 'Upserted user'); + return true; } /** @@ -998,7 +983,10 @@ export function createWorkOSWebhooksRouter(): Router { case 'user.created': { const user = event.data as unknown as UserData; - await upsertUser(user); + if (!(await upsertUser(user))) { + invalidateUnifiedUsersCache(); + break; + } // Try to auto-link to Slack account by email const linkResult = await tryAutoLinkWebsiteUserToSlack(user.id, user.email); if (linkResult.linked) { @@ -1114,7 +1102,10 @@ export function createWorkOSWebhooksRouter(): Router { case 'user.updated': { const user = event.data as unknown as UserData; - await upsertUser(user); + if (!(await upsertUser(user))) { + invalidateUnifiedUsersCache(); + break; + } await updateUserAcrossMemberships(user); invalidateUnifiedUsersCache(); break; @@ -1124,9 +1115,7 @@ export function createWorkOSWebhooksRouter(): Router { const user = event.data as unknown as UserData; // Provider deletion revokes this credential. Do not infer a new // primary: that would redirect surviving credentials' authority. - const affectedUserIds = await deleteIdentityCredential(user.id); - invalidateSessionsForUsers(affectedUserIds); - invalidateUnifiedUsersCache(); + await deleteIdentityCredential(user.id, 'workos_webhook'); break; } @@ -1383,23 +1372,13 @@ export async function backfillUsers(): Promise<{ processedUserIds.add(user.id); try { - await pool.query( - `INSERT INTO users ( - workos_user_id, email, first_name, last_name, - email_verified, workos_created_at, workos_updated_at, - created_at, updated_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) - ON CONFLICT (workos_user_id) DO UPDATE SET - email = EXCLUDED.email, - first_name = COALESCE(NULLIF(TRIM(EXCLUDED.first_name), ''), users.first_name), - last_name = COALESCE(NULLIF(TRIM(EXCLUDED.last_name), ''), users.last_name), - email_verified = EXCLUDED.email_verified, - workos_updated_at = EXCLUDED.workos_updated_at, - updated_at = NOW()`, - [user.id, user.email, user.firstName, user.lastName, - user.emailVerified, user.createdAt, user.updatedAt] - ); - result.usersCreated++; + const upserted = await upsertWorkosUserUnlessConfirmedDeleted(user); + if (upserted) { + result.usersCreated++; + } else { + result.usersSkipped++; + logger.warn({ userId: user.id }, 'Backfill: ignored stale user after confirmed deletion'); + } } catch (userError) { logger.warn({ error: userError, userId: user.id }, 'Backfill: failed to upsert user'); result.errors.push(`Failed to upsert user ${user.id}`); @@ -1502,22 +1481,17 @@ export async function backfillUsers(): Promise<{ result.usersSkipped++; } catch (getErr: any) { if (getErr?.status === 404 || getErr?.code === 'entity_not_found') { - const client = await pool.connect(); try { - await client.query('BEGIN'); - await client.query(`DELETE FROM organization_memberships WHERE workos_user_id = $1`, [row.workos_user_id]); - await client.query(`DELETE FROM users WHERE workos_user_id = $1`, [row.workos_user_id]); - await client.query('COMMIT'); - result.usersRemoved++; - logger.info({ userId: row.workos_user_id }, 'Backfill: removed user confirmed deleted from WorkOS'); + const deletion = await deleteIdentityCredential(row.workos_user_id, 'sync_users_backfill'); + if (deletion.deleted) { + result.usersRemoved++; + logger.info({ userId: row.workos_user_id }, 'Backfill: removed user confirmed deleted from WorkOS'); + } else { + result.usersSkipped++; + } } catch (err) { - await client.query('ROLLBACK'); - // FK constraints prevent deletion of users with platform activity - // (community_points, certifications, etc.) — this is expected result.usersSkipped++; - logger.info({ error: err, userId: row.workos_user_id }, 'Backfill: user deleted from WorkOS but retained locally due to platform activity'); - } finally { - client.release(); + logger.warn({ error: err, userId: row.workos_user_id }, 'Backfill: confirmed user deletion failed locally'); } } else { // WorkOS API error — don't delete, log full error server-side only @@ -1529,7 +1503,8 @@ export async function backfillUsers(): Promise<{ } } - // Invalidate cache after backfill + // Upserts need one final invalidation; confirmed deletions invalidate + // their session and unified-user caches inside deleteIdentityCredential. invalidateUnifiedUsersCache(); logger.info(result, 'Completed users backfill'); diff --git a/server/src/services/identity-credential-deletion.ts b/server/src/services/identity-credential-deletion.ts new file mode 100644 index 00000000000..d4486372649 --- /dev/null +++ b/server/src/services/identity-credential-deletion.ts @@ -0,0 +1,71 @@ +/** + * Single route-facing owner for confirmed WorkOS credential deletion. + * + * The DB helper commits locks, membership deletion, epochs, and durable audit + * together. This service then evicts process-local caches and raises the + * operator alert. No caller may infer or promote a replacement primary. + */ + +import { + deleteIdentityCredentialTransaction, + type IdentityCredentialDeletionResult, + type IdentityCredentialDeletionSource, +} from '../db/identity-db.js'; +import { invalidateSessionsForUsers } from '../middleware/auth.js'; +import { invalidateUnifiedUsersCache } from '../cache/unified-users.js'; +import { notifySystemError } from '../addie/error-notifier.js'; +import { createLogger } from '../logger.js'; +import { + invalidateSlackAdminStatusCache, + invalidateWebAdminStatusCache, +} from '../addie/admin-status-cache.js'; +import { invalidateMemberContextCache } from '../addie/member-context-cache.js'; +import { invalidateHomeCache } from '../addie/home/cache.js'; + +const logger = createLogger('identity-credential-deletion'); + +export async function deleteIdentityCredential( + workosUserId: string, + deletionSource: IdentityCredentialDeletionSource, +): Promise<IdentityCredentialDeletionResult> { + const result = await deleteIdentityCredentialTransaction(workosUserId, deletionSource); + + invalidateSessionsForUsers(result.affectedUserIds); + invalidateUnifiedUsersCache(); + for (const workosUserId of result.affectedUserIds) { + invalidateWebAdminStatusCache(workosUserId); + } + for (const slackUserId of result.affectedSlackUserIds) { + invalidateSlackAdminStatusCache(slackUserId); + invalidateMemberContextCache(slackUserId); + invalidateHomeCache(slackUserId); + } + + if (result.quarantine) { + const { quarantine } = result; + logger.warn({ + identityId: quarantine.identity_id, + deletedWorkosUserId: quarantine.deleted_workos_user_id, + recoveryState: quarantine.recovery_state, + auditId: quarantine.audit_id, + }, 'Primary credential deletion requires manual recovery'); + try { + notifySystemError({ + source: `identity-primary-deletion-quarantine:${quarantine.identity_id}`, + errorMessage: [ + `Primary credential ${quarantine.deleted_workos_user_id} was deleted without promotion.`, + `Identity ${quarantine.identity_id} is quarantined.`, + `recovery_state=${quarantine.recovery_state}.`, + `audit_id=${quarantine.audit_id}.`, + ].join(' '), + }); + } catch (err) { + // Durable recovery state is already committed. Alert transport failure + // must not undo containment or prevent post-commit cache eviction. + logger.error({ err, identityId: quarantine.identity_id, auditId: quarantine.audit_id }, + 'Failed to deliver primary-deletion recovery alert'); + } + } + + return result; +} diff --git a/server/tests/integration/workos-user-deleted-primary.test.ts b/server/tests/integration/workos-user-deleted-primary.test.ts index 7a62c1b040b..17e615fe4df 100644 --- a/server/tests/integration/workos-user-deleted-primary.test.ts +++ b/server/tests/integration/workos-user-deleted-primary.test.ts @@ -5,20 +5,31 @@ */ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; + +vi.mock('../../src/addie/error-notifier.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/addie/error-notifier.js')>()), + notifySystemError: vi.fn(), +})); + import { initializeDatabase, closeDatabase } from '../../src/db/client.js'; import { runMigrations } from '../../src/db/migrate.js'; -import { deleteIdentityCredential, promoteSecondaryIfPrimaryDeleted } from '../../src/db/identity-db.js'; +import { + getIdentityRecoveryQuarantine, + IDENTITY_RECOVERY_STATE, + promoteSecondaryIfPrimaryDeleted, +} from '../../src/db/identity-db.js'; +import { deleteIdentityCredential } from '../../src/services/identity-credential-deletion.js'; import { bumpAuthorizationEpochs, getAuthorizationFingerprint } from '../../src/db/authorization-epoch-db.js'; +import { notifySystemError } from '../../src/addie/error-notifier.js'; +import { isSlackUserAAOAdmin } from '../../src/addie/mcp/admin-tools.js'; +import { + getUnifiedUsersCache, + setUnifiedUsersCache, +} from '../../src/cache/unified-users.js'; import type { Pool } from 'pg'; -const { notifySystemError, identityLogger } = vi.hoisted(() => ({ - notifySystemError: vi.fn(), - identityLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, -})); -vi.mock('../../src/addie/error-notifier.js', () => ({ notifySystemError })); -vi.mock('../../src/logger.js', () => ({ createLogger: () => identityLogger })); - const TEST_USER_PREFIX = 'user_wh_deleted_test_'; +const TEST_SLACK_PREFIX = 'slack_wh_deleted_test_'; const TEST_ORG = 'org_wh_deleted_test_pinnacle'; describe('user.deleted: automatic promotion containment (#6827)', () => { @@ -44,11 +55,19 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { }); async function cleanup() { + await pool.query( + `DELETE FROM registry_audit_log + WHERE action IN ('identity_primary_deletion_quarantined', 'identity_credential_deleted') + AND workos_user_id LIKE $1`, + [`${TEST_USER_PREFIX}%`], + ); await pool.query(`DELETE FROM organization_memberships WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); await pool.query(`DELETE FROM working_group_memberships WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); await pool.query(`DELETE FROM working_group_leaders WHERE user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); + await pool.query(`DELETE FROM slack_user_mappings WHERE slack_user_id LIKE $1`, [`${TEST_SLACK_PREFIX}%`]); await pool.query(`DELETE FROM users WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); await pool.query(`DELETE FROM organizations WHERE workos_organization_id = $1`, [TEST_ORG]); + vi.mocked(notifySystemError).mockClear(); } async function insertUser(suffix: string, email: string): Promise<string> { @@ -172,14 +191,25 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { await pool.query( `INSERT INTO working_group_memberships (working_group_id, workos_user_id, status) SELECT id, $1, 'active' FROM working_groups WHERE slug = 'aao-admin' - UNION ALL SELECT id, $2, 'inactive' FROM working_groups WHERE slug = 'aao-admin'`, [bystander, secondary], + UNION ALL SELECT id, $2, 'active' FROM working_groups WHERE slug = 'aao-admin' + UNION ALL SELECT id, $3, 'inactive' FROM working_groups WHERE slug = 'aao-admin'`, + [primary, bystander, secondary], ); await pool.query( `INSERT INTO working_group_leaders (working_group_id, user_id) - SELECT id, $1 FROM working_groups WHERE slug = 'aao-admin'`, [bystander], + SELECT id, $1 FROM working_groups WHERE slug = 'aao-admin' + UNION ALL SELECT id, $2 FROM working_groups WHERE slug = 'aao-admin'`, + [primary, bystander], + ); + const slackUserId = `${TEST_SLACK_PREFIX}primary`; + await pool.query( + `INSERT INTO slack_user_mappings ( + slack_user_id, slack_email, workos_user_id, mapping_status, mapping_source, mapped_at + ) VALUES ($1, 'jordan-revoke@pinnacle.example', $2, 'mapped', 'manual_admin', NOW())`, + [slackUserId, primary], ); await bumpAuthorizationEpochs(pool, [primary, secondary, bystander]); - return { primary, secondary, bystander, identityId }; + return { primary, secondary, bystander, identityId, slackUserId }; } async function survivorAuthority(ids: string[]) { @@ -189,6 +219,7 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { ['organization_memberships', 'workos_user_id'], ['working_group_memberships', 'workos_user_id'], ['working_group_leaders', 'user_id'], + ['slack_user_mappings', 'workos_user_id'], ]; const state: Record<string, unknown> = {}; for (const [table, column] of tables) { @@ -201,15 +232,81 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { return state; } - it('atomically revokes former sibling caches without promoting or changing their authority', async () => { - const { primary, secondary, bystander, identityId } = await seedDeletionAuthority(); + async function fullBeforeGraph(identityId: string, ids: string[]) { + const queries = { + identity: [`SELECT to_jsonb(i) AS row FROM identities i WHERE id = $1`, [identityId]], + identity_workos_users: [ + `SELECT to_jsonb(iwu) AS row FROM identity_workos_users iwu + WHERE iwu.identity_id = $1 ORDER BY iwu.workos_user_id`, + [identityId], + ], + users: [ + `SELECT to_jsonb(u) AS row FROM users u + WHERE u.workos_user_id = ANY($1) ORDER BY u.workos_user_id`, + [ids], + ], + organization_memberships: [ + `SELECT to_jsonb(om) AS row FROM organization_memberships om + WHERE om.workos_user_id = ANY($1) + ORDER BY om.workos_user_id, om.workos_organization_id`, + [ids], + ], + working_group_memberships: [ + `SELECT to_jsonb(wgm) AS row FROM working_group_memberships wgm + WHERE wgm.workos_user_id = ANY($1) + ORDER BY wgm.workos_user_id, wgm.working_group_id`, + [ids], + ], + working_group_leaders: [ + `SELECT to_jsonb(wgl) AS row FROM working_group_leaders wgl + WHERE wgl.user_id = ANY($1) ORDER BY wgl.user_id, wgl.working_group_id`, + [ids], + ], + slack_user_mappings: [ + `SELECT to_jsonb(sm) AS row FROM slack_user_mappings sm + WHERE sm.workos_user_id = ANY($1) + ORDER BY sm.workos_user_id, sm.slack_user_id`, + [ids], + ], + authorization_epochs: [ + `SELECT to_jsonb(ae) AS row FROM authorization_epochs ae + WHERE ae.workos_user_id = ANY($1) ORDER BY ae.workos_user_id`, + [ids], + ], + } as const; + const entries = await Promise.all(Object.entries(queries).map(async ([key, [sql, params]]) => { + const query = await pool.query(sql, params); + return [key, key === 'identity' ? query.rows[0].row : query.rows.map(({ row }) => row)]; + })); + return Object.fromEntries(entries); + } + + it('serializes webhook-vs-backfill deletion, revokes caches, and records the exact recovery graph once', async () => { + const { primary, secondary, bystander, identityId, slackUserId } = await seedDeletionAuthority(); const survivors = [secondary, bystander]; + const allCredentials = [primary, secondary, bystander].sort(); const authorityBefore = await survivorAuthority(survivors); + const beforeGraph = await fullBeforeGraph(identityId, allCredentials); const fingerprintsBefore = await Promise.all(survivors.map((id) => getAuthorizationFingerprint([id]))); + expect(await isSlackUserAAOAdmin(slackUserId)).toBe(true); + setUnifiedUsersCache(new Map([[TEST_ORG, [{ id: primary, email: 'jordan-revoke@pinnacle.example', firstName: 'Jordan', lastName: 'Test' }]]])); + expect(getUnifiedUsersCache()).not.toBeNull(); + // Alert transport is post-commit and best-effort. Its exact failure path + // must preserve durable quarantine, revocation, and cache invalidation. + vi.mocked(notifySystemError).mockImplementation(() => { + throw new Error('injected operator alert transport failure'); + }); - const results = await Promise.all([deleteIdentityCredential(primary), deleteIdentityCredential(primary)]); + const results = await Promise.all([ + deleteIdentityCredential(primary, 'workos_webhook'), + deleteIdentityCredential(primary, 'sync_users_backfill'), + ]); - expect(results.flat()).toEqual(expect.arrayContaining([primary, secondary, bystander])); + expect(results.filter(({ deleted }) => deleted)).toHaveLength(1); + expect(results.flatMap(({ affectedUserIds }) => affectedUserIds)).toEqual( + expect.arrayContaining([primary, secondary, bystander]), + ); + expect(getUnifiedUsersCache()).toBeNull(); expect(await survivorAuthority(survivors)).toEqual(authorityBefore); for (const [index, id] of survivors.entries()) { expect(await getAuthorizationFingerprint([id])).not.toBe(fingerprintsBefore[index]); @@ -220,25 +317,77 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { `SELECT workos_user_id FROM identity_workos_users WHERE identity_id = $1 AND is_primary = TRUE`, [identityId], )).rows).toEqual([]); - expect(notifySystemError).toHaveBeenCalledOnce(); - expect(identityLogger.warn).toHaveBeenCalledOnce(); - expect(identityLogger.warn.mock.calls[0][0]).toEqual({ - deletedUserId: primary, - identityId, - survivingCredentialCount: 2, + const auditRows = await pool.query( + `SELECT id, details FROM registry_audit_log + WHERE action = 'identity_primary_deletion_quarantined' + AND resource_type = 'identity_recovery' + AND resource_id = $1`, + [identityId], + ); + expect(auditRows.rows).toHaveLength(1); + const durable = await getIdentityRecoveryQuarantine(identityId); + expect(durable).toEqual({ + audit_id: auditRows.rows[0].id, + identity_id: identityId, + deleted_workos_user_id: primary, + deletion_source: expect.stringMatching(/^(workos_webhook|sync_users_backfill)$/), + actor: { + type: 'workos_provider', + source: expect.stringMatching(/^(workos_webhook|sync_users_backfill)$/), + workos_user_id: primary, + }, + recovery_state: IDENTITY_RECOVERY_STATE, + before_graph: beforeGraph, }); + expect(notifySystemError).toHaveBeenCalledTimes(1); expect(notifySystemError).toHaveBeenCalledWith({ - source: 'identity-primary-missing', - errorMessage: expect.stringContaining(identityId), + source: `identity-primary-deletion-quarantine:${identityId}`, + errorMessage: [ + `Primary credential ${primary} was deleted without promotion.`, + `Identity ${identityId} is quarantined.`, + `recovery_state=${IDENTITY_RECOVERY_STATE}.`, + `audit_id=${auditRows.rows[0].id}.`, + ].join(' '), }); - expect(JSON.stringify(notifySystemError.mock.calls)).not.toContain('@'); + expect((await pool.query( + `SELECT status FROM working_group_memberships WHERE workos_user_id = $1`, + [primary], + )).rows).toEqual([{ status: 'inactive' }]); + expect((await pool.query( + `SELECT 1 FROM working_group_leaders WHERE user_id = $1`, + [primary], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT workos_user_id, mapping_status FROM slack_user_mappings WHERE slack_user_id = $1`, + [slackUserId], + )).rows).toEqual([{ workos_user_id: null, mapping_status: 'unmapped' }]); + expect(await isSlackUserAAOAdmin(slackUserId)).toBe(false); const fingerprintAfter = await getAuthorizationFingerprint(survivors); - await deleteIdentityCredential(primary); + await deleteIdentityCredential(primary, 'workos_webhook'); expect(await getAuthorizationFingerprint(survivors)).toBe(fingerprintAfter); expect(await survivorAuthority(survivors)).toEqual(authorityBefore); - expect(notifySystemError).toHaveBeenCalledOnce(); - expect(identityLogger.warn).toHaveBeenCalledOnce(); + expect((await pool.query( + `SELECT 1 FROM registry_audit_log + WHERE action = 'identity_primary_deletion_quarantined' AND resource_id = $1`, + [identityId], + )).rows).toHaveLength(1); + expect(notifySystemError).toHaveBeenCalledTimes(1); + }); + + it('deletes a non-primary credential without quarantining or changing the primary', async () => { + const { primary, secondary, identityId } = await seedDeletionAuthority(); + + const deletion = await deleteIdentityCredential(secondary, 'workos_webhook'); + + expect(deletion).toMatchObject({ deleted: true, quarantine: null }); + expect(await getIdentityRecoveryQuarantine(identityId)).toBeNull(); + expect((await pool.query( + `SELECT workos_user_id FROM identity_workos_users + WHERE identity_id = $1 AND is_primary = TRUE`, + [identityId], + )).rows).toEqual([{ workos_user_id: primary }]); + expect(notifySystemError).not.toHaveBeenCalled(); }); it('rolls back epoch changes and deletion together when the database refuses deletion', async () => { @@ -254,118 +403,49 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { ); try { await pool.query(`INSERT INTO identity_containment_delete_guard VALUES ($1)`, [primary]); - await expect(deleteIdentityCredential(primary)).rejects.toThrow(); + await expect(deleteIdentityCredential(primary, 'workos_webhook')).rejects.toThrow(); expect(await survivorAuthority(credentials)).toEqual(before); expect(await getAuthorizationFingerprint(credentials)).toBe(fingerprintBefore); + expect(await getIdentityRecoveryQuarantine((await pool.query<{ identity_id: string }>( + `SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1`, + [primary], + )).rows[0].identity_id)).toBeNull(); expect(notifySystemError).not.toHaveBeenCalled(); - expect(identityLogger.warn).not.toHaveBeenCalled(); } finally { await pool.query(`DROP TABLE identity_containment_delete_guard`); } }); - it('signals only after commit and connection release, with committed deletion visible to another connection', async () => { + it('rolls back deletion and epochs when the durable quarantine audit write fails', async () => { const { primary, secondary, bystander, identityId } = await seedDeletionAuthority(); - const observer = await pool.connect(); + const credentials = [primary, secondary, bystander]; + const before = await survivorAuthority(credentials); + const fingerprintBefore = await getAuthorizationFingerprint(credentials); + await pool.query(` + CREATE OR REPLACE FUNCTION reject_identity_quarantine_audit() RETURNS trigger AS $$ + BEGIN + IF NEW.action = 'identity_primary_deletion_quarantined' THEN + RAISE EXCEPTION 'injected quarantine audit failure'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + CREATE TRIGGER reject_identity_quarantine_audit + BEFORE INSERT ON registry_audit_log + FOR EACH ROW EXECUTE FUNCTION reject_identity_quarantine_audit(); + `); try { - // Keep an independent connection for visibility checks and leave another - // idle connection for deletion, so release ordering is observable. - await pool.query('SELECT 1'); - const idleBeforeDeletion = pool.idleCount; - expect(idleBeforeDeletion).toBeGreaterThan(0); - let idleAtSignal = -1; - let stateAtSignal: Promise<{ rows: unknown[] }> | undefined; - notifySystemError.mockImplementation(() => { - idleAtSignal = pool.idleCount; - stateAtSignal = observer.query( - `SELECT - EXISTS (SELECT 1 FROM users WHERE workos_user_id = $1) AS deleted_user_exists, - EXISTS (SELECT 1 FROM organization_memberships WHERE workos_user_id = $1) AS deleted_membership_exists, - (SELECT COUNT(*)::int FROM identity_workos_users WHERE identity_id = $2) AS survivors, - (SELECT COUNT(*)::int FROM identity_workos_users WHERE identity_id = $2 AND is_primary) AS primaries, - (SELECT MIN(epoch)::int FROM authorization_epochs WHERE workos_user_id = ANY($3)) AS survivor_epoch`, - [primary, identityId, [secondary, bystander]], - ); - }); - - await deleteIdentityCredential(primary); - - expect(notifySystemError).toHaveBeenCalledOnce(); - expect(idleAtSignal).toBe(idleBeforeDeletion); - expect((await stateAtSignal)?.rows).toEqual([{ - deleted_user_exists: false, - deleted_membership_exists: false, - survivors: 2, - primaries: 0, - survivor_epoch: 2, - }]); + await expect(deleteIdentityCredential(primary, 'sync_users_backfill')) + .rejects.toThrow('injected quarantine audit failure'); + expect(await survivorAuthority(credentials)).toEqual(before); + expect(await getAuthorizationFingerprint(credentials)).toBe(fingerprintBefore); + expect(await getIdentityRecoveryQuarantine(identityId)).toBeNull(); + expect(notifySystemError).not.toHaveBeenCalled(); } finally { - observer.release(); - } - }); - - it.each([true, false])('does not signal when deleting the last binding (is_primary=%s) or replaying it', async (isPrimary) => { - const userId = await insertUser('signal_singleton', 'jordan-singleton@pinnacle.example'); - if (!isPrimary) { - await pool.query(`UPDATE identity_workos_users SET is_primary = FALSE WHERE workos_user_id = $1`, [userId]); + await pool.query(`DROP TRIGGER reject_identity_quarantine_audit ON registry_audit_log`); + await pool.query(`DROP FUNCTION reject_identity_quarantine_audit()`); } - - expect(await deleteIdentityCredential(userId)).toEqual([userId]); - expect(await deleteIdentityCredential(userId)).toEqual([userId]); - - expect(notifySystemError).not.toHaveBeenCalled(); - expect(identityLogger.warn).not.toHaveBeenCalled(); - }); - - it('does not signal secondary deletion while the original primary survives', async () => { - const { primary, secondary, bystander } = await seedDeletionAuthority(); - const before = await survivorAuthority([primary, bystander]); - - await deleteIdentityCredential(secondary); - - expect(await survivorAuthority([primary, bystander])).toEqual(before); - expect(notifySystemError).not.toHaveBeenCalled(); - expect(identityLogger.warn).not.toHaveBeenCalled(); }); - - it('signals an existing identity without a primary when another credential is deleted', async () => { - const { primary, secondary, bystander, identityId } = await seedDeletionAuthority(); - // Historical invalid state is an input, never permission to choose a primary. - await pool.query(`UPDATE identity_workos_users SET is_primary = FALSE WHERE workos_user_id = $1`, [primary]); - const before = await survivorAuthority([primary, bystander]); - - await deleteIdentityCredential(secondary); - - expect(await survivorAuthority([primary, bystander])).toEqual(before); - expect(notifySystemError).toHaveBeenCalledWith({ - source: 'identity-primary-missing', - errorMessage: expect.stringContaining(identityId), - }); - expect(identityLogger.warn.mock.calls[0][0]).toEqual({ - deletedUserId: secondary, - identityId, - survivingCredentialCount: 2, - }); - }); - - it('returns cache invalidation IDs and preserves committed revocation if notifying throws', async () => { - const { primary, secondary, bystander } = await seedDeletionAuthority(); - const before = await survivorAuthority([secondary, bystander]); - const notificationFailure = new Error('Notification transport unavailable'); - notifySystemError.mockImplementation(() => { throw notificationFailure; }); - - await expect(deleteIdentityCredential(primary)).resolves.toEqual(expect.arrayContaining([primary, secondary, bystander])); - - expect((await pool.query(`SELECT 1 FROM users WHERE workos_user_id = $1`, [primary])).rows).toEqual([]); - expect(await survivorAuthority([secondary, bystander])).toEqual(before); - expect(await getAuthorizationFingerprint([secondary])).toBe(`${secondary}:2`); - expect(await getAuthorizationFingerprint([bystander])).toBe(`${bystander}:2`); - expect(identityLogger.error).toHaveBeenCalledWith( - expect.objectContaining({ err: notificationFailure }), - expect.any(String), - ); - }); - it('deletes an account with accumulated community data', async () => { const userId = await insertUser('community', 'community@test.example'); await pool.query( diff --git a/server/tests/integration/workos-user-deletion-race.test.ts b/server/tests/integration/workos-user-deletion-race.test.ts new file mode 100644 index 00000000000..3a0927735bc --- /dev/null +++ b/server/tests/integration/workos-user-deletion-race.test.ts @@ -0,0 +1,368 @@ +import express from 'express'; +import request from 'supertest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Pool } from 'pg'; + +const mocks = vi.hoisted(() => ({ + constructEvent: vi.fn(), + getUser: vi.fn(), + listUsers: vi.fn(), + invalidateSessionsForUsers: vi.fn(), + notifySystemError: vi.fn(), + deletionEntrants: [] as Array<{ workosUserId: string; source: string }>, + deletionWaiters: [] as Array<() => void>, +})); + +vi.mock('../../src/auth/workos-client.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/auth/workos-client.js')>()), + getWorkos: () => ({ + webhooks: { constructEvent: mocks.constructEvent }, + userManagement: { + getUser: mocks.getUser, + listUsers: mocks.listUsers, + }, + }), +})); + +vi.mock('../../src/middleware/auth.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/middleware/auth.js')>()), + invalidateSessionsForUsers: mocks.invalidateSessionsForUsers, +})); + +vi.mock('../../src/addie/error-notifier.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/addie/error-notifier.js')>()), + notifySystemError: mocks.notifySystemError, +})); + +vi.mock('../../src/services/identity-credential-deletion.js', async (importOriginal) => { + const actual = await importOriginal<typeof import('../../src/services/identity-credential-deletion.js')>(); + return { + ...actual, + deleteIdentityCredential: async ( + workosUserId: string, + source: 'workos_webhook' | 'sync_users_backfill', + ) => { + if (workosUserId === 'user_wh_backfill_race_primary') { + mocks.deletionEntrants.push({ workosUserId, source }); + await new Promise<void>((resolve) => { + mocks.deletionWaiters.push(resolve); + if (mocks.deletionWaiters.length === 2) { + for (const release of mocks.deletionWaiters.splice(0)) release(); + } + }); + } + return actual.deleteIdentityCredential(workosUserId, source); + }, + }; +}); + +process.env.WORKOS_WEBHOOK_SECRET = 'test_workos_webhook_secret'; + +const { initializeDatabase, closeDatabase } = await import('../../src/db/client.js'); +const { runMigrations } = await import('../../src/db/migrate.js'); +const { getAuthorizationFingerprint } = await import('../../src/db/authorization-epoch-db.js'); +const { getIdentityRecoveryQuarantine, IDENTITY_RECOVERY_STATE } = await import('../../src/db/identity-db.js'); +const { createWorkOSWebhooksRouter, backfillUsers } = await import('../../src/routes/workos-webhooks.js'); +const { getUnifiedUsersCache, setUnifiedUsersCache } = await import('../../src/cache/unified-users.js'); +const { isSlackUserAAOAdmin } = await import('../../src/addie/mcp/admin-tools.js'); + +const PREFIX = 'user_wh_backfill_race_'; +const PRIMARY = `${PREFIX}primary`; +const SECONDARY = `${PREFIX}secondary`; +const BYSTANDER = `${PREFIX}bystander`; +const ORG = 'org_wh_backfill_race_pinnacle'; +const SLACK = 'slack_wh_backfill_race_primary'; + +describe('WorkOS webhook vs sync-users deletion', () => { + let pool: Pool; + + beforeAll(async () => { + pool = initializeDatabase({ + connectionString: + process.env.DATABASE_URL || 'postgresql://adcp:localdev@localhost:5432/adcp_test', + }); + await runMigrations(); + }, 60_000); + + afterAll(async () => { + await cleanup(); + await closeDatabase(); + }); + + beforeEach(async () => { + await cleanup(); + vi.clearAllMocks(); + mocks.deletionEntrants.length = 0; + mocks.deletionWaiters.length = 0; + }); + + async function cleanup() { + await pool.query(`DROP TRIGGER IF EXISTS delay_racing_user_delete ON users`); + await pool.query(`DROP FUNCTION IF EXISTS delay_racing_user_delete()`); + await pool.query( + `DELETE FROM registry_audit_log + WHERE action IN ('identity_primary_deletion_quarantined', 'identity_credential_deleted') + AND workos_user_id LIKE $1`, + [`${PREFIX}%`], + ); + await pool.query(`DELETE FROM working_group_memberships WHERE workos_user_id LIKE $1`, [`${PREFIX}%`]); + await pool.query(`DELETE FROM working_group_leaders WHERE user_id LIKE $1`, [`${PREFIX}%`]); + await pool.query(`DELETE FROM slack_user_mappings WHERE slack_user_id = $1`, [SLACK]); + await pool.query(`DELETE FROM organization_memberships WHERE workos_user_id LIKE $1`, [`${PREFIX}%`]); + await pool.query(`DELETE FROM users WHERE workos_user_id LIKE $1`, [`${PREFIX}%`]); + await pool.query(`DELETE FROM organizations WHERE workos_organization_id = $1`, [ORG]); + } + + async function seedIdentity() { + await pool.query( + `INSERT INTO organizations (workos_organization_id, name) VALUES ($1, 'Pinnacle Agency')`, + [ORG], + ); + for (const [id, email] of [ + [PRIMARY, 'jordan-race@pinnacle.example'], + [SECONDARY, 'sam-race@pinnacle.example'], + [BYSTANDER, 'maya-race@pinnacle.example'], + ]) { + await pool.query( + `INSERT INTO users ( + workos_user_id, email, first_name, last_name, email_verified, + workos_created_at, workos_updated_at, created_at, updated_at, + primary_organization_id + ) VALUES ($1, $2, 'Test', 'User', TRUE, NOW(), NOW(), NOW(), NOW(), $3)`, + [id, email, ORG], + ); + await pool.query( + `INSERT INTO organization_memberships ( + workos_user_id, workos_organization_id, email, role + ) VALUES ($1, $2, $3, 'member')`, + [id, ORG, email], + ); + } + + const identities = await pool.query<{ workos_user_id: string; identity_id: string }>( + `SELECT workos_user_id, identity_id FROM identity_workos_users + WHERE workos_user_id = ANY($1)`, + [[PRIMARY, SECONDARY, BYSTANDER]], + ); + const byUser = new Map(identities.rows.map((row) => [row.workos_user_id, row.identity_id])); + const identityId = byUser.get(PRIMARY)!; + await pool.query( + `UPDATE identity_workos_users + SET identity_id = $1, is_primary = FALSE + WHERE workos_user_id = ANY($2)`, + [identityId, [SECONDARY, BYSTANDER]], + ); + await pool.query(`DELETE FROM identities WHERE id = ANY($1)`, [ + [byUser.get(SECONDARY), byUser.get(BYSTANDER)], + ]); + await pool.query( + `INSERT INTO working_group_memberships (working_group_id, workos_user_id, status) + SELECT id, $1, 'active' FROM working_groups WHERE slug = 'aao-admin'`, + [PRIMARY], + ); + await pool.query( + `INSERT INTO working_group_leaders (working_group_id, user_id) + SELECT id, $1 FROM working_groups WHERE slug = 'aao-admin'`, + [PRIMARY], + ); + await pool.query( + `INSERT INTO slack_user_mappings ( + slack_user_id, slack_email, workos_user_id, mapping_status, mapping_source, mapped_at + ) VALUES ($1, 'jordan-race@pinnacle.example', $2, 'mapped', 'manual_admin', NOW())`, + [SLACK, PRIMARY], + ); + return identityId; + } + + it('serializes both real callers and invalidates cached authority exactly through the helper', async () => { + const identityId = await seedIdentity(); + expect((await pool.query<{ workos_user_id: string }>( + `SELECT workos_user_id FROM users WHERE workos_user_id LIKE $1 ORDER BY workos_user_id`, + [`${PREFIX}%`], + )).rows.map((row) => row.workos_user_id)).toEqual([BYSTANDER, PRIMARY, SECONDARY].sort()); + expect(await isSlackUserAAOAdmin(SLACK)).toBe(true); + const survivorFingerprintsBefore = await Promise.all( + [SECONDARY, BYSTANDER].map((id) => getAuthorizationFingerprint([id])), + ); + setUnifiedUsersCache(new Map([[ORG, [{ + id: PRIMARY, + email: 'jordan-race@pinnacle.example', + firstName: 'Jordan', + lastName: 'Test', + }]]])); + + mocks.listUsers.mockImplementation(async (input: { organizationId?: string }) => ({ + data: input.organizationId + ? [] + : [ + { + id: SECONDARY, + email: 'sam-race@pinnacle.example', + firstName: 'Sam', + lastName: 'Test', + emailVerified: true, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + { + id: BYSTANDER, + email: 'maya-race@pinnacle.example', + firstName: 'Maya', + lastName: 'Test', + emailVerified: true, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + ], + })); + mocks.getUser.mockImplementation(async (workosUserId: string) => { + if (workosUserId === PRIMARY) { + throw Object.assign(new Error('not found'), { status: 404 }); + } + return { id: workosUserId }; + }); + mocks.constructEvent.mockResolvedValue(undefined); + + // Keep the winning transaction open briefly so the other real caller + // reaches the identity lock and exercises PostgreSQL serialization. + await pool.query(` + CREATE FUNCTION delay_racing_user_delete() RETURNS trigger AS $$ + BEGIN + IF OLD.workos_user_id = '${PRIMARY}' THEN + PERFORM pg_sleep(0.15); + END IF; + RETURN OLD; + END; + $$ LANGUAGE plpgsql; + CREATE TRIGGER delay_racing_user_delete + BEFORE DELETE ON users + FOR EACH ROW EXECUTE FUNCTION delay_racing_user_delete(); + `); + + const app = express(); + app.use('/api/webhooks', createWorkOSWebhooksRouter()); + const webhookEvent = { + id: 'event_wh_backfill_race', + event: 'user.deleted', + data: { id: PRIMARY }, + created_at: new Date().toISOString(), + }; + + const [webhookResponse, backfill] = await Promise.all([ + request(app) + .post('/api/webhooks/workos') + .set('Content-Type', 'application/json') + .set('WorkOS-Signature', 'valid-test-signature') + .send(webhookEvent), + backfillUsers(), + ]); + + expect(webhookResponse.status).toBe(200); + expect(mocks.deletionEntrants).toEqual(expect.arrayContaining([ + { workosUserId: PRIMARY, source: 'workos_webhook' }, + { workosUserId: PRIMARY, source: 'sync_users_backfill' }, + ])); + expect(mocks.deletionEntrants).toHaveLength(2); + expect(backfill.usersRemoved + backfill.usersSkipped).toBeGreaterThanOrEqual(1); + expect((await pool.query(`SELECT 1 FROM users WHERE workos_user_id = $1`, [PRIMARY])).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM organization_memberships WHERE workos_user_id = $1`, + [PRIMARY], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT workos_user_id FROM identity_workos_users + WHERE identity_id = $1 AND is_primary = TRUE`, + [identityId], + )).rows).toEqual([]); + expect(await getIdentityRecoveryQuarantine(identityId)).toMatchObject({ + identity_id: identityId, + deleted_workos_user_id: PRIMARY, + deletion_source: expect.stringMatching(/^(workos_webhook|sync_users_backfill)$/), + recovery_state: IDENTITY_RECOVERY_STATE, + }); + const quarantineAudits = (await pool.query<{ id: string }>( + `SELECT id FROM registry_audit_log + WHERE action = 'identity_primary_deletion_quarantined' AND resource_id = $1`, + [identityId], + )).rows; + expect(quarantineAudits).toHaveLength(1); + for (const [index, id] of [SECONDARY, BYSTANDER].entries()) { + expect(await getAuthorizationFingerprint([id])).not.toBe(survivorFingerprintsBefore[index]); + } + expect(mocks.invalidateSessionsForUsers).toHaveBeenCalledWith( + expect.arrayContaining([PRIMARY, SECONDARY, BYSTANDER]), + ); + expect(getUnifiedUsersCache()).toBeNull(); + expect(mocks.notifySystemError).toHaveBeenCalledTimes(1); + expect(mocks.notifySystemError).toHaveBeenCalledWith({ + source: `identity-primary-deletion-quarantine:${identityId}`, + errorMessage: [ + `Primary credential ${PRIMARY} was deleted without promotion.`, + `Identity ${identityId} is quarantined.`, + `recovery_state=${IDENTITY_RECOVERY_STATE}.`, + `audit_id=${quarantineAudits[0].id}.`, + ].join(' '), + }); + expect((await pool.query( + `SELECT status FROM working_group_memberships WHERE workos_user_id = $1`, + [PRIMARY], + )).rows).toEqual([{ status: 'inactive' }]); + expect((await pool.query( + `SELECT 1 FROM working_group_leaders WHERE user_id = $1`, + [PRIMARY], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT workos_user_id, mapping_status FROM slack_user_mappings WHERE slack_user_id = $1`, + [SLACK], + )).rows).toEqual([{ workos_user_id: null, mapping_status: 'unmapped' }]); + expect(await isSlackUserAAOAdmin(SLACK)).toBe(false); + + // A stale listUsers page after the signed deletion must not resurrect the + // users row, automatic identity binding, or any authority. + mocks.listUsers.mockImplementation(async (input: { organizationId?: string }) => ({ + data: input.organizationId + ? [] + : [ + { + id: PRIMARY, + email: 'jordan-race@pinnacle.example', + firstName: 'Jordan', + lastName: 'Test', + emailVerified: true, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + { + id: SECONDARY, + email: 'sam-race@pinnacle.example', + firstName: 'Sam', + lastName: 'Test', + emailVerified: true, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + { + id: BYSTANDER, + email: 'maya-race@pinnacle.example', + firstName: 'Maya', + lastName: 'Test', + emailVerified: true, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + ], + })); + const staleReplay = await backfillUsers(); + expect(staleReplay.usersSkipped).toBeGreaterThanOrEqual(1); + expect((await pool.query(`SELECT 1 FROM users WHERE workos_user_id = $1`, [PRIMARY])).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM identity_workos_users WHERE workos_user_id = $1`, + [PRIMARY], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM organization_memberships WHERE workos_user_id = $1`, + [PRIMARY], + )).rows).toEqual([]); + expect(await isSlackUserAAOAdmin(SLACK)).toBe(false); + }); +}); diff --git a/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts b/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts index 03d56407d57..2a183583d4f 100644 --- a/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts +++ b/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts @@ -9,6 +9,8 @@ import * as path from 'path'; const WEBHOOK_FILE = path.resolve(__dirname, '../../src/routes/workos-webhooks.ts'); const webhookSource = fs.readFileSync(WEBHOOK_FILE, 'utf-8'); +const DELETION_SERVICE_FILE = path.resolve(__dirname, '../../src/services/identity-credential-deletion.ts'); +const deletionServiceSource = fs.readFileSync(DELETION_SERVICE_FILE, 'utf-8'); describe('user.deleted containment wiring (#6827)', () => { const userDeletedBlockMatch = webhookSource.match( @@ -22,11 +24,29 @@ describe('user.deleted containment wiring (#6827)', () => { it('preserves credential revocation without promoting a successor', () => { expect(block).not.toContain('promoteSecondaryIfPrimaryDeleted'); - expect(block).toContain('await deleteIdentityCredential(user.id)'); + expect(block).toContain("await deleteIdentityCredential(user.id, 'workos_webhook')"); }); - it('invalidates session caches for the deleted user', () => { - expect(block).toContain('invalidateSessionsForUsers(affectedUserIds)'); - expect(block.indexOf('await deleteIdentityCredential')).toBeLessThan(block.indexOf('invalidateSessionsForUsers')); + it('keeps session and unified cache invalidation inside the deletion helper', () => { + expect(block).not.toContain('invalidateSessionsForUsers'); + expect(block).not.toContain('invalidateUnifiedUsersCache'); + expect(deletionServiceSource).toContain('invalidateSessionsForUsers(result.affectedUserIds)'); + expect(deletionServiceSource).toContain('invalidateUnifiedUsersCache()'); + expect(deletionServiceSource).toContain('invalidateSlackAdminStatusCache(slackUserId)'); + expect(deletionServiceSource).toContain('invalidateWebAdminStatusCache(workosUserId)'); + expect(deletionServiceSource).toContain('invalidateMemberContextCache(slackUserId)'); + }); + + it('routes sync-users confirmed deletion through the same helper', () => { + const backfillBlock = webhookSource.slice( + webhookSource.indexOf('export async function backfillUsers'), + webhookSource.indexOf('export async function backfillOrganizationDomains'), + ); + expect(backfillBlock).toContain( + "deleteIdentityCredential(row.workos_user_id, 'sync_users_backfill')", + ); + expect(backfillBlock).toContain('upsertWorkosUserUnlessConfirmedDeleted(user)'); + expect(backfillBlock).not.toMatch(/DELETE FROM organization_memberships/); + expect(backfillBlock).not.toMatch(/DELETE FROM users/); }); }); From 666908562a8e72331886dcd8a026b9e56c4af4db Mon Sep 17 00:00:00 2001 From: Brian O'Kelley <bokelley@scope3.com> Date: Sun, 13 Sep 2026 15:53:16 +0000 Subject: [PATCH 5/6] fix(auth): close credential deletion races --- server/src/db/authorization-epoch-db.ts | 137 ++- server/src/db/identity-db.ts | 886 ++++++++++++++++-- server/src/db/mcp-oauth-state-db.ts | 5 +- server/src/db/meetings-db.ts | 25 +- server/src/db/membership-db.ts | 73 +- server/src/db/organization-db.ts | 15 +- server/src/db/slack-db.ts | 15 +- server/src/db/working-group-db.ts | 257 ++--- server/src/http.ts | 116 ++- server/src/mcp/oauth-provider.ts | 89 +- server/src/middleware/auth.ts | 147 +-- server/src/routes/workos-webhooks.ts | 386 ++++---- .../services/identity-credential-deletion.ts | 6 +- server/src/slack/sync.ts | 22 +- .../tests/integration/admin-add-users.test.ts | 13 + ...thorization-deletion-cross-replica.test.ts | 163 ++++ .../integration/authorization-epoch.test.ts | 20 +- .../mcp-oauth-deletion-race.test.ts | 114 +++ .../integration/membership-webhook.test.ts | 116 ++- .../workos-user-deleted-primary.test.ts | 660 ++++++++++++- .../workos-user-deletion-race.test.ts | 672 ++++++++++++- .../unit/aao-admin-membership-audit.test.ts | 32 +- server/tests/unit/aao-admin-status.test.ts | 7 + .../unit/addie-admin-global-auth.test.ts | 69 ++ .../unit/authorization-epoch-session.test.ts | 174 +++- .../unit/login-name-preservation.test.ts | 41 +- .../unit/native-auth-http-wiring.test.ts | 31 +- .../unit/network-health-security.test.ts | 20 + .../working-group-admin-global-auth.test.ts | 20 + ...workos-webhook-inactive-membership.test.ts | 2 +- ...workos-webhook-user-deleted-wiring.test.ts | 44 + 31 files changed, 3684 insertions(+), 693 deletions(-) create mode 100644 server/tests/integration/authorization-deletion-cross-replica.test.ts create mode 100644 server/tests/integration/mcp-oauth-deletion-race.test.ts diff --git a/server/src/db/authorization-epoch-db.ts b/server/src/db/authorization-epoch-db.ts index 493b3747d22..aba3f6bec66 100644 --- a/server/src/db/authorization-epoch-db.ts +++ b/server/src/db/authorization-epoch-db.ts @@ -7,9 +7,10 @@ * the auth middleware stamps the observed fingerprint onto its cache entries * and drops any entry whose fingerprint no longer matches. * - * The fingerprint is compared for *inequality*, never ordering: a CASCADE - * delete (WorkOS `user.deleted`) removes a row, which moves the fingerprint - * backwards and must still invalidate. + * The fingerprint is compared for *inequality*, never ordering. Confirmed + * deletion contributes its durable, non-cascading audit tombstone, so even an + * epoch-0 credential moves from the empty fingerprint and cannot survive in a + * different process's session/JWT cache after the users-row CASCADE. */ import type { Pool, PoolClient } from 'pg'; @@ -18,6 +19,24 @@ import { query } from './client.js'; /** Anything that can run a parameterized statement — pool or in-transaction client. */ type Queryable = Pick<Pool | PoolClient, 'query'>; +export interface ActiveCredentialAuthorizationSnapshot { + workos_user_id: string; + email: string | null; + first_name: string | null; + last_name: string | null; + identity_id: string; + primary_workos_user_id: string | null; + fingerprint: string; +} + +export type CredentialAuthorizationLifecycle = + | { status: 'active'; snapshot: ActiveCredentialAuthorizationSnapshot } + | { + status: 'terminal'; + reason: 'deleted_or_quarantined' | 'missing_user' | 'missing_binding' | 'missing_primary'; + } + | { status: 'unavailable' }; + /** * Bump the authorization epoch for each credential, in the caller's * transaction. Call this in the same transaction as the binding mutation: @@ -31,17 +50,19 @@ type Queryable = Pick<Pool | PoolClient, 'query'>; export async function bumpAuthorizationEpochs( db: Queryable, workosUserIds: string[], -): Promise<void> { +): Promise<string[]> { const ids = [...new Set(workosUserIds.filter(Boolean))]; - if (ids.length === 0) return; + if (ids.length === 0) return []; - await db.query( + const bumped = await db.query<{ workos_user_id: string }>( `INSERT INTO authorization_epochs (workos_user_id, epoch) SELECT u.workos_user_id, 1 FROM users u WHERE u.workos_user_id = ANY($1) ON CONFLICT (workos_user_id) DO UPDATE - SET epoch = authorization_epochs.epoch + 1, updated_at = NOW()`, + SET epoch = authorization_epochs.epoch + 1, updated_at = NOW() + RETURNING workos_user_id`, [ids], ); + return bumped.rows.map((row) => row.workos_user_id); } /** @@ -60,11 +81,105 @@ export async function getAuthorizationFingerprint( if (ids.length === 0) return ''; const result = await query<{ fingerprint: string | null }>( - `SELECT string_agg(workos_user_id || ':' || epoch, ',' ORDER BY workos_user_id) - AS fingerprint - FROM authorization_epochs - WHERE workos_user_id = ANY($1)`, + `WITH fingerprint_parts AS ( + SELECT ae.workos_user_id || ':epoch:' || ae.epoch AS part + FROM authorization_epochs ae + WHERE ae.workos_user_id = ANY($1) + UNION ALL + SELECT ral.workos_user_id || ':deleted:' || ral.id AS part + FROM registry_audit_log ral + WHERE ral.workos_user_id = ANY($1) + AND ral.action IN ('identity_credential_deleted', 'identity_primary_deletion_quarantined') + ) + SELECT string_agg(part, ',' ORDER BY part) AS fingerprint + FROM fingerprint_parts`, [ids], ); return result.rows[0]?.fingerprint ?? ''; } + +/** + * Authoritative positive-authentication snapshot for one real credential. + * + * A fingerprint alone is not proof that the credential is live: after + * deletion the durable marker is itself a stable fingerprint. Positive auth + * cache acceptance and writes must therefore read the local user, identity + * binding, absence of the deletion tombstone, canonical route, and epoch in + * one database statement. Missing/corrupt/deleted credentials are terminal; + * database errors become an explicit unavailable state so callers fail closed. + */ +export async function readCredentialAuthorizationLifecycle( + workosUserId: string, +): Promise<CredentialAuthorizationLifecycle> { + try { + const result = await query<ActiveCredentialAuthorizationSnapshot & { + user_exists: boolean; + binding_exists: boolean; + primary_count: string; + terminal_marker: boolean; + }>( + `WITH requested AS ( + SELECT $1::text AS workos_user_id + ) + SELECT requested.workos_user_id, + u.email, + u.first_name, + u.last_name, + iwu.identity_id, + primary_binding.workos_user_id AS primary_workos_user_id, + CASE WHEN ae.workos_user_id IS NULL THEN '' + ELSE ae.workos_user_id || ':epoch:' || ae.epoch + END AS fingerprint, + (u.workos_user_id IS NOT NULL) AS user_exists, + (iwu.workos_user_id IS NOT NULL) AS binding_exists, + COALESCE(primary_binding.primary_count, 0)::text AS primary_count, + EXISTS ( + SELECT 1 + FROM registry_audit_log ral + WHERE ral.workos_user_id = requested.workos_user_id + AND ral.action IN ( + 'identity_credential_deleted', + 'identity_primary_deletion_quarantined' + ) + ) AS terminal_marker + FROM requested + LEFT JOIN users u + ON u.workos_user_id = requested.workos_user_id + LEFT JOIN identity_workos_users iwu + ON iwu.workos_user_id = requested.workos_user_id + LEFT JOIN LATERAL ( + SELECT COUNT(*) AS primary_count, + MIN(candidate.workos_user_id) AS workos_user_id + FROM identity_workos_users candidate + WHERE candidate.identity_id = iwu.identity_id + AND candidate.is_primary = TRUE + ) primary_binding ON TRUE + LEFT JOIN authorization_epochs ae + ON ae.workos_user_id = requested.workos_user_id`, + [workosUserId], + ); + const row = result.rows[0]; + if (!row || row.terminal_marker) { + return { status: 'terminal', reason: 'deleted_or_quarantined' }; + } + if (!row.user_exists) return { status: 'terminal', reason: 'missing_user' }; + if (!row.binding_exists) return { status: 'terminal', reason: 'missing_binding' }; + if (Number(row.primary_count) !== 1 || !row.primary_workos_user_id) { + return { status: 'terminal', reason: 'missing_primary' }; + } + return { + status: 'active', + snapshot: { + workos_user_id: row.workos_user_id, + email: row.email, + first_name: row.first_name, + last_name: row.last_name, + identity_id: row.identity_id, + primary_workos_user_id: row.primary_workos_user_id, + fingerprint: row.fingerprint, + }, + }; + } catch { + return { status: 'unavailable' }; + } +} diff --git a/server/src/db/identity-db.ts b/server/src/db/identity-db.ts index 33dfb41c8be..32c5fafb3a9 100644 --- a/server/src/db/identity-db.ts +++ b/server/src/db/identity-db.ts @@ -11,6 +11,7 @@ */ import { getPool } from './client.js'; +import type { PoolClient } from 'pg'; import { assertIdentityConsolidationAllowed } from './identity-mutation-policy.js'; import { bumpAuthorizationEpochs } from './authorization-epoch-db.js'; import { createLogger } from '../logger.js'; @@ -19,6 +20,13 @@ import { notifySystemError } from '../addie/error-notifier.js'; const logger = createLogger('identity-db'); export const IDENTITY_RECOVERY_STATE = 'manual_primary_selection_required' as const; +export const IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION = 10_000; +export const IDENTITY_DELETION_SNAPSHOT_MAX_DETAILS_BYTES = 4 * 1024 * 1024; +export const IDENTITY_DELETION_SNAPSHOT_POLICY = { + completeness: 'complete_or_fail_closed', + max_rows_per_collection: IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION, + max_details_bytes: IDENTITY_DELETION_SNAPSHOT_MAX_DETAILS_BYTES, +} as const; export type IdentityCredentialDeletionSource = 'workos_webhook' | 'sync_users_backfill'; @@ -29,8 +37,27 @@ export interface IdentityBeforeGraph { organization_memberships: Record<string, unknown>[]; working_group_memberships: Record<string, unknown>[]; working_group_leaders: Record<string, unknown>[]; + working_group_topic_subscriptions: Record<string, unknown>[]; slack_user_mappings: Record<string, unknown>[]; authorization_epochs: Record<string, unknown>[]; + /** Every current ON DELETE CASCADE child of users, keyed by table name. */ + user_cascade_rows: Record<string, Record<string, unknown>[]>; + /** Attribution rows whose user FK is cleared with ON DELETE SET NULL. */ + user_set_null_attributions: Record<string, Record<string, unknown>[]>; +} + +export interface AAOAdminParallelRevocationEvidence { + event_type: 'revoked'; + target_user_id: string; + working_group_id: string; + mechanism: 'aao_admin_working_group'; + actor: { + type: 'workos_provider'; + source: IdentityCredentialDeletionSource; + workos_user_id: string; + }; + evidence_ledger: 'registry_audit_log'; + dedicated_ledger_limitation: 'provider_deletion_actor_not_representable'; } export interface IdentityRecoveryQuarantine { @@ -44,7 +71,9 @@ export interface IdentityRecoveryQuarantine { workos_user_id: string; }; recovery_state: typeof IDENTITY_RECOVERY_STATE; + snapshot_policy: typeof IDENTITY_DELETION_SNAPSHOT_POLICY; before_graph: IdentityBeforeGraph; + aao_admin_revocations: AAOAdminParallelRevocationEvidence[]; } export interface IdentityCredentialDeletionResult { @@ -52,6 +81,12 @@ export interface IdentityCredentialDeletionResult { affectedUserIds: string[]; affectedSlackUserIds: string[]; quarantine: IdentityRecoveryQuarantine | null; + replay: { + audit_id: string; + kind: 'same_source' | 'different_source'; + original_deletion_source: IdentityCredentialDeletionSource; + requested_deletion_source: IdentityCredentialDeletionSource; + } | null; } export interface WorkosUserUpsert { @@ -68,15 +103,549 @@ interface JsonRow { row: Record<string, unknown>; } +/** + * Current users-row CASCADE inventory. Keep this explicit and test it against + * pg_constraint so a future authority/provenance table cannot disappear + * without becoming part of the durable deletion snapshot. + */ +export const USER_CASCADE_SNAPSHOT_INVENTORY = [ + { table: 'authorization_epochs', columns: ['workos_user_id'] }, + { table: 'certification_attempts', columns: ['workos_user_id'] }, + { table: 'certification_learner_feedback', columns: ['workos_user_id'] }, + { table: 'community_points', columns: ['workos_user_id'] }, + { table: 'connections', columns: ['requester_user_id', 'recipient_user_id'] }, + { + table: 'email_link_tokens', + columns: ['primary_workos_user_id'], + redactColumns: ['token'], + }, + { table: 'identity_workos_users', columns: ['workos_user_id'] }, + { table: 'learner_progress', columns: ['workos_user_id'] }, + { + table: 'member_portraits', + columns: ['user_id'], + redactBinaryColumns: ['portrait_data'], + }, + { table: 'organization_credential_grants', columns: ['workos_user_id'] }, + { table: 'teaching_checkpoints', columns: ['workos_user_id'] }, + { + table: 'user_avatar_uploads', + columns: ['workos_user_id'], + redactBinaryColumns: ['image_data'], + }, + { table: 'user_badges', columns: ['workos_user_id'] }, + { table: 'user_credentials', columns: ['workos_user_id'] }, + { table: 'user_email_aliases', columns: ['workos_user_id'] }, +] as const; + +export const USER_SET_NULL_SNAPSHOT_INVENTORY = [ + { table: 'flagged_conversations', columns: ['reviewed_by'] }, + { table: 'known_media_contacts', columns: ['added_by'] }, +] as const; + +type UserSnapshotSpec = { + readonly table: string; + readonly columns: readonly string[]; + /** Preserve row metadata and payload length without retaining raw binary PII. */ + readonly redactBinaryColumns?: readonly string[]; + /** Preserve row provenance without retaining bearer-like secret values. */ + readonly redactColumns?: readonly string[]; +}; + +function assertSnapshotCollectionBound( + collection: string, + rows: readonly unknown[], +): void { + if (rows.length > IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION) { + throw new Error( + `Identity deletion snapshot ${collection} exceeds the fail-closed row limit`, + ); + } +} + +function quoteSnapshotIdentifier(identifier: string): string { + if (!/^[a-z_][a-z0-9_]*$/.test(identifier)) { + throw new Error(`Unsafe deletion snapshot identifier: ${identifier}`); + } + return `"${identifier}"`; +} + +async function snapshotAndLockUserChildren( + client: PoolClient, + specs: readonly UserSnapshotSpec[], + affectedUserIds: string[], +): Promise<Record<string, Record<string, unknown>[]>> { + const snapshot: Record<string, Record<string, unknown>[]> = {}; + for (const spec of specs) { + const table = quoteSnapshotIdentifier(spec.table); + const binaryColumns = spec.redactBinaryColumns ?? []; + const binaryRedactedExpression = binaryColumns.reduce((expression, column) => { + const quotedColumn = quoteSnapshotIdentifier(column); + return `(${expression} - '${column}') || jsonb_build_object( + '${column}_redacted', child.${quotedColumn} IS NOT NULL, + '${column}_byte_length', octet_length(child.${quotedColumn}) + )`; + }, 'to_jsonb(child)'); + const jsonExpression = (spec.redactColumns ?? []).reduce((expression, column) => { + const quotedColumn = quoteSnapshotIdentifier(column); + return `(${expression} - '${column}') || jsonb_build_object( + '${column}_redacted', child.${quotedColumn} IS NOT NULL + )`; + }, binaryRedactedExpression); + const predicates = spec.columns + .map((column) => `child.${quoteSnapshotIdentifier(column)} = ANY($1)`) + .join(' OR '); + const result = await client.query<JsonRow>( + `SELECT ${jsonExpression} AS row + FROM ${table} child + WHERE ${predicates} + ORDER BY to_jsonb(child)::text + LIMIT $2 + FOR UPDATE OF child`, + [affectedUserIds, IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION + 1], + ); + assertSnapshotCollectionBound(spec.table, result.rows); + snapshot[spec.table] = rowsAsJson(result); + } + return snapshot; +} + +export const CREDENTIAL_MUTATION_LOCK_TIMEOUT_MS = 1_000; +export const CREDENTIAL_MUTATION_MAX_ATTEMPTS = 3; +export const CREDENTIAL_MUTATION_ADMISSION_TIMEOUT_MS = 1_000; + +/** + * Keep one connection free for confirmed deletion and bound admission wait. + * Guard callbacks must use their supplied client and must not perform provider + * I/O, so each admitted mutation consumes exactly one pool connection. + */ +const credentialMutationAdmission = { + active: 0, + waiters: [] as Array<(release: () => void) => void>, +}; + +async function acquireCredentialMutationPoolCapacity(): Promise<() => void> { + const configuredMax = getPool().options.max; + if (configuredMax < 4) { + throw new Error('Credential event mutation requires a database pool with at least four connections'); + } + const limit = Math.max(1, configuredMax - 1); + + const makeRelease = (): (() => void) => { + let released = false; + return () => { + if (released) return; + released = true; + const next = credentialMutationAdmission.waiters.shift(); + if (next) { + // Transfer this occupied admission directly to the oldest waiter so + // a newly arriving mutation cannot steal capacity between turns. + next(makeRelease()); + } else { + credentialMutationAdmission.active--; + } + }; + }; + + if (credentialMutationAdmission.active < limit) { + credentialMutationAdmission.active++; + return makeRelease(); + } + return new Promise((resolve, reject) => { + const waiter = (release: () => void) => { + clearTimeout(timeout); + resolve(release); + }; + const timeout = setTimeout(() => { + const index = credentialMutationAdmission.waiters.indexOf(waiter); + if (index >= 0) credentialMutationAdmission.waiters.splice(index, 1); + reject(new Error('Credential event mutation admission timed out')); + }, CREDENTIAL_MUTATION_ADMISSION_TIMEOUT_MS); + credentialMutationAdmission.waiters.push(waiter); + }); +} + function rowsAsJson(result: { rows: JsonRow[] }): Record<string, unknown>[] { return result.rows.map(({ row }) => row); } -async function lockCredentialMutation( +async function lockCredentialMutations( client: { query: (text: string, params?: unknown[]) => Promise<unknown> }, + workosUserIds: string[], +): Promise<void> { + for (const workosUserId of [...new Set(workosUserIds)].sort()) { + await client.query(`SELECT pg_advisory_xact_lock(hashtextextended($1, 6827))`, [workosUserId]); + } +} + +async function lockCredentialThenIdentityMutation( + client: PoolClient, + workosUserId: string, +): Promise<{ identityId: string | undefined; identity: { rows: JsonRow[] } }> { + await lockCredentialMutations(client, [workosUserId]); + const identityLookup = await client.query<{ identity_id: string }>( + `SELECT identity_id FROM identity_workos_users + WHERE workos_user_id = $1 + FOR UPDATE`, + [workosUserId], + ); + const identityId = identityLookup.rows[0]?.identity_id; + const identity = identityId + ? await client.query<JsonRow>( + `SELECT to_jsonb(i) AS row FROM identities i WHERE id = $1 FOR UPDATE OF i`, + [identityId], + ) + : { rows: [] as JsonRow[] }; + return { identityId, identity }; +} + +async function hasConfirmedDeletionTombstone( + client: PoolClient, + workosUserId: string, +): Promise<boolean> { + const tombstone = await client.query( + `SELECT 1 FROM registry_audit_log + WHERE workos_user_id = $1 + AND action IN ('identity_credential_deleted', 'identity_primary_deletion_quarantined') + LIMIT 1`, + [workosUserId], + ); + return tombstone.rowCount !== null && tombstone.rowCount > 0; +} + +type ConfirmedDeletionAuditAction = + | 'identity_credential_deleted' + | 'identity_primary_deletion_quarantined'; + +async function getConfirmedDeletionReplay( + client: PoolClient, + workosUserId: string, + requestedSource: IdentityCredentialDeletionSource, +): Promise<IdentityCredentialDeletionResult | null> { + const audits = await client.query<{ + id: string; + action: ConfirmedDeletionAuditAction; + details: Record<string, unknown>; + }>( + `SELECT id, action, details + FROM registry_audit_log + WHERE workos_user_id = $1 + AND action IN ('identity_credential_deleted', 'identity_primary_deletion_quarantined') + ORDER BY created_at ASC, id ASC`, + [workosUserId], + ); + if (audits.rows.length === 0) return null; + + const prior = audits.rows[0]; + const details = prior.details; + const originalSource = details?.deletion_source; + const beforeGraph = details?.before_graph; + if ((originalSource !== 'workos_webhook' && originalSource !== 'sync_users_backfill') + || details?.deleted_workos_user_id !== workosUserId + || typeof beforeGraph !== 'object' + || beforeGraph === null + || Array.isArray(beforeGraph)) { + throw new Error('Confirmed credential deletion replay audit is malformed'); + } + + const graph = beforeGraph as unknown as IdentityBeforeGraph; + if (!Array.isArray(graph.identity_workos_users) + || !Array.isArray(graph.slack_user_mappings)) { + throw new Error('Confirmed credential deletion replay before graph is malformed'); + } + const affectedUserIds = [...new Set([ + workosUserId, + ...graph.identity_workos_users.map((row) => String(row.workos_user_id)), + ])].sort(); + const affectedSlackUserIds = graph.slack_user_mappings + .filter((row) => row.workos_user_id === workosUserId) + .map((row) => String(row.slack_user_id)); + + let quarantine: IdentityRecoveryQuarantine | null = null; + if (prior.action === 'identity_primary_deletion_quarantined') { + if (details.recovery_state !== IDENTITY_RECOVERY_STATE + || typeof details.identity_id !== 'string' + || !Array.isArray(details.aao_admin_revocations)) { + throw new Error('Confirmed credential deletion replay quarantine is malformed'); + } + quarantine = { + audit_id: prior.id, + identity_id: details.identity_id, + deleted_workos_user_id: workosUserId, + deletion_source: originalSource, + actor: details.actor as IdentityRecoveryQuarantine['actor'], + recovery_state: IDENTITY_RECOVERY_STATE, + snapshot_policy: IDENTITY_DELETION_SNAPSHOT_POLICY, + before_graph: graph, + aao_admin_revocations: + details.aao_admin_revocations as AAOAdminParallelRevocationEvidence[], + }; + } + + return { + // `deleted` means this invocation changed the live row. Replays return the + // original durable outcome below without incrementing caller counters. + deleted: false, + affectedUserIds, + affectedSlackUserIds, + quarantine, + replay: { + audit_id: prior.id, + kind: originalSource === requestedSource ? 'same_source' : 'different_source', + original_deletion_source: originalSource, + requested_deletion_source: requestedSource, + }, + }; +} + +function isRetryableCredentialLockError(error: unknown): boolean { + const code = typeof error === 'object' && error !== null && 'code' in error + ? String((error as { code?: unknown }).code) + : ''; + return code === '55P03' || code === '40P01'; +} + +export interface CredentialEventMutationResult<T> { + applied: boolean; + value?: T; +} + +type CredentialMutationMode = 'active' | 'provider_upsert'; + +/** + * Run every authority-bearing side effect of one provider event while the + * same transaction-scoped credential lock used by deletion remains held. + * Tombstones are checked after locking. Deletion uses the same lock through + * its tombstone commit, so the marker cannot appear before this guard releases. + * Lock timeout/deadlock failures retry a bounded number of times, then throw + * so the signed webhook returns 500 and WorkOS can retry; no unlocked fallback + * is permitted. Once the callback starts it is never retried here: every + * local write must use the supplied client and a callback failure rolls the + * transaction back, but replaying arbitrary callback code is still unsafe. + */ +async function withCredentialEventMutation<T>( + workosUserId: string, + mode: CredentialMutationMode, + mutation: (client: PoolClient) => Promise<T>, +): Promise<CredentialEventMutationResult<T>> { + const releaseCapacity = await acquireCredentialMutationPoolCapacity(); + try { + for (let attempt = 1; attempt <= CREDENTIAL_MUTATION_MAX_ATTEMPTS; attempt++) { + const client = await getPool().connect(); + let mutationStarted = false; + try { + await client.query('BEGIN'); + await client.query(`SET LOCAL lock_timeout = '${CREDENTIAL_MUTATION_LOCK_TIMEOUT_MS}ms'`); + const { identityId } = await lockCredentialThenIdentityMutation(client, workosUserId); + if (await hasConfirmedDeletionTombstone(client, workosUserId)) { + await client.query('COMMIT'); + return { applied: false }; + } + const lifecycle = await client.query<{ + user_exists: boolean; + binding_exists: boolean; + primary_count: string; + }>( + `SELECT EXISTS ( + SELECT 1 FROM users WHERE workos_user_id = $1 + ) AS user_exists, + EXISTS ( + SELECT 1 FROM identity_workos_users WHERE workos_user_id = $1 + ) AS binding_exists, + COALESCE(( + SELECT COUNT(*) + FROM identity_workos_users primary_iwu + WHERE primary_iwu.identity_id = $2 + AND primary_iwu.is_primary = TRUE + ), 0)::text AS primary_count`, + [workosUserId, identityId ?? null], + ); + const state = lifecycle.rows[0]; + const isActive = state?.user_exists === true + && state.binding_exists === true + && Number(state.primary_count) === 1; + const isUnseen = state?.user_exists === false && state.binding_exists === false; + if ((mode === 'active' && !isActive) + || (mode === 'provider_upsert' && !isActive && !isUnseen)) { + await client.query('COMMIT'); + return { applied: false }; + } + + mutationStarted = true; + const value = await mutation(client); + const bumped = await bumpAuthorizationEpochs(client, [workosUserId]); + if (bumped.length !== 1 || bumped[0] !== workosUserId) { + throw new Error('Credential event mutation did not bump the live authorization epoch'); + } + await client.query('COMMIT'); + return { applied: true, value }; + } catch (error) { + await client.query('ROLLBACK').catch(() => undefined); + if (!mutationStarted + && attempt < CREDENTIAL_MUTATION_MAX_ATTEMPTS + && isRetryableCredentialLockError(error)) { + await new Promise((resolve) => setTimeout(resolve, attempt * 25)); + continue; + } + throw error; + } finally { + client.release(); + } + } + throw new Error('Credential event mutation retry bound exhausted'); + } finally { + releaseCapacity(); + } +} + +export async function withActiveCredentialEventMutation<T>( + workosUserId: string, + mutation: (client: PoolClient) => Promise<T>, +): Promise<CredentialEventMutationResult<T>> { + return withCredentialEventMutation(workosUserId, 'active', mutation); +} + +/** Apply one atomic authority mutation to a sorted set of live credentials. */ +export async function withActiveCredentialSetMutation<T>( + workosUserIds: string[], + mutation: (client: PoolClient) => Promise<T>, +): Promise<CredentialEventMutationResult<T>> { + const ids = [...new Set(workosUserIds.filter(Boolean))].sort(); + if (ids.length === 0) return { applied: false }; + const releaseCapacity = await acquireCredentialMutationPoolCapacity(); + try { + for (let attempt = 1; attempt <= CREDENTIAL_MUTATION_MAX_ATTEMPTS; attempt++) { + const client = await getPool().connect(); + let mutationStarted = false; + try { + await client.query('BEGIN'); + await client.query(`SET LOCAL lock_timeout = '${CREDENTIAL_MUTATION_LOCK_TIMEOUT_MS}ms'`); + await lockCredentialMutations(client, ids); + const bindings = await client.query<{ workos_user_id: string; identity_id: string }>( + `SELECT workos_user_id, identity_id + FROM identity_workos_users + WHERE workos_user_id = ANY($1) + ORDER BY workos_user_id + FOR UPDATE`, + [ids], + ); + const identityIds = [...new Set(bindings.rows.map((row) => row.identity_id))].sort(); + if (identityIds.length > 0) { + await client.query( + `SELECT id FROM identities + WHERE id = ANY($1) + ORDER BY id + FOR UPDATE`, + [identityIds], + ); + } + const lifecycle = await client.query<{ + workos_user_id: string; + user_exists: boolean; + binding_exists: boolean; + primary_count: string; + terminal_marker: boolean; + }>( + `SELECT requested.workos_user_id, + (u.workos_user_id IS NOT NULL) AS user_exists, + (iwu.workos_user_id IS NOT NULL) AS binding_exists, + COALESCE(( + SELECT COUNT(*) FROM identity_workos_users primary_iwu + WHERE primary_iwu.identity_id = iwu.identity_id + AND primary_iwu.is_primary = TRUE + ), 0)::text AS primary_count, + EXISTS ( + SELECT 1 FROM registry_audit_log ral + WHERE ral.workos_user_id = requested.workos_user_id + AND ral.action IN ( + 'identity_credential_deleted', + 'identity_primary_deletion_quarantined' + ) + ) AS terminal_marker + FROM unnest($1::text[]) AS requested(workos_user_id) + LEFT JOIN users u USING (workos_user_id) + LEFT JOIN identity_workos_users iwu USING (workos_user_id) + ORDER BY requested.workos_user_id`, + [ids], + ); + if (lifecycle.rows.length !== ids.length + || lifecycle.rows.some((row) => !row.user_exists + || !row.binding_exists + || row.terminal_marker + || Number(row.primary_count) !== 1)) { + await client.query('COMMIT'); + return { applied: false }; + } + + mutationStarted = true; + const value = await mutation(client); + const bumped = (await bumpAuthorizationEpochs(client, ids)).sort(); + if (bumped.length !== ids.length || bumped.some((id, index) => id !== ids[index])) { + throw new Error('Authority mutation did not bump every live credential epoch'); + } + await client.query('COMMIT'); + return { applied: true, value }; + } catch (error) { + await client.query('ROLLBACK').catch(() => undefined); + if (!mutationStarted + && attempt < CREDENTIAL_MUTATION_MAX_ATTEMPTS + && isRetryableCredentialLockError(error)) { + await new Promise((resolve) => setTimeout(resolve, attempt * 25)); + continue; + } + throw error; + } finally { + client.release(); + } + } + throw new Error('Credential authority mutation retry bound exhausted'); + } finally { + releaseCapacity(); + } +} + +export async function withCredentialCreationEventMutation<T>( workosUserId: string, + mutation: (client: PoolClient) => Promise<T>, +): Promise<CredentialEventMutationResult<T>> { + return withCredentialEventMutation(workosUserId, 'provider_upsert', mutation); +} + +export async function upsertWorkosUserInCredentialEvent( + credentialLockClient: PoolClient, + user: WorkosUserUpsert, + nameConflictPolicy: 'provider_authoritative' | 'preserve_existing', ): Promise<void> { - await client.query(`SELECT pg_advisory_xact_lock(hashtextextended($1, 6827))`, [workosUserId]); + await credentialLockClient.query( + `INSERT INTO users ( + workos_user_id, email, first_name, last_name, + email_verified, workos_created_at, workos_updated_at, + created_at, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) + ON CONFLICT (workos_user_id) DO UPDATE SET + email = EXCLUDED.email, + first_name = CASE WHEN $8::text = 'preserve_existing' + THEN COALESCE(NULLIF(TRIM(users.first_name), ''), EXCLUDED.first_name) + ELSE COALESCE(NULLIF(TRIM(EXCLUDED.first_name), ''), users.first_name) + END, + last_name = CASE WHEN $8::text = 'preserve_existing' + THEN COALESCE(NULLIF(TRIM(users.last_name), ''), EXCLUDED.last_name) + ELSE COALESCE(NULLIF(TRIM(EXCLUDED.last_name), ''), users.last_name) + END, + email_verified = EXCLUDED.email_verified, + workos_updated_at = EXCLUDED.workos_updated_at, + updated_at = NOW()`, + [ + user.id, + user.email, + user.firstName, + user.lastName, + user.emailVerified, + user.createdAt, + user.updatedAt, + nameConflictPolicy, + ], + ); } /** @@ -86,44 +655,10 @@ async function lockCredentialMutation( export async function upsertWorkosUserUnlessConfirmedDeleted( user: WorkosUserUpsert, ): Promise<boolean> { - const client = await getPool().connect(); - try { - await client.query('BEGIN'); - await lockCredentialMutation(client, user.id); - const tombstone = await client.query( - `SELECT 1 FROM registry_audit_log - WHERE workos_user_id = $1 - AND action IN ('identity_credential_deleted', 'identity_primary_deletion_quarantined') - LIMIT 1`, - [user.id], - ); - if (tombstone.rowCount) { - await client.query('COMMIT'); - return false; - } - await client.query( - `INSERT INTO users ( - workos_user_id, email, first_name, last_name, - email_verified, workos_created_at, workos_updated_at, - created_at, updated_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) - ON CONFLICT (workos_user_id) DO UPDATE SET - email = EXCLUDED.email, - first_name = COALESCE(NULLIF(TRIM(EXCLUDED.first_name), ''), users.first_name), - last_name = COALESCE(NULLIF(TRIM(EXCLUDED.last_name), ''), users.last_name), - email_verified = EXCLUDED.email_verified, - workos_updated_at = EXCLUDED.workos_updated_at, - updated_at = NOW()`, - [user.id, user.email, user.firstName, user.lastName, user.emailVerified, user.createdAt, user.updatedAt], - ); - await client.query('COMMIT'); - return true; - } catch (err) { - await client.query('ROLLBACK').catch(() => undefined); - throw err; - } finally { - client.release(); - } + const result = await withCredentialCreationEventMutation(user.id, async (client) => { + await upsertWorkosUserInCredentialEvent(client, user, 'provider_authoritative'); + }); + return result.applied; } /** Retrieve the durable, unresolved recovery state for an identity. */ @@ -163,34 +698,63 @@ export async function deleteIdentityCredentialTransaction( try { await client.query('BEGIN'); - const identityLookup = await client.query<{ identity_id: string }>( + await client.query(`SET LOCAL lock_timeout = '${CREDENTIAL_MUTATION_LOCK_TIMEOUT_MS}ms'`); + const discoveredBinding = await client.query<{ identity_id: string }>( `SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1`, [workosUserId], ); - const identityId = identityLookup.rows[0]?.identity_id; - - // The identity row is the serialization point. Lock it before child rows - // so webhook/backfill deletion of siblings cannot deadlock by first - // holding different credential rows. + const discoveredIdentityId = discoveredBinding.rows[0]?.identity_id; + const discoveredSiblings = discoveredIdentityId + ? await client.query<{ workos_user_id: string }>( + `SELECT workos_user_id FROM identity_workos_users + WHERE identity_id = $1 + ORDER BY workos_user_id + LIMIT $2`, + [discoveredIdentityId, IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION + 1], + ) + : { rows: [] as Array<{ workos_user_id: string }> }; + assertSnapshotCollectionBound('discovered_identity_workos_users', discoveredSiblings.rows); + const discoveredUserIds = [...new Set([ + workosUserId, + ...discoveredSiblings.rows.map((row) => row.workos_user_id), + ])].sort(); + await lockCredentialMutations(client, discoveredUserIds); + const lockedBinding = await client.query<{ identity_id: string }>( + `SELECT identity_id FROM identity_workos_users + WHERE workos_user_id = $1 + FOR UPDATE`, + [workosUserId], + ); + const identityId = lockedBinding.rows[0]?.identity_id; + const replay = await getConfirmedDeletionReplay(client, workosUserId, deletionSource); + if (replay) { + if (identityId) { + throw new Error('Confirmed credential deletion audit conflicts with a live identity binding'); + } + await client.query('COMMIT'); + return replay; + } + if (identityId !== discoveredIdentityId) { + throw new Error('Credential identity changed while acquiring deletion locks'); + } const identity = identityId ? await client.query<JsonRow>( `SELECT to_jsonb(i) AS row FROM identities i WHERE id = $1 FOR UPDATE OF i`, [identityId], ) : { rows: [] as JsonRow[] }; - // Serialize this credential with WorkOS create/update/backfill upserts. - // Identity comes first whenever it exists, preserving the shared lock order. - await lockCredentialMutation(client, workosUserId); const bound = identityId ? await client.query<JsonRow>( `SELECT to_jsonb(iwu) AS row FROM identity_workos_users iwu WHERE iwu.identity_id = $1 ORDER BY iwu.workos_user_id + LIMIT $2 FOR UPDATE OF iwu`, - [identityId], + [identityId, IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION + 1], ) : { rows: [] as JsonRow[] }; + assertSnapshotCollectionBound('identity_workos_users', bound.rows); const bindingRows = rowsAsJson(bound); const targetBinding = bindingRows.find((row) => row.workos_user_id === workosUserId); if (identityId && !targetBinding) { @@ -202,12 +766,17 @@ export async function deleteIdentityCredentialTransaction( affectedUserIds: [workosUserId], affectedSlackUserIds: [], quarantine: null, + replay: null, }; } const affectedUserIds = [...new Set([ workosUserId, ...bindingRows.map((row) => String(row.workos_user_id)), ])].sort(); + if (affectedUserIds.length !== discoveredUserIds.length + || affectedUserIds.some((id, index) => id !== discoveredUserIds[index])) { + throw new Error('Credential sibling graph changed while acquiring deletion locks'); + } // Lock and capture the complete authority-bearing before graph in stable // orders. The graph becomes the durable recovery evidence if this is the @@ -216,58 +785,107 @@ export async function deleteIdentityCredentialTransaction( `SELECT to_jsonb(u) AS row FROM users u WHERE u.workos_user_id = ANY($1) ORDER BY u.workos_user_id + LIMIT $2 FOR UPDATE OF u`, - [affectedUserIds], + [affectedUserIds, IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION + 1], ); + assertSnapshotCollectionBound('users', users.rows); const organizationMemberships = await client.query<JsonRow>( `SELECT to_jsonb(om) AS row FROM organization_memberships om WHERE om.workos_user_id = ANY($1) ORDER BY om.workos_user_id, om.workos_organization_id + LIMIT $2 FOR UPDATE OF om`, - [affectedUserIds], + [affectedUserIds, IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION + 1], ); + assertSnapshotCollectionBound('organization_memberships', organizationMemberships.rows); + const slackUserMappings = await client.query<JsonRow>( + `SELECT to_jsonb(sm) AS row FROM slack_user_mappings sm + WHERE sm.workos_user_id = ANY($1) + ORDER BY sm.workos_user_id, sm.slack_user_id + LIMIT $2 + FOR UPDATE OF sm`, + [affectedUserIds, IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION + 1], + ); + assertSnapshotCollectionBound('slack_user_mappings', slackUserMappings.rows); + const targetSlackUserIds = rowsAsJson(slackUserMappings) + .filter((row) => row.workos_user_id === workosUserId) + .map((row) => String(row.slack_user_id)); + const targetAuthorityPrincipalIds = [...new Set([ + workosUserId, + ...targetSlackUserIds, + ])].sort(); + const snapshotAuthorityPrincipalIds = [...new Set([ + ...affectedUserIds, + ...targetSlackUserIds, + ])].sort(); const workingGroupMemberships = await client.query<JsonRow>( `SELECT to_jsonb(wgm) AS row FROM working_group_memberships wgm WHERE wgm.workos_user_id = ANY($1) ORDER BY wgm.workos_user_id, wgm.working_group_id + LIMIT $2 FOR UPDATE OF wgm`, - [affectedUserIds], + [ + snapshotAuthorityPrincipalIds, + IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION + 1, + ], ); + assertSnapshotCollectionBound('working_group_memberships', workingGroupMemberships.rows); const workingGroupLeaders = await client.query<JsonRow>( `SELECT to_jsonb(wgl) AS row FROM working_group_leaders wgl WHERE wgl.user_id = ANY($1) ORDER BY wgl.user_id, wgl.working_group_id + LIMIT $2 FOR UPDATE OF wgl`, - [affectedUserIds], + [ + snapshotAuthorityPrincipalIds, + IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION + 1, + ], ); - const slackUserMappings = await client.query<JsonRow>( - `SELECT to_jsonb(sm) AS row FROM slack_user_mappings sm - WHERE sm.workos_user_id = ANY($1) - ORDER BY sm.workos_user_id, sm.slack_user_id - FOR UPDATE OF sm`, - [affectedUserIds], + assertSnapshotCollectionBound('working_group_leaders', workingGroupLeaders.rows); + const workingGroupTopicSubscriptions = await client.query<JsonRow>( + `SELECT to_jsonb(wgts) AS row FROM working_group_topic_subscriptions wgts + WHERE wgts.workos_user_id = ANY($1) + ORDER BY wgts.workos_user_id, wgts.working_group_id + LIMIT $2 + FOR UPDATE OF wgts`, + [ + snapshotAuthorityPrincipalIds, + IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION + 1, + ], + ); + assertSnapshotCollectionBound( + 'working_group_topic_subscriptions', + workingGroupTopicSubscriptions.rows, ); const authorizationEpochs = await client.query<JsonRow>( `SELECT to_jsonb(ae) AS row FROM authorization_epochs ae WHERE ae.workos_user_id = ANY($1) ORDER BY ae.workos_user_id + LIMIT $2 FOR UPDATE OF ae`, - [affectedUserIds], + [affectedUserIds, IDENTITY_DELETION_SNAPSHOT_MAX_ROWS_PER_COLLECTION + 1], + ); + assertSnapshotCollectionBound('authorization_epochs', authorizationEpochs.rows); + const userCascadeRows = await snapshotAndLockUserChildren( + client, + USER_CASCADE_SNAPSHOT_INVENTORY.filter( + ({ table }) => table !== 'identity_workos_users' && table !== 'authorization_epochs', + ), + affectedUserIds, + ); + userCascadeRows.identity_workos_users = bindingRows; + userCascadeRows.authorization_epochs = rowsAsJson(authorizationEpochs); + const userSetNullAttributions = await snapshotAndLockUserChildren( + client, + USER_SET_NULL_SNAPSHOT_INVENTORY, + affectedUserIds, ); const userRows = rowsAsJson(users); const targetUser = userRows.find((row) => row.workos_user_id === workosUserId); - if (!targetUser) { - await client.query('COMMIT'); - return { - deleted: false, - affectedUserIds: [workosUserId], - affectedSlackUserIds: [], - quarantine: null, - }; - } const survivingBindings = bindingRows.filter((row) => row.workos_user_id !== workosUserId); - const needsQuarantine = targetBinding?.is_primary === true + const needsQuarantine = identityId !== undefined && survivingBindings.length > 0 && survivingBindings.every((row) => row.is_primary !== true); let quarantine: IdentityRecoveryQuarantine | null = null; @@ -278,14 +896,17 @@ export async function deleteIdentityCredentialTransaction( organization_memberships: rowsAsJson(organizationMemberships), working_group_memberships: rowsAsJson(workingGroupMemberships), working_group_leaders: rowsAsJson(workingGroupLeaders), + working_group_topic_subscriptions: rowsAsJson(workingGroupTopicSubscriptions), slack_user_mappings: rowsAsJson(slackUserMappings), authorization_epochs: rowsAsJson(authorizationEpochs), + user_cascade_rows: userCascadeRows, + user_set_null_attributions: userSetNullAttributions, }; const targetMembership = beforeGraph.organization_memberships.find( (row) => row.workos_user_id === workosUserId, ); const auditOrganizationId = String( - targetUser.primary_organization_id + targetUser?.primary_organization_id ?? targetMembership?.workos_organization_id ?? 'identity-recovery-unscoped', ); @@ -294,12 +915,37 @@ export async function deleteIdentityCredentialTransaction( source: deletionSource, workos_user_id: workosUserId, }; + const aaoAdminGroup = await client.query<{ id: string }>( + `SELECT id FROM working_groups WHERE slug = 'aao-admin' LIMIT 1`, + ); + const aaoAdminGroupId = aaoAdminGroup.rows[0]?.id; + // Migration 578 cannot truthfully encode a provider-deletion actor in + // actor_authorization_mechanism. Record compatible immutable revocation + // evidence in the generic deletion audit instead of forging one of its + // human/admin mechanisms; the full membership row is in before_graph. + const aaoAdminRevocations: AAOAdminParallelRevocationEvidence[] = aaoAdminGroupId + ? beforeGraph.working_group_memberships + .filter((row) => targetAuthorityPrincipalIds.includes(String(row.workos_user_id)) + && row.working_group_id === aaoAdminGroupId + && row.status === 'active') + .map((row) => ({ + event_type: 'revoked' as const, + target_user_id: String(row.workos_user_id), + working_group_id: aaoAdminGroupId, + mechanism: 'aao_admin_working_group' as const, + actor, + evidence_ledger: 'registry_audit_log' as const, + dedicated_ledger_limitation: 'provider_deletion_actor_not_representable' as const, + })) + : []; const baseDetails = { identity_id: identityId ?? null, deleted_workos_user_id: workosUserId, deletion_source: deletionSource, actor, + snapshot_policy: IDENTITY_DELETION_SNAPSHOT_POLICY, before_graph: beforeGraph, + aao_admin_revocations: aaoAdminRevocations, }; const details = needsQuarantine && identityId ? { ...baseDetails, identity_id: identityId, recovery_state: IDENTITY_RECOVERY_STATE } @@ -309,13 +955,18 @@ export async function deleteIdentityCredentialTransaction( : 'identity_credential_deleted'; const auditResourceType = needsQuarantine ? 'identity_recovery' : 'identity_credential'; const auditResourceId = identityId ?? workosUserId; + const serializedDetails = JSON.stringify(details); + if (Buffer.byteLength(serializedDetails, 'utf8') + > IDENTITY_DELETION_SNAPSHOT_MAX_DETAILS_BYTES) { + throw new Error('Identity deletion snapshot exceeds the fail-closed byte limit'); + } const audit = await client.query<{ id: string }>( `INSERT INTO registry_audit_log ( workos_organization_id, workos_user_id, action, resource_type, resource_id, details ) VALUES ($1, $2, $3, $4, $5, $6::jsonb) RETURNING id`, - [auditOrganizationId, workosUserId, auditAction, auditResourceType, auditResourceId, JSON.stringify(details)], + [auditOrganizationId, workosUserId, auditAction, auditResourceType, auditResourceId, serializedDetails], ); if (audit.rowCount !== 1 || audit.rows.length !== 1 || !audit.rows[0]?.id) { throw new Error('Confirmed identity credential deletion audit was not durably recorded'); @@ -326,20 +977,68 @@ export async function deleteIdentityCredentialTransaction( // Authentication checks the actual credential's epoch, so deleting the // primary's epoch alone would leave each surviving credential's cache valid. - await bumpAuthorizationEpochs(client, affectedUserIds); - await client.query('DELETE FROM organization_memberships WHERE workos_user_id = $1', [workosUserId]); + const expectedEpochUserIds = userRows + .map((row) => String(row.workos_user_id)) + .sort(); + const bumpedEpochUserIds = (await bumpAuthorizationEpochs(client, expectedEpochUserIds)).sort(); + if (bumpedEpochUserIds.length !== expectedEpochUserIds.length + || bumpedEpochUserIds.some((id, index) => id !== expectedEpochUserIds[index])) { + throw new Error('Confirmed credential deletion did not bump every live authorization epoch'); + } + const deletedOrganizationMemberships = await client.query( + `DELETE FROM organization_memberships WHERE workos_user_id = $1 RETURNING workos_organization_id`, + [workosUserId], + ); + const expectedOrganizationMemberships = beforeGraph.organization_memberships + .filter((row) => row.workos_user_id === workosUserId).length; + if (deletedOrganizationMemberships.rowCount !== expectedOrganizationMemberships) { + throw new Error('Confirmed credential deletion did not revoke every organization membership'); + } // Preserve membership history while revoking live group authority. - await client.query( + const deactivatedWorkingGroupMemberships = await client.query<{ working_group_id: string }>( `UPDATE working_group_memberships SET status = 'inactive', updated_at = NOW() - WHERE workos_user_id = $1 AND status <> 'inactive'`, - [workosUserId], + WHERE workos_user_id = ANY($1) AND status <> 'inactive' + RETURNING working_group_id`, + [targetAuthorityPrincipalIds], ); + const expectedWorkingGroupDeactivations = beforeGraph.working_group_memberships + .filter((row) => targetAuthorityPrincipalIds.includes(String(row.workos_user_id)) + && row.status !== 'inactive').length; + if (deactivatedWorkingGroupMemberships.rowCount !== expectedWorkingGroupDeactivations) { + throw new Error('Confirmed credential deletion did not deactivate every working-group membership'); + } + const deactivatedAAOAdminRows = aaoAdminGroupId + ? deactivatedWorkingGroupMemberships.rows + .filter((row) => row.working_group_id === aaoAdminGroupId).length + : 0; + if (deactivatedAAOAdminRows !== aaoAdminRevocations.length) { + throw new Error('AAO-admin revocation evidence cardinality did not match live deactivation'); + } // Leadership rows have no inactive state; the complete rows are retained // in before_graph before live authority is removed. - await client.query('DELETE FROM working_group_leaders WHERE user_id = $1', [workosUserId]); + const deletedLeaderships = await client.query( + `DELETE FROM working_group_leaders WHERE user_id = ANY($1) RETURNING working_group_id`, + [targetAuthorityPrincipalIds], + ); + const expectedLeaderships = beforeGraph.working_group_leaders + .filter((row) => targetAuthorityPrincipalIds.includes(String(row.user_id))).length; + if (deletedLeaderships.rowCount !== expectedLeaderships) { + throw new Error('Confirmed credential deletion did not revoke every working-group leadership'); + } + const deletedTopicSubscriptions = await client.query( + `DELETE FROM working_group_topic_subscriptions + WHERE workos_user_id = ANY($1) + RETURNING working_group_id`, + [targetAuthorityPrincipalIds], + ); + const expectedTopicSubscriptions = beforeGraph.working_group_topic_subscriptions + .filter((row) => targetAuthorityPrincipalIds.includes(String(row.workos_user_id))).length; + if (deletedTopicSubscriptions.rowCount !== expectedTopicSubscriptions) { + throw new Error('Confirmed credential deletion did not revoke every topic subscription'); + } // Preserve the Slack contact row while severing its WorkOS authority link. - await client.query( + const unlinkedSlackMappings = await client.query( `UPDATE slack_user_mappings SET workos_user_id = NULL, mapping_status = 'unmapped', @@ -347,24 +1046,35 @@ export async function deleteIdentityCredentialTransaction( mapped_at = NULL, mapped_by_user_id = NULL, updated_at = NOW() - WHERE workos_user_id = $1`, - [workosUserId], - ); - const deleted = await client.query<{ workos_user_id: string }>( - 'DELETE FROM users WHERE workos_user_id = $1 RETURNING workos_user_id', + WHERE workos_user_id = $1 + RETURNING slack_user_id`, [workosUserId], ); - if (quarantine && deleted.rowCount !== 1) { - throw new Error('Quarantined primary credential deletion did not delete exactly one user'); + const expectedSlackMappings = beforeGraph.slack_user_mappings + .filter((row) => row.workos_user_id === workosUserId).length; + if (unlinkedSlackMappings.rowCount !== expectedSlackMappings) { + throw new Error('Confirmed credential deletion did not unlink every Slack mapping'); + } + let deletedRow = false; + if (targetUser) { + const deleted = await client.query<{ workos_user_id: string }>( + 'DELETE FROM users WHERE workos_user_id = $1 RETURNING workos_user_id', + [workosUserId], + ); + if (deleted.rowCount !== 1 || deleted.rows.length !== 1) { + throw new Error('Confirmed credential deletion did not delete exactly one locked user'); + } + deletedRow = true; } await client.query('COMMIT'); result = { - deleted: deleted.rowCount === 1, + deleted: deletedRow, affectedUserIds, affectedSlackUserIds: beforeGraph.slack_user_mappings .filter((row) => row.workos_user_id === workosUserId) .map((row) => String(row.slack_user_id)), quarantine, + replay: null, }; } catch (err) { await client.query('ROLLBACK').catch(() => undefined); diff --git a/server/src/db/mcp-oauth-state-db.ts b/server/src/db/mcp-oauth-state-db.ts index affb95a31f5..7dd6177133f 100644 --- a/server/src/db/mcp-oauth-state-db.ts +++ b/server/src/db/mcp-oauth-state-db.ts @@ -9,8 +9,10 @@ import { query, isDatabaseInitialized } from './client.js'; import { encrypt, decrypt } from './encryption.js'; import { createLogger } from '../logger.js'; +import type { Pool, PoolClient } from 'pg'; const logger = createLogger('mcp-oauth-state-db'); +type Queryable = Pick<Pool | PoolClient, 'query'>; // --------------------------------------------------------------------------- @@ -120,11 +122,12 @@ const AUTH_CODE_TTL_MS = 5 * 60 * 1000; // 5 minutes export async function setAuthCode( code: string, data: AuthCodeData, + db?: Queryable, ): Promise<void> { const expiresAt = new Date(Date.now() + AUTH_CODE_TTL_MS); const stored = encryptAuthCodeData(data); try { - await query( + await (db?.query.bind(db) ?? query)( `INSERT INTO mcp_oauth_auth_codes (code, data, expires_at) VALUES ($1, $2, $3)`, [code, JSON.stringify(stored), expiresAt], diff --git a/server/src/db/meetings-db.ts b/server/src/db/meetings-db.ts index 5d9582bf9f6..c21c0312723 100644 --- a/server/src/db/meetings-db.ts +++ b/server/src/db/meetings-db.ts @@ -1,4 +1,5 @@ import { query } from './client.js'; +import { withActiveCredentialEventMutation } from './identity-db.js'; import type { Meeting, MeetingSeries, @@ -652,15 +653,21 @@ export class MeetingsDatabase { * Update topic subscription (create if not exists) */ async updateTopicSubscription(input: UpdateTopicSubscriptionInput): Promise<WorkingGroupTopicSubscription> { - const result = await query<WorkingGroupTopicSubscription>( - `INSERT INTO working_group_topic_subscriptions (working_group_id, workos_user_id, topic_slugs) - VALUES ($1, $2, $3) - ON CONFLICT (working_group_id, workos_user_id) - DO UPDATE SET topic_slugs = $3, updated_at = NOW() - RETURNING *`, - [input.working_group_id, input.workos_user_id, input.topic_slugs] - ); - return result.rows[0]; + const guarded = await withActiveCredentialEventMutation(input.workos_user_id, async (client) => { + const result = await client.query<WorkingGroupTopicSubscription>( + `INSERT INTO working_group_topic_subscriptions (working_group_id, workos_user_id, topic_slugs) + VALUES ($1, $2, $3) + ON CONFLICT (working_group_id, workos_user_id) + DO UPDATE SET topic_slugs = $3, updated_at = NOW() + RETURNING *`, + [input.working_group_id, input.workos_user_id, input.topic_slugs], + ); + return result.rows[0]; + }); + if (!guarded.applied || !guarded.value) { + throw new Error('Cannot update topic subscriptions for an inactive credential'); + } + return guarded.value; } /** diff --git a/server/src/db/membership-db.ts b/server/src/db/membership-db.ts index 199570b5e3a..5f4cf9e37c5 100644 --- a/server/src/db/membership-db.ts +++ b/server/src/db/membership-db.ts @@ -10,10 +10,15 @@ import type { PoolClient } from 'pg'; import { getPool, getClient } from './client.js'; import { findPayingOrgForDomain } from './org-filters.js'; import { createLogger } from '../logger.js'; +import { withActiveCredentialEventMutation } from './identity-db.js'; const logger = createLogger('membership-db'); const OWNERLESS_PROMOTION_LOCK_TIMEOUT_MS = 5_000; +function shouldDeferAutomaticProviderAuthorityWrites(): boolean { + return true; +} + // ── Types ──────────────────────────────────────────────────────────── /** Tracks how each organization_memberships row came to exist. */ @@ -118,10 +123,21 @@ async function withOwnerlessOrgPromotionLock<T>( */ export async function upsertOrganizationMembership( params: MembershipUpsertParams, + externalClient?: PoolClient, ): Promise<MembershipUpsertResult> { - const pool = getPool(); + if (!externalClient) { + const guarded = await withActiveCredentialEventMutation( + params.user_id, + (client) => upsertOrganizationMembership(params, client), + ); + if (!guarded.applied || !guarded.value) { + throw new Error('Cannot write organization authority for an inactive credential'); + } + return guarded.value; + } + const db = externalClient ?? getPool(); - const result = await pool.query<{ role: string }>( + const result = await db.query<{ role: string }>( `INSERT INTO organization_memberships ( workos_user_id, workos_organization_id, @@ -207,8 +223,20 @@ export async function resolveRoleWithWorkosFirstPromote(args: { }> { const { workos, membershipId, userId, organizationId, incomingRole } = args; + // Upstream role mutation needs a durable intent and reconciliation record: + // a timeout or concurrent credential deletion otherwise leaves an unknown + // provider-side authority outcome that cannot safely be replayed. Preserve + // the signed provider role and defer owner selection to an explicit workflow. + void workos; + logger.warn( + { membershipId, userId, orgId: organizationId, incomingRole }, + 'Automatic ownerless promotion deferred: no durable provider intent journal', + ); + return { role: incomingRole, promoted: false }; + // Only the 'member' input is eligible for ownerless-org promotion. Any // explicit role from WorkOS passes through unchanged. + /* c8 ignore start -- retained decisioning for an eventual journaled workflow */ if (incomingRole !== 'member') { return { role: incomingRole, promoted: false }; } @@ -273,6 +301,7 @@ export async function resolveRoleWithWorkosFirstPromote(args: { ); return { role: 'member', promoted: false, promotionError: err }; } + /* c8 ignore stop */ } // ── Delete ─────────────────────────────────────────────────────────── @@ -339,10 +368,11 @@ export async function deleteOrganizationMembership( export async function consumeInvitationSeatType( organizationId: string, email: string, + externalClient?: PoolClient, ): Promise<{ seat_type: string; source: ProvisioningSource | null } | null> { - const pool = getPool(); + const db = externalClient ?? getPool(); - const result = await pool.query<{ seat_type: string; source: string | null }>( + const result = await db.query<{ seat_type: string; source: string | null }>( `DELETE FROM invitation_seat_types WHERE workos_organization_id = $1 AND lower(email) = lower($2) RETURNING seat_type, source`, @@ -390,13 +420,16 @@ export async function setMembershipRole( organizationId: string, role: string, ): Promise<void> { - const pool = getPool(); - - await pool.query( - `UPDATE organization_memberships SET role = $3, updated_at = NOW() - WHERE workos_user_id = $1 AND workos_organization_id = $2`, - [userId, organizationId, role], - ); + const guarded = await withActiveCredentialEventMutation(userId, async (client) => { + await client.query( + `UPDATE organization_memberships SET role = $3, updated_at = NOW() + WHERE workos_user_id = $1 AND workos_organization_id = $2`, + [userId, organizationId, role], + ); + }); + if (!guarded.applied) { + throw new Error('Cannot update organization authority for an inactive credential'); + } } // ── Auto-link by verified domain ──────────────────────────────────── @@ -411,8 +444,9 @@ export interface DomainLinkResult { * Check whether a user's email domain matches a verified domain on an * organization with an active subscription — directly or via the brand * registry hierarchy (e.g. AnalyticsIQ → Alliant). If so, and the org has - * consented to the relevant auto-provisioning class, create a WorkOS - * membership for them on the resolved paying org. + * consented to the relevant auto-provisioning class, this formerly created a + * WorkOS membership on the resolved paying org. Automatic upstream authority + * is currently deferred until a durable intent/reconciliation journal exists. * * Two consent flags on the resolved org, with different defaults: * - auto_provision_verified_domain (default true) gates DIRECT matches @@ -424,9 +458,9 @@ export interface DomainLinkResult { * LLM classification or admin PATCH, ages on M&A, and the joining user * gets no domain-level confirmation. Opt-in. * - * Idempotent: short-circuits when the user is already in the resolved org's - * local membership cache, and treats `organization_membership_already_exists` - * from WorkOS as success. Safe to call on every authenticated request. + * Calls currently fail closed without contacting WorkOS or staging a local + * membership. Keep the legacy decisioning below for an eventual explicitly + * journaled workflow; do not enable it as an untracked automatic mutation. * * Hierarchy walk uses the same trust gates as resolveEffectiveMembership * (high-confidence classifications, 180-day TTL, max 4 hops up) so pre-link @@ -437,6 +471,13 @@ export async function autoLinkByVerifiedDomain( userId: string, email: string, ): Promise<DomainLinkResult | null> { + if (shouldDeferAutomaticProviderAuthorityWrites()) { + logger.warn( + { userId }, + 'Verified-domain auto-provision deferred: no durable exactly-once provider intent journal', + ); + return null; + } const pool = getPool(); const emailDomain = email.split('@')[1]?.toLowerCase(); if (!emailDomain) return null; diff --git a/server/src/db/organization-db.ts b/server/src/db/organization-db.ts index a305e6c087d..ed0be179e77 100644 --- a/server/src/db/organization-db.ts +++ b/server/src/db/organization-db.ts @@ -502,12 +502,13 @@ export async function getSeatUsage(orgId: string): Promise<{ contributor: number */ export async function canAddSeat( orgId: string, - seatType: SeatType + seatType: SeatType, + externalClient?: PoolClient, ): Promise<{ allowed: boolean; reason?: string }> { - const pool = getPool(); - const client = await pool.connect(); + const client = externalClient ?? await getPool().connect(); + const ownsTransaction = externalClient === undefined; try { - await client.query('BEGIN'); + if (ownsTransaction) await client.query('BEGIN'); // Lock the org row to serialize concurrent seat checks and resolve // the tier from the same (locked) read. @@ -554,7 +555,7 @@ export async function canAddSeat( community_only: (total - contributors) + pendingCommunity, }; - await client.query('COMMIT'); + if (ownsTransaction) await client.query('COMMIT'); const limit = seatType === 'contributor' ? limits.contributor : limits.community; const used = seatType === 'contributor' ? usage.contributor : usage.community_only; @@ -564,10 +565,10 @@ export async function canAddSeat( if (used >= limit) return { allowed: false, reason: `All ${limit} ${seatType === 'contributor' ? 'contributor' : 'community'} seats are in use. Upgrade at https://agenticadvertising.org/dashboard/membership to add more.` }; return { allowed: true }; } catch (error) { - await client.query('ROLLBACK'); + if (ownsTransaction) await client.query('ROLLBACK'); throw error; } finally { - client.release(); + if (ownsTransaction) client.release(); } } diff --git a/server/src/db/slack-db.ts b/server/src/db/slack-db.ts index e07928f4f77..2459caef3d3 100644 --- a/server/src/db/slack-db.ts +++ b/server/src/db/slack-db.ts @@ -7,6 +7,8 @@ import type { } from '../slack/types.js'; import { FREE_EMAIL_PROVIDER_DOMAINS } from '../services/identifier-normalization.js'; import { splitFullName } from '../utils/resolve-user-name.js'; +import type { PoolClient } from 'pg'; +import { withActiveCredentialEventMutation } from './identity-db.js'; /** * Escape LIKE pattern wildcards to prevent SQL injection @@ -116,8 +118,15 @@ export class SlackDatabase { workos_user_id: string; mapping_source: SlackMappingSource; mapped_by_user_id?: string; - }): Promise<SlackUserMapping | null> { - const result = await query<SlackUserMapping>( + }, externalClient?: PoolClient): Promise<SlackUserMapping | null> { + if (!externalClient) { + const guarded = await withActiveCredentialEventMutation( + input.workos_user_id, + (client) => this.mapUser(input, client), + ); + return guarded.applied ? guarded.value ?? null : null; + } + const result = await externalClient.query<SlackUserMapping>( `UPDATE slack_user_mappings SET workos_user_id = $1, mapping_status = 'mapped', @@ -146,7 +155,7 @@ export class SlackDatabase { const slackName = mapping.slack_real_name || mapping.slack_display_name; if (slackName?.trim()) { const { firstName, lastName } = splitFullName(slackName); - await query( + await externalClient.query( `UPDATE users SET first_name = COALESCE(NULLIF(TRIM(first_name), ''), $1), last_name = COALESCE(NULLIF(TRIM(last_name), ''), $2), diff --git a/server/src/db/working-group-db.ts b/server/src/db/working-group-db.ts index c095ec07a39..e313da00c38 100644 --- a/server/src/db/working-group-db.ts +++ b/server/src/db/working-group-db.ts @@ -3,6 +3,10 @@ import type { PoolClient } from 'pg'; import { computeJourneyStage } from '../addie/services/journey-computation.js'; import { CommunityDatabase } from './community-db.js'; import { createLogger } from '../logger.js'; +import { + withActiveCredentialEventMutation, + withActiveCredentialSetMutation, +} from './identity-db.js'; import type { WorkingGroup, WorkingGroupLeader, @@ -95,11 +99,19 @@ export class WorkingGroupDatabase { * narrow bypass: they resolve the server-owned slug in their transaction * and do not call this generic mutation surface. */ - private async assertGenericMutationAllowed(workingGroupId: string): Promise<void> { - const result = await query<{ slug: string }>( - 'SELECT slug FROM working_groups WHERE id = $1', - [workingGroupId], - ); + private async assertGenericMutationAllowed( + workingGroupId: string, + externalClient?: PoolClient, + ): Promise<void> { + const result = externalClient + ? await externalClient.query<{ slug: string }>( + 'SELECT slug FROM working_groups WHERE id = $1', + [workingGroupId], + ) + : await query<{ slug: string }>( + 'SELECT slug FROM working_groups WHERE id = $1', + [workingGroupId], + ); if (result.rows[0]?.slug === AAO_ADMIN_WORKING_GROUP_SLUG) { throw new Error('AAO site-admin group can only be changed through the dedicated audited workflow'); } @@ -755,28 +767,12 @@ export class WorkingGroupDatabase { ); const subgroup = subgroupResult.rows[0]; - // 4. Promote subscribers to members - const subResult = await client.query<{ workos_user_id: string }>( - `SELECT DISTINCT workos_user_id FROM working_group_topic_subscriptions - WHERE working_group_id = $1 AND $2 = ANY(topic_slugs)`, - [parent.id, topicSlug] - ); - let addedMembers = 0; - for (const row of subResult.rows) { - const existing = await client.query<{ id: string }>( - `SELECT id FROM working_group_memberships - WHERE working_group_id = $1 AND workos_user_id = $2`, - [subgroup.id, row.workos_user_id] - ); - if (existing.rows[0]) continue; - await client.query( - `INSERT INTO working_group_memberships ( - working_group_id, workos_user_id, status, joined_at, updated_at - ) VALUES ($1, $2, 'active', NOW(), NOW())`, - [subgroup.id, row.workos_user_id] - ); - addedMembers++; - } + // 4. Do not turn subscriptions into authority automatically. Without a + // durable graduation intent journal, a subscriber can be deleted while + // this long content transaction is in flight and then be recreated as + // an active subgroup member. Operators may add active subscribers via + // the lifecycle-fenced membership workflow after graduation. + const addedMembers = 0; // 5. Strip the topic slug from parent subscription rows; delete empty ones await client.query( @@ -921,24 +917,30 @@ export class WorkingGroupDatabase { // Resolve Slack IDs to canonical WorkOS IDs to prevent duplicates const canonicalUserId = await this.resolveToCanonicalUserId(input.workos_user_id); - const result = await query<WorkingGroupMembership>( - `INSERT INTO working_group_memberships ( - working_group_id, workos_user_id, user_email, user_name, user_org_name, - workos_organization_id, added_by_user_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (working_group_id, workos_user_id) - DO UPDATE SET status = 'active', updated_at = NOW() - RETURNING *`, - [ - input.working_group_id, - canonicalUserId, - input.user_email || null, - input.user_name || null, - input.user_org_name || null, - input.workos_organization_id || null, - input.added_by_user_id || null, - ] - ); + const guarded = await withActiveCredentialEventMutation(canonicalUserId, async (client) => { + const result = await client.query<WorkingGroupMembership>( + `INSERT INTO working_group_memberships ( + working_group_id, workos_user_id, user_email, user_name, user_org_name, + workos_organization_id, added_by_user_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (working_group_id, workos_user_id) + DO UPDATE SET status = 'active', updated_at = NOW() + RETURNING *`, + [ + input.working_group_id, + canonicalUserId, + input.user_email || null, + input.user_name || null, + input.user_org_name || null, + input.workos_organization_id || null, + input.added_by_user_id || null, + ], + ); + return result.rows[0]; + }); + if (!guarded.applied || !guarded.value) { + throw new Error('Cannot add working-group authority to an inactive credential'); + } // Fire-and-forget journey recomputation const orgId = input.workos_organization_id || (await query<{ workos_organization_id: string }>( @@ -950,7 +952,7 @@ export class WorkingGroupDatabase { .catch((err) => { logger.error({ err, workingGroupId: input.working_group_id }, 'Journey stage computation failed'); }); } - return result.rows[0]; + return guarded.value; } /** @@ -1012,9 +1014,8 @@ export class WorkingGroupDatabase { async grantAAOAdminMembership( input: AAOAdminMembershipMutationInput, ): Promise<WorkingGroupMembership> { - const client = await getPool().connect(); - try { - await client.query('BEGIN'); + const targetUserId = await this.resolveToCanonicalUserId(input.targetUserId); + const guarded = await withActiveCredentialEventMutation(targetUserId, async (client) => { const groupResult = await client.query<{ id: string }>( 'SELECT id FROM working_groups WHERE slug = $1 FOR SHARE', [AAO_ADMIN_WORKING_GROUP_SLUG], @@ -1022,7 +1023,6 @@ export class WorkingGroupDatabase { const group = groupResult.rows[0]; if (!group) throw new Error('AAO admin working group not found'); - const targetUserId = await this.resolveCanonicalUserIdWithClient(client, input.targetUserId); const membershipResult = await client.query<WorkingGroupMembership>( `INSERT INTO working_group_memberships ( working_group_id, workos_user_id, added_by_user_id @@ -1039,16 +1039,12 @@ export class WorkingGroupDatabase { ) VALUES ('granted', $1, $2, 'aao_admin_working_group', $3, $4)`, [input.actorUserId, targetUserId, input.actorAuthorizationMechanism, input.reason], ); - await client.query('COMMIT'); return membershipResult.rows[0]; - } catch (error) { - await client.query('ROLLBACK').catch((rollbackError) => { - logger.warn({ err: rollbackError }, 'AAO admin grant rollback failed'); - }); - throw error; - } finally { - client.release(); + }); + if (!guarded.applied || !guarded.value) { + throw new Error('Cannot grant AAO admin authority to an inactive credential'); } + return guarded.value; } /** @@ -1373,26 +1369,36 @@ export class WorkingGroupDatabase { ); const oldLeaderIds = oldLeadersResult.rows.map(r => r.user_id); - // Remove existing leaders - await query( - 'DELETE FROM working_group_leaders WHERE working_group_id = $1', - [workingGroupId] - ); - - // Add new leaders in a single bulk insert - if (uniqueUserIds.length > 0) { - const values = uniqueUserIds.map((_, i) => `($1, $${i + 2})`).join(', '); - await query( - `INSERT INTO working_group_leaders (working_group_id, user_id) - VALUES ${values} - ON CONFLICT DO NOTHING`, - [workingGroupId, ...uniqueUserIds] - ); + if (uniqueUserIds.length === 0) { + await query('DELETE FROM working_group_leaders WHERE working_group_id = $1', [workingGroupId]); + } else { + const guarded = await withActiveCredentialSetMutation(uniqueUserIds, async (client) => { + await client.query( + 'DELETE FROM working_group_leaders WHERE working_group_id = $1', + [workingGroupId], + ); + const values = uniqueUserIds.map((_, i) => `($1, $${i + 2})`).join(', '); + await client.query( + `INSERT INTO working_group_leaders (working_group_id, user_id) + VALUES ${values} + ON CONFLICT DO NOTHING`, + [workingGroupId, ...uniqueUserIds], + ); + await client.query( + `INSERT INTO working_group_memberships (working_group_id, workos_user_id, status) + SELECT $1, user_id, 'active' + FROM working_group_leaders + WHERE working_group_id = $1 + ON CONFLICT (working_group_id, workos_user_id) + DO UPDATE SET status = 'active', updated_at = NOW()`, + [workingGroupId], + ); + }); + if (!guarded.applied) { + throw new Error('Cannot assign leadership to an inactive credential'); + } } - // Ensure leaders are members - await this.ensureLeadersAreMembers(workingGroupId); - // Fire-and-forget journey recomputation for affected orgs const allAffectedUserIds = [...new Set([...oldLeaderIds, ...uniqueUserIds])]; if (allAffectedUserIds.length > 0) { @@ -1410,20 +1416,40 @@ export class WorkingGroupDatabase { /** * Add a leader to a working group */ - async addLeader(workingGroupId: string, userId: string): Promise<void> { - await this.assertGenericMutationAllowed(workingGroupId); + async addLeader( + workingGroupId: string, + userId: string, + externalClient?: PoolClient, + ): Promise<void> { + await this.assertGenericMutationAllowed(workingGroupId, externalClient); // Resolve Slack IDs to canonical WorkOS IDs to prevent duplicates - const canonicalUserId = await this.resolveToCanonicalUserId(userId); + const canonicalUserId = externalClient + ? await this.resolveCanonicalUserIdWithClient(externalClient, userId) + : await this.resolveToCanonicalUserId(userId); - await query( - `INSERT INTO working_group_leaders (working_group_id, user_id) - VALUES ($1, $2) - ON CONFLICT DO NOTHING`, - [workingGroupId, canonicalUserId] - ); + if (externalClient) { + await externalClient.query( + `INSERT INTO working_group_leaders (working_group_id, user_id) + VALUES ($1, $2) + ON CONFLICT DO NOTHING`, + [workingGroupId, canonicalUserId], + ); + await externalClient.query( + `INSERT INTO working_group_memberships (working_group_id, workos_user_id, status) + VALUES ($1, $2, 'active') + ON CONFLICT (working_group_id, workos_user_id) + DO UPDATE SET status = 'active', updated_at = NOW()`, + [workingGroupId, canonicalUserId], + ); + return; + } - // Ensure leader is a member - await this.ensureLeadersAreMembers(workingGroupId); + const guarded = await withActiveCredentialEventMutation(canonicalUserId, async (client) => { + await this.addLeader(workingGroupId, canonicalUserId, client); + }); + if (!guarded.applied) { + throw new Error('Cannot assign leadership to an inactive credential'); + } // Fire-and-forget journey recomputation const orgResult = await query<{ workos_organization_id: string }>( @@ -2029,32 +2055,37 @@ export class WorkingGroupDatabase { // Resolve Slack IDs to canonical WorkOS IDs to prevent duplicates const canonicalUserId = await this.resolveToCanonicalUserId(input.workos_user_id); - const result = await query<WorkingGroupMembership>( - `INSERT INTO working_group_memberships ( - working_group_id, workos_user_id, user_email, user_name, user_org_name, - workos_organization_id, added_by_user_id, interest_level, interest_source - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (working_group_id, workos_user_id) - DO UPDATE SET - status = 'active', - interest_level = COALESCE(EXCLUDED.interest_level, working_group_memberships.interest_level), - interest_source = COALESCE(EXCLUDED.interest_source, working_group_memberships.interest_source), - updated_at = NOW() - RETURNING *`, - [ - input.working_group_id, - canonicalUserId, - input.user_email || null, - input.user_name || null, - input.user_org_name || null, - input.workos_organization_id || null, - input.added_by_user_id || null, - input.interest_level || null, - input.interest_source || null, - ] - ); - - return result.rows[0]; + const guarded = await withActiveCredentialEventMutation(canonicalUserId, async (client) => { + const result = await client.query<WorkingGroupMembership>( + `INSERT INTO working_group_memberships ( + working_group_id, workos_user_id, user_email, user_name, user_org_name, + workos_organization_id, added_by_user_id, interest_level, interest_source + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (working_group_id, workos_user_id) + DO UPDATE SET + status = 'active', + interest_level = COALESCE(EXCLUDED.interest_level, working_group_memberships.interest_level), + interest_source = COALESCE(EXCLUDED.interest_source, working_group_memberships.interest_source), + updated_at = NOW() + RETURNING *`, + [ + input.working_group_id, + canonicalUserId, + input.user_email || null, + input.user_name || null, + input.user_org_name || null, + input.workos_organization_id || null, + input.added_by_user_id || null, + input.interest_level || null, + input.interest_source || null, + ], + ); + return result.rows[0]; + }); + if (!guarded.applied || !guarded.value) { + throw new Error('Cannot add working-group authority to an inactive credential'); + } + return guarded.value; } /** diff --git a/server/src/http.ts b/server/src/http.ts index 56fdd272441..d3408d19409 100644 --- a/server/src/http.ts +++ b/server/src/http.ts @@ -58,12 +58,16 @@ import { PropertyDatabase } from "./db/property-db.js"; import * as manifestRefsDb from "./db/manifest-refs-db.js"; import { JoinRequestDatabase } from "./db/join-request-db.js"; import { SlackDatabase } from "./db/slack-db.js"; -import { autoLinkByVerifiedDomain } from "./db/membership-db.js"; +import { autoLinkByVerifiedDomain, upsertOrganizationMembership } from "./db/membership-db.js"; import { syncSlackUsers, getSyncStatus, tryAutoLinkWebsiteUserToSlack } from "./slack/sync.js"; import { isSlackConfigured, testSlackConnection } from "./slack/client.js"; import { handleSlashCommand } from "./slack/commands.js"; import { getCompanyDomain, getGoogleEmailAliases } from "./utils/email-domain.js"; import { assertIdentityConsolidationAllowed } from "./db/identity-mutation-policy.js"; +import { + upsertWorkosUserInCredentialEvent, + withCredentialCreationEventMutation, +} from "./db/identity-db.js"; import { hasActiveSlackLink } from "./utils/slack-linkage.js"; import { isUuid } from "./utils/uuid.js"; import { resolveUserNameWithFallbacks, sanitizeName } from "./utils/resolve-user-name.js"; @@ -7764,25 +7768,27 @@ ${p.category ? `<category>${p.category}</category>\n` : ''}<url>${publishedUrl}< // On INSERT, use WorkOS values — falling back to existing DB / Slack // mapping when WorkOS itself has empty names. On UPDATE, preserve // user-set names: only fill in names that are currently empty. - try { - const pool = getPool(); - const { firstName, lastName } = await resolveUserNameWithFallbacks( - pool, user.id, user.firstName, user.lastName, - ); - await pool.query( - `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, workos_created_at, workos_updated_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) - ON CONFLICT (workos_user_id) DO UPDATE SET - email = EXCLUDED.email, - first_name = COALESCE(NULLIF(TRIM(users.first_name), ''), EXCLUDED.first_name), - last_name = COALESCE(NULLIF(TRIM(users.last_name), ''), EXCLUDED.last_name), - email_verified = EXCLUDED.email_verified, - workos_updated_at = EXCLUDED.workos_updated_at, - updated_at = NOW()`, - [user.id, user.email, firstName, lastName, user.emailVerified, user.createdAt, user.updatedAt] - ); - } catch (upsertError) { - logger.error({ error: upsertError, userId: user.id }, 'Failed to upsert user on login'); + const pool = getPool(); + const { firstName, lastName } = await resolveUserNameWithFallbacks( + pool, user.id, user.firstName, user.lastName, + ); + const localFinalize = await withCredentialCreationEventMutation(user.id, async (client) => { + await upsertWorkosUserInCredentialEvent(client, { + id: user.id, + email: user.email, + firstName, + lastName, + emailVerified: user.emailVerified, + createdAt: user.createdAt, + updatedAt: user.updatedAt, + }, 'preserve_existing'); + }); + if (!localFinalize.applied) { + logger.warn({ userId: user.id }, 'Refused OAuth callback for terminal local credential'); + return res.status(403).json({ + error: 'Account unavailable', + message: 'This account cannot be authenticated. Contact support if this is unexpected.', + }); } // Auto-merge duplicate accounts caused by Google email aliases. @@ -8177,30 +8183,40 @@ ${p.category ? `<category>${p.category}</category>\n` : ''}<url>${publishedUrl}< if (existingMapping && !existingMapping.workos_user_id) { // Link the Slack user to the newly authenticated WorkOS user - await slackDb.mapUser({ + const mapped = await slackDb.mapUser({ slack_user_id: slackUserIdToLink, workos_user_id: user.id, mapping_source: 'user_claimed', }); - accountLinked = true; - accountNewlyLinked = true; - logger.info( - { slackUserId: slackUserIdToLink, workosUserId: user.id }, - 'Auto-linked Slack account after signup' - ); + if (!mapped) { + logger.warn( + { slackUserId: slackUserIdToLink, workosUserId: user.id }, + 'Skipped Slack account link because the local credential is no longer active', + ); + } else { + accountLinked = true; + accountNewlyLinked = true; + logger.info( + { slackUserId: slackUserIdToLink, workosUserId: user.id }, + 'Auto-linked Slack account after signup' + ); + } - // Record account linking in the relationship system - try { - const { resolvePersonId } = await import('./db/relationship-db.js'); - const { recordEvent } = await import('./db/person-events-db.js'); - const personId = await resolvePersonId({ slack_user_id: slackUserIdToLink, workos_user_id: user.id }); - await recordEvent(personId, 'account_linked', { - channel: 'web', - data: { workos_user_id: user.id }, - }); - logger.info({ slackUserId: slackUserIdToLink, personId }, 'Recorded account_linked event'); - } catch (trackingError) { - logger.warn({ error: trackingError, slackUserId: slackUserIdToLink }, 'Failed to record account_linked event'); + // Record account linking only after the lifecycle-fenced map + // actually commits. + if (mapped) { + try { + const { resolvePersonId } = await import('./db/relationship-db.js'); + const { recordEvent } = await import('./db/person-events-db.js'); + const personId = await resolvePersonId({ slack_user_id: slackUserIdToLink, workos_user_id: user.id }); + await recordEvent(personId, 'account_linked', { + channel: 'web', + data: { workos_user_id: user.id }, + }); + logger.info({ slackUserId: slackUserIdToLink, personId }, 'Recorded account_linked event'); + } catch (trackingError) { + logger.warn({ error: trackingError, slackUserId: slackUserIdToLink }, 'Failed to record account_linked event'); + } } } else if (!existingMapping) { @@ -9267,13 +9283,21 @@ ${p.category ? `<category>${p.category}</category>\n` : ''}<url>${publishedUrl}< role: roleSlug, }, 'User auto-added to organization via verified domain'); - // Mirror membership locally so it's visible immediately - const pool2 = getPool(); - await pool2.query(` - INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role, created_at, updated_at, synced_at) - VALUES ($1, $2, $3, $4, NOW(), NOW(), NOW()) - ON CONFLICT (workos_user_id, workos_organization_id) DO UPDATE SET role = $4, updated_at = NOW() - `, [user.id, organization_id, user.email, roleSlug]); + // Mirror only while the exact credential is still active. A + // provider response that settles after deletion must not restore + // local organization authority. + await upsertOrganizationMembership({ + user_id: user.id, + organization_id, + membership_id: membership.id, + email: user.email, + first_name: user.firstName ?? null, + last_name: user.lastName ?? null, + role: roleSlug, + seat_type: 'community_only', + has_explicit_seat_type: false, + provisioning_source: 'verified_domain', + }); // Record audit log await orgDb.recordAuditLog({ diff --git a/server/src/mcp/oauth-provider.ts b/server/src/mcp/oauth-provider.ts index c3957f246a2..dd66b1ff1b3 100644 --- a/server/src/mcp/oauth-provider.ts +++ b/server/src/mcp/oauth-provider.ts @@ -22,6 +22,11 @@ import { verifyWorkOSJWT } from '../auth/workos-jwt.js'; import { getAuthorizationUrl, refreshTokenRaw, authenticateWithCodeForTokens } from '../auth/workos-client.js'; import * as mcpClientsDb from '../db/mcp-clients-db.js'; import * as mcpOAuthStateDb from '../db/mcp-oauth-state-db.js'; +import { + upsertWorkosUserInCredentialEvent, + withCredentialCreationEventMutation, +} from '../db/identity-db.js'; +import { readCredentialAuthorizationLifecycle } from '../db/authorization-epoch-db.js'; const logger = createLogger('mcp-oauth'); @@ -49,6 +54,17 @@ async function verifyAccessTokenJWT(token: string): Promise<AuthInfo> { throw new InvalidTokenError('Invalid or expired token'); } + if (!verified.isM2M && verified.sub) { + const lifecycle = await readCredentialAuthorizationLifecycle(verified.sub); + if (lifecycle.status !== 'active') { + logger.warn( + { sub: verified.sub, lifecycle: lifecycle.status }, + 'MCP OAuth: local credential lifecycle rejected token', + ); + throw new InvalidTokenError('Credential is unavailable'); + } + } + return { token, clientId: verified.clientId, @@ -253,46 +269,45 @@ export async function handleMCPOAuthCallback( return; } - // Upsert the user into our local table so downstream code (REST requireAuth, - // /api/me/*, dashboard queries) can find them by workos_user_id. The - // cookie-based /auth/callback path does the same upsert; without it, - // users who first arrive via the MCP OAuth flow don't exist locally and - // REST auth rejects their JWT. - // - // On failure we log and continue — we still issue the OAuth code so /mcp - // works. The user will see a 401 the first time they call /api/*, with a - // corresponding warn log ("Bearer JWT verified but user not found in - // local DB"). A retry (re-SSO) recovers; a persistent failure indicates - // a DB problem that operators need to investigate, not a per-user issue. + // Finalize the local credential and MCP authorization code atomically under + // the same terminal lifecycle lock used by provider deletion. The WorkOS + // exchange above performs provider I/O before lock acquisition. A deletion + // that wins during that exchange leaves a tombstone and this finalize emits + // no local code. + const localCode = crypto.randomBytes(32).toString('hex'); + let finalized = false; try { - const { getPool } = await import('../db/client.js'); const { user } = authResult; - await getPool().query( - `INSERT INTO users (workos_user_id, email, first_name, last_name, email_verified, workos_created_at, workos_updated_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW()) - ON CONFLICT (workos_user_id) DO UPDATE SET - email = EXCLUDED.email, - first_name = COALESCE(NULLIF(TRIM(users.first_name), ''), EXCLUDED.first_name), - last_name = COALESCE(NULLIF(TRIM(users.last_name), ''), EXCLUDED.last_name), - email_verified = EXCLUDED.email_verified, - workos_updated_at = EXCLUDED.workos_updated_at, - updated_at = NOW()`, - [user.id, user.email, user.firstName, user.lastName, user.emailVerified, user.createdAt, user.updatedAt], - ); - } catch (upsertErr) { - logger.error({ err: upsertErr }, 'MCP OAuth: Failed to upsert user on callback'); + const result = await withCredentialCreationEventMutation(user.id, async (client) => { + await upsertWorkosUserInCredentialEvent(client, { + id: user.id, + email: user.email, + firstName: user.firstName ?? null, + lastName: user.lastName ?? null, + emailVerified: user.emailVerified, + createdAt: user.createdAt, + updatedAt: user.updatedAt, + }, 'preserve_existing'); + await mcpOAuthStateDb.setAuthCode(localCode, { + clientId: pending.clientId, + codeChallenge: pending.codeChallenge, + redirectUri: pending.redirectUri, + accessToken: authResult.accessToken, + refreshToken: authResult.refreshToken, + }, client); + }); + finalized = result.applied; + } catch (finalizeErr) { + logger.error({ err: finalizeErr }, 'MCP OAuth: Failed closed during local callback finalize'); + } + if (!finalized) { + const errorUrl = new URL(pending.redirectUri); + errorUrl.searchParams.set('error', 'access_denied'); + errorUrl.searchParams.set('error_description', 'Local credential is unavailable'); + if (pending.state) errorUrl.searchParams.set('state', pending.state); + res.redirect(errorUrl.toString()); + return; } - - // Generate local authorization code - const localCode = crypto.randomBytes(32).toString('hex'); - - await mcpOAuthStateDb.setAuthCode(localCode, { - clientId: pending.clientId, - codeChallenge: pending.codeChallenge, - redirectUri: pending.redirectUri, - accessToken: authResult.accessToken, - refreshToken: authResult.refreshToken, - }); // Redirect to MCP client's callback URL const redirectUrl = new URL(pending.redirectUri); diff --git a/server/src/middleware/auth.ts b/server/src/middleware/auth.ts index 9e005c8e431..902ffb9ac50 100644 --- a/server/src/middleware/auth.ts +++ b/server/src/middleware/auth.ts @@ -14,7 +14,11 @@ import { isWorkOSApiKeyFormat } from './api-key-format.js'; import { verifyWorkOSJWT, looksLikeJWT } from '../auth/workos-jwt.js'; import { storeRefreshedSession, getRefreshedSession, cleanExpiredRefreshes } from '../db/session-refresh-db.js'; import { getPool } from '../db/client.js'; -import { getAuthorizationFingerprint } from '../db/authorization-epoch-db.js'; +import { + readCredentialAuthorizationLifecycle, + type ActiveCredentialAuthorizationSnapshot, + type CredentialAuthorizationLifecycle, +} from '../db/authorization-epoch-db.js'; import { getOrganizationAuthorizationUserId } from '../auth/organization-principal.js'; import { constantTimeEqual } from '../utils/constant-time-equal.js'; import { resolveEffectiveMembership } from '../db/org-filters.js'; @@ -380,34 +384,31 @@ export async function validateWorkOSBearerJWT(req: Request): Promise<ValidatedBe if (verified.isM2M || !verified.sub) return null; - // Confirm the subject corresponds to a real local user. This catches - // tokens from WorkOS accounts that have been deleted or never synced, - // and gives us names for the synthesized WorkOSUser. - const pool = getPool(); - const localUser = await pool.query<{ - first_name: string | null; - last_name: string | null; - email: string | null; - }>( - `SELECT first_name, last_name, email FROM users WHERE workos_user_id = $1`, - [verified.sub], - ); - if (localUser.rowCount === 0) { - logger.warn({ sub: verified.sub }, 'Bearer JWT verified but user not found in local DB'); + // A valid WorkOS signature is not enough: accept and cache only an + // authoritative local credential snapshot that includes the binding and + // proves no durable deletion tombstone exists. + const lifecycle = await readCredentialAuthorizationLifecycle(verified.sub); + if (lifecycle.status === 'unavailable') { + logger.warn({ sub: verified.sub }, 'Bearer JWT local credential lifecycle unavailable — failed closed'); + return null; + } + if (lifecycle.status === 'terminal') { + logger.warn({ sub: verified.sub }, 'Bearer JWT verified but credential is not active locally'); return null; } + const activeSnapshot = lifecycle.snapshot; - const row = localUser.rows[0]; - const email = verified.email ?? row.email ?? ''; + const email = verified.email ?? activeSnapshot.email ?? ''; const user: WorkOSUser = { id: verified.sub, email, - firstName: row.first_name?.trim() || undefined, - lastName: row.last_name?.trim() || undefined, + firstName: activeSnapshot.first_name?.trim() || undefined, + lastName: activeSnapshot.last_name?.trim() || undefined, emailVerified: true, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }; + applyActiveCredentialSnapshot(user, activeSnapshot); const tokenExpMs = verified.expiresAt ? verified.expiresAt * 1000 : Infinity; const cacheUntil = Math.min(now + BEARER_JWT_CACHE_TTL_MS, tokenExpMs); @@ -420,7 +421,7 @@ export async function validateWorkOSBearerJWT(req: Request): Promise<ValidatedBe user, orgId: verified.orgId, expiresAt: cacheUntil, - authorizationFingerprint: await authorizationFingerprintFor(user), + authorizationFingerprint: activeSnapshot.fingerprint, }); } @@ -787,6 +788,19 @@ async function attachIdentityId(user: WorkOSUser): Promise<void> { } } +function applyActiveCredentialSnapshot( + user: WorkOSUser, + snapshot: ActiveCredentialAuthorizationSnapshot, +): void { + const authenticatedWorkosUserId = snapshot.workos_user_id; + user.identityId = snapshot.identity_id; + if (snapshot.primary_workos_user_id + && snapshot.primary_workos_user_id !== authenticatedWorkosUserId) { + user.authWorkosUserId = authenticatedWorkosUserId; + user.id = snapshot.primary_workos_user_id; + } +} + /** * Authorization fingerprint for the credential that actually authenticated * this session. `attachIdentityId` may swap `user.id` to the identity's @@ -798,19 +812,11 @@ async function attachIdentityId(user: WorkOSUser): Promise<void> { * Synthetic principals (admin API key, WorkOS API key) have no credential * row and no identity binding; they return '' and never mismatch. */ -async function authorizationFingerprintFor( +async function activeAuthorizationSnapshotFor( user: WorkOSUser, -): Promise<string | undefined> { - if (isSyntheticUser(user.id)) return ''; - try { - return await getAuthorizationFingerprint([getOrganizationAuthorizationUserId(user)]); - } catch (err) { - // Never fail authentication over the stamp: an unstamped entry reads as - // stale on its next hit, so the session is re-validated rather than - // served from a fingerprint we could not confirm. - logger.warn({ err }, 'Authorization epoch stamp failed — session will not be served from cache'); - return undefined; - } +): Promise<CredentialAuthorizationLifecycle> { + if (isSyntheticUser(user.id)) return { status: 'terminal', reason: 'missing_user' }; + return readCredentialAuthorizationLifecycle(getOrganizationAuthorizationUserId(user)); } /** @@ -828,9 +834,11 @@ async function isAuthorizationFingerprintCurrent( ): Promise<boolean> { if (isSyntheticUser(user.id)) return true; try { - return (await authorizationFingerprintFor(user)) === (stamped ?? ''); + const lifecycle = await activeAuthorizationSnapshotFor(user); + return lifecycle.status === 'active' + && lifecycle.snapshot.fingerprint === (stamped ?? ''); } catch (err) { - logger.warn({ err }, 'Authorization epoch check failed — bypassing session cache'); + logger.warn({ err }, 'Authoritative credential check failed — bypassing session cache'); return false; } } @@ -908,7 +916,6 @@ export async function requireAuth(req: Request, res: Response, next: NextFunctio logger.warn({ err: banError, userId: jwtAuth.user.id, path: req.path }, 'Ban check failed — allowing request through'); } - await attachIdentityId(req.user); return next(); } @@ -1168,21 +1175,22 @@ export async function requireAuth(req: Request, res: Response, next: NextFunctio let firstName = result.user.firstName ?? undefined; let lastName = result.user.lastName ?? undefined; - // Verify the user exists in our local DB. This catches stale sessions - // from deleted/merged WorkOS accounts whose JWTs haven't expired yet. - // Also resolves names from local DB when WorkOS doesn't provide them. - try { - const pool = getPool(); - const localUser = await pool.query<{ - first_name: string | null; - last_name: string | null; - }>( - `SELECT first_name, last_name FROM users WHERE workos_user_id = $1`, - [result.user.id] - ); - if (localUser.rows.length === 0) { + // WorkOS session validity alone cannot authorize a deleted credential. + // Read the user, binding, canonical route, tombstone absence, and epoch + // atomically before accepting or caching this session. + const lifecycle = await readCredentialAuthorizationLifecycle(result.user.id); + if (lifecycle.status === 'unavailable') { + sessionCache.delete(cacheKey); + logger.warn({ userId: result.user.id, path: req.path }, + 'Local credential lifecycle unavailable — failing authentication closed'); + return res.status(503).json({ + error: 'Service temporarily unavailable', + message: 'Unable to verify your account right now. Please try again shortly.', + }); + } + if (lifecycle.status === 'terminal') { logger.warn({ userId: result.user.id, email: result.user.email, path: req.path }, - 'Authenticated user not found in local DB — forcing re-login'); + 'Authenticated credential is not active in local DB — forcing re-login'); sessionCache.delete(cacheKey); if (deadSessionCache.size >= DEAD_SESSION_MAX_SIZE) { const oldest = deadSessionCache.keys().next().value; @@ -1197,15 +1205,11 @@ export async function requireAuth(req: Request, res: Response, next: NextFunctio message: 'Your account could not be verified. Please log in again.', login_url: '/auth/login', }); - } - if (!firstName?.trim()) { - const row = localUser.rows[0]; - if (row.first_name?.trim()) firstName = row.first_name; - if (row.last_name?.trim()) lastName = row.last_name; - } - } catch (err) { - // Fail open: DB errors should not block authenticated users - logger.warn({ err, userId: result.user.id }, 'Local user check failed — allowing request through'); + } + const activeSnapshot = lifecycle.snapshot; + if (!firstName?.trim()) { + if (activeSnapshot.first_name?.trim()) firstName = activeSnapshot.first_name; + if (activeSnapshot.last_name?.trim()) lastName = activeSnapshot.last_name; } const user: WorkOSUser = { @@ -1227,8 +1231,7 @@ export async function requireAuth(req: Request, res: Response, next: NextFunctio ); } - // Resolve identityId once before caching so cache hits inherit it. - await attachIdentityId(user); + applyActiveCredentialSnapshot(user, activeSnapshot); // Cache the validated session sessionCache.set(cacheKey, { @@ -1236,7 +1239,7 @@ export async function requireAuth(req: Request, res: Response, next: NextFunctio accessToken: result.accessToken, expiresAt: now + SESSION_CACHE_TTL_MS, newSealedSession, - authorizationFingerprint: await authorizationFingerprintFor(user), + authorizationFingerprint: activeSnapshot.fingerprint, }); req.user = user; @@ -1963,7 +1966,6 @@ export async function optionalAuth(req: Request, res: Response, next: NextFuncti logger.warn({ err: banError, userId: jwtAuth.user.id, path: req.path }, 'Ban check failed — allowing optional-auth request through'); } - await attachIdentityId(req.user); return next(); } @@ -2144,10 +2146,21 @@ export async function optionalAuth(req: Request, res: Response, next: NextFuncti ); } - // Resolve identityId before caching — sessionCache is shared with - // requireAuth, so skipping it here would let an optionalAuth request - // poison subsequent requireAuth cache hits with identityId=undefined. - await attachIdentityId(user); + // The shared cache may be consumed by requireAuth. Only an authoritative + // active local credential snapshot may create a positive entry. + const lifecycle = await readCredentialAuthorizationLifecycle(user.id); + if (lifecycle.status !== 'active') { + sessionCache.delete(cacheKey); + return next(); + } + const activeSnapshot = lifecycle.snapshot; + if (!user.firstName?.trim() && activeSnapshot.first_name?.trim()) { + user.firstName = activeSnapshot.first_name; + } + if (!user.lastName?.trim() && activeSnapshot.last_name?.trim()) { + user.lastName = activeSnapshot.last_name; + } + applyActiveCredentialSnapshot(user, activeSnapshot); // Cache the validated session sessionCache.set(cacheKey, { @@ -2155,7 +2168,7 @@ export async function optionalAuth(req: Request, res: Response, next: NextFuncti accessToken: result.accessToken, expiresAt: now + SESSION_CACHE_TTL_MS, newSealedSession, - authorizationFingerprint: await authorizationFingerprintFor(user), + authorizationFingerprint: activeSnapshot.fingerprint, }); req.user = user; diff --git a/server/src/routes/workos-webhooks.ts b/server/src/routes/workos-webhooks.ts index 5671e7da3f6..a99b32e39c1 100644 --- a/server/src/routes/workos-webhooks.ts +++ b/server/src/routes/workos-webhooks.ts @@ -19,17 +19,23 @@ */ import { Router, Request, Response } from 'express'; +import type { PoolClient } from 'pg'; import { createLogger } from '../logger.js'; import { getPool } from '../db/client.js'; import { deleteIdentityCredential } from '../services/identity-credential-deletion.js'; -import { upsertWorkosUserUnlessConfirmedDeleted } from '../db/identity-db.js'; +import { + upsertWorkosUserInCredentialEvent, + upsertWorkosUserUnlessConfirmedDeleted, + withActiveCredentialEventMutation, + withCredentialCreationEventMutation, +} from '../db/identity-db.js'; import { upsertWorkosDomain, autoPromotePrimaryIfNone, removeWorkosDomainAndReselectPrimary, } from '../db/organization-domains-db.js'; import { BrandDatabase } from '../db/brand-db.js'; -import { getWorkos, getOwnerlessPromotionWorkos } from '../auth/workos-client.js'; +import { getWorkos } from '../auth/workos-client.js'; import { invalidateUnifiedUsersCache } from '../cache/unified-users.js'; import { tryAutoLinkWebsiteUserToSlack } from '../slack/sync.js'; import { resolveUserNameWithFallbacks } from '../utils/resolve-user-name.js'; @@ -40,7 +46,6 @@ import { notifyBrandClaimOpportunity } from '../notifications/registry.js'; import { getNudgeDismissal, recordNudgeDismissal } from '../db/user-nudges-db.js'; import { getCompanyDomain } from '../utils/email-domain.js'; import { canonicalizeBrandDomain, assertClaimableBrandDomain } from '../services/identifier-normalization.js'; -import { resolvePreferredOrganization, backfillPrimaryOrganization } from '../db/users-db.js'; import { notifySystemError } from '../addie/error-notifier.js'; import { canAddSeat, type SeatType, OrganizationDatabase } from '../db/organization-db.js'; import { sendToOrgAdmins, escapeSlackMrkdwn } from '../slack/org-group-dm.js'; @@ -48,10 +53,6 @@ import { upsertOrganizationMembership, deleteOrganizationMembership, consumeInvitationSeatType, - findSuccessorForPromotion, - setMembershipRole, - autoLinkByVerifiedDomain, - resolveRoleWithWorkosFirstPromote, } from '../db/membership-db.js'; import { boundedRawJson, type RawJsonRequest } from '../middleware/bounded-raw-json.js'; @@ -81,6 +82,42 @@ interface UserData { updated_at: string; } +const PROVIDER_PREFETCH_TIMEOUT_MS = Number( + process.env.WORKOS_PROVIDER_PREFETCH_TIMEOUT_MS ?? 5_000, +); + +async function boundedProviderPrefetch<T>(operation: Promise<T>): Promise<T> { + let timeout: ReturnType<typeof setTimeout> | undefined; + try { + return await Promise.race([ + operation, + new Promise<never>((_resolve, reject) => { + timeout = setTimeout( + () => reject(new Error('Provider prefetch timed out')), + PROVIDER_PREFETCH_TIMEOUT_MS, + ); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +async function fetchMembershipUser(membership: OrganizationMembershipData): Promise<UserData> { + const workosUser = await boundedProviderPrefetch( + getWorkos().userManagement.getUser(membership.user_id), + ); + return { + id: workosUser.id, + email: workosUser.email, + first_name: workosUser.firstName, + last_name: workosUser.lastName, + email_verified: workosUser.emailVerified, + created_at: workosUser.createdAt, + updated_at: workosUser.updatedAt, + }; +} + interface OrganizationDomainData { domain: string; state: 'verified' | 'pending'; @@ -195,34 +232,17 @@ async function notifyAdminsOfRefusedMembership(input: { async function upsertMembership( membership: OrganizationMembershipData, - user?: UserData + user: UserData, + client: PoolClient, ): Promise<void> { - // If we don't have user data, fetch it from WorkOS - let userData = user; - if (!userData) { - try { - const workosUser = await getWorkos().userManagement.getUser(membership.user_id); - userData = { - id: workosUser.id, - email: workosUser.email, - first_name: workosUser.firstName, - last_name: workosUser.lastName, - email_verified: workosUser.emailVerified, - created_at: workosUser.createdAt, - updated_at: workosUser.updatedAt, - }; - } catch (error) { - logger.error({ error, userId: membership.user_id }, 'Failed to fetch user from WorkOS'); - return; - } - } + const userData = user; if (membership.status !== 'active') { logger.info( { membershipId: membership.id, status: membership.status, userId: membership.user_id, orgId: membership.organization_id }, 'Removing non-active organization membership from local cache', ); - await deleteInactiveMembershipCache(membership); + await deleteInactiveMembershipCache(membership, client); return; } @@ -232,19 +252,24 @@ async function upsertMembership( // auto-promote this user (ownerless-org safety net), push the change to // WorkOS first and only then write the resolved role locally. WorkOS is // the source of truth — local must never get ahead of it. - const resolution = await resolveRoleWithWorkosFirstPromote({ - workos: getOwnerlessPromotionWorkos(), - membershipId: membership.id, - userId: membership.user_id, - organizationId: membership.organization_id, - incomingRole, - }); + // Provider-side owner promotion cannot be made exactly-once without a + // durable event/intent journal. Preserve the signed WorkOS role locally and + // defer ownerless reconciliation rather than perform an upstream authority + // write whose outcome could become unknown during deletion. + const resolution: { role: string; promoted: boolean; promotionError?: unknown } = { + role: incomingRole, + promoted: false, + }; // Consume any pending seat_type + provisioning_source staged by the // endpoint that triggered this membership creation. Falls back to defaults // when no row was staged (e.g. someone added the membership directly in // WorkOS rather than through one of our endpoints). - const consumed = await consumeInvitationSeatType(membership.organization_id, userData.email); + const consumed = await consumeInvitationSeatType( + membership.organization_id, + userData.email, + client, + ); const hasExplicitSeatType = consumed !== null; const seatType: SeatType = consumed?.seat_type === 'contributor' ? 'contributor' : 'community_only'; const provisioningSource = consumed?.source || 'webhook'; @@ -258,7 +283,7 @@ async function upsertMembership( // and notify so a multi-user company can't squeeze onto a 1-seat // individual sub by adding members in WorkOS directly. if (!hasExplicitSeatType) { - const availability = await canAddSeat(membership.organization_id, seatType); + const availability = await canAddSeat(membership.organization_id, seatType, client); if (!availability.allowed) { logger.warn({ orgId: membership.organization_id, @@ -288,7 +313,7 @@ async function upsertMembership( seat_type: seatType, has_explicit_seat_type: hasExplicitSeatType, provisioning_source: provisioningSource, - }); + }, client); // Audit-log both outcomes of the ownerless-org auto-promote path so future // role-drift questions have a paper trail in registry_audit_log (and we @@ -340,17 +365,6 @@ async function upsertMembership( } } - // Set primary_organization_id if not already set (prefer paying orgs). - // Best-effort — same rationale as upsertUser: a transient backfill failure - // shouldn't fail the membership webhook. Integrity invariant catches drift. - try { - const preferredOrg = await resolvePreferredOrganization(membership.user_id); - if (preferredOrg) { - await backfillPrimaryOrganization(membership.user_id, preferredOrg); - } - } catch (err) { - logger.warn({ err, userId: membership.user_id }, 'primary_organization_id backfill failed during membership upsert'); - } } /** @@ -360,8 +374,15 @@ async function upsertMembership( * remove local access immediately, but it must not run owner-succession logic: * promotion is reserved for explicit organization_membership.deleted events. */ -async function deleteInactiveMembershipCache(membership: OrganizationMembershipData): Promise<void> { - const deletedRole = await deleteOrganizationMembership(membership.user_id, membership.organization_id); +async function deleteInactiveMembershipCache( + membership: OrganizationMembershipData, + client: PoolClient, +): Promise<void> { + const deletedRole = await deleteOrganizationMembership( + membership.user_id, + membership.organization_id, + client, + ); logger.info({ membershipId: membership.id, @@ -385,51 +406,17 @@ async function deleteMembership(membership: OrganizationMembershipData): Promise role: deletedRole, }, 'Deleted organization membership'); - // If an admin/owner was removed, check if the org still has one. - // Promote the longest-tenured remaining member to prevent ownerless orgs. + // Do not automatically promote a successor in WorkOS. Without a durable + // provider-intent journal, deletion can race the upstream call and leave an + // unknown authority outcome that is unsafe to replay. The signed deletion + // remains locally revoked; operators reconcile an ownerless organization + // through the explicit administration workflow. if (deletedRole === 'admin' || deletedRole === 'owner') { - try { - const target = await findSuccessorForPromotion(membership.organization_id); - if (!target) return; - - // Promote in WorkOS first, then mirror locally - let promotedInWorkos = false; - if (target.workos_membership_id) { - await getWorkos().userManagement.updateOrganizationMembership( - target.workos_membership_id, - { roleSlug: 'owner' } - ); - promotedInWorkos = true; - } else { - // No cached membership ID — look it up from WorkOS - const memberships = await getWorkos().userManagement.listOrganizationMemberships({ - organizationId: membership.organization_id, - userId: target.workos_user_id, - }); - if (memberships.data.length > 0) { - await getWorkos().userManagement.updateOrganizationMembership( - memberships.data[0].id, - { roleSlug: 'owner' } - ); - promotedInWorkos = true; - } else { - logger.warn({ - orgId: membership.organization_id, - userId: target.workos_user_id, - }, 'Successor has no WorkOS membership — cannot promote, org may be ownerless'); - } - } - if (promotedInWorkos) { - await setMembershipRole(target.workos_user_id, membership.organization_id, 'owner'); - logger.info({ - orgId: membership.organization_id, - promotedUserId: target.workos_user_id, - previousOwnerId: membership.user_id, - }, 'Promoted longest-tenured member to owner after admin/owner removal'); - } - } catch (err) { - logger.warn({ err, orgId: membership.organization_id }, 'Failed to promote successor after owner removal'); - } + logger.warn({ + orgId: membership.organization_id, + previousOwnerId: membership.user_id, + recoveryState: 'manual_owner_selection_required', + }, 'Automatic successor promotion deferred: no durable provider intent journal'); } } @@ -437,14 +424,14 @@ async function deleteMembership(membership: OrganizationMembershipData): Promise * Upsert user to local users table * Called on user.created and user.updated events */ -async function upsertUser(user: UserData): Promise<boolean> { +async function upsertUser(user: UserData, credentialClient?: PoolClient): Promise<boolean> { const pool = getPool(); const { firstName, lastName } = await resolveUserNameWithFallbacks( - pool, user.id, user.first_name, user.last_name, + credentialClient ?? pool, user.id, user.first_name, user.last_name, ); - const upserted = await upsertWorkosUserUnlessConfirmedDeleted({ + const normalizedUser = { id: user.id, email: user.email, firstName, @@ -452,7 +439,14 @@ async function upsertUser(user: UserData): Promise<boolean> { emailVerified: user.email_verified, createdAt: user.created_at, updatedAt: user.updated_at, - }); + }; + const upserted = credentialClient + ? (await upsertWorkosUserInCredentialEvent( + credentialClient, + normalizedUser, + 'provider_authoritative', + ), true) + : await upsertWorkosUserUnlessConfirmedDeleted(normalizedUser); if (!upserted) { logger.warn({ userId: user.id }, 'Ignored stale WorkOS user event after confirmed deletion'); return false; @@ -467,15 +461,6 @@ async function upsertUser(user: UserData): Promise<boolean> { // Failures here don't fail the user upsert — the integrity invariant // surfaces any pointer that doesn't get set, and the next authenticated // request opportunistically backfills via resolvePrimaryOrganization. - try { - const preferredOrg = await resolvePreferredOrganization(user.id); - if (preferredOrg) { - await backfillPrimaryOrganization(user.id, preferredOrg); - } - } catch (err) { - logger.warn({ err, userId: user.id }, 'primary_organization_id backfill failed during user upsert'); - } - logger.info({ userId: user.id, email: user.email }, 'Upserted user'); return true; } @@ -485,10 +470,8 @@ async function upsertUser(user: UserData): Promise<boolean> { * Reads the resolved name from the users table (which may have been enriched * from Slack) rather than using raw WorkOS data that might be empty. */ -async function updateUserAcrossMemberships(user: UserData): Promise<void> { - const pool = getPool(); - - const result = await pool.query( +async function updateUserAcrossMemberships(user: UserData, client: PoolClient): Promise<void> { + const result = await client.query( `UPDATE organization_memberships om SET email = $1, first_name = COALESCE(NULLIF(TRIM(u.first_name), ''), om.first_name), @@ -504,7 +487,7 @@ async function updateUserAcrossMemberships(user: UserData): Promise<void> { // person_relationships denormalizes users.email too. Keep it in sync so // the admin person-detail header doesn't lag the canonical email after a // WorkOS-side update (dashboard edit, OIDC profile sync, primary swap). - await pool.query( + await client.query( `UPDATE person_relationships SET email = $1, updated_at = NOW() WHERE workos_user_id = $2 AND email IS DISTINCT FROM $1`, @@ -920,23 +903,18 @@ export function createWorkOSWebhooksRouter(): Router { switch (event.event) { case 'organization_membership.created': { const membership = event.data as unknown as OrganizationMembershipData; - await upsertMembership(membership); + const userData = await fetchMembershipUser(membership); + const guarded = await withActiveCredentialEventMutation(membership.user_id, async (client) => { + await upsertMembership(membership, userData, client); + }); // Try to auto-link to Slack account by email (in case user.created didn't catch it) - if (membership.status === 'active') { - let workosUser: any; - try { - workosUser = await getWorkos().userManagement.getUser(membership.user_id); - } catch (error) { - logger.debug({ error, userId: membership.user_id }, 'Could not fetch user for auto-link on membership'); - } - - if (workosUser) { + if (guarded.applied && membership.status === 'active') { // Slack auto-link try { - const linkResult = await tryAutoLinkWebsiteUserToSlack(membership.user_id, workosUser.email); + const linkResult = await tryAutoLinkWebsiteUserToSlack(membership.user_id, userData.email); if (linkResult.linked) { logger.info( - { userId: membership.user_id, email: workosUser.email, slackUserId: linkResult.slack_user_id }, + { userId: membership.user_id, email: userData.email, slackUserId: linkResult.slack_user_id }, 'Auto-linked website user to Slack account on membership creation' ); } @@ -949,19 +927,22 @@ export function createWorkOSWebhooksRouter(): Router { const { matchExpectationToUser } = await import('../db/certification-db.js'); const matched = await matchExpectationToUser( membership.organization_id, - workosUser.email, + userData.email, membership.user_id ); if (matched) { logger.info( - { userId: membership.user_id, email: workosUser.email, orgId: membership.organization_id }, + { userId: membership.user_id, email: userData.email, orgId: membership.organization_id }, 'Matched certification expectation to new org member' ); } } catch (certErr) { logger.warn({ error: certErr, userId: membership.user_id }, 'Could not match cert expectation on membership'); } - } + } + if (!guarded.applied) { + logger.warn({ userId: membership.user_id }, + 'Ignored stale WorkOS membership.created event after confirmed deletion'); } invalidateUnifiedUsersCache(); break; @@ -969,7 +950,14 @@ export function createWorkOSWebhooksRouter(): Router { case 'organization_membership.updated': { const membership = event.data as unknown as OrganizationMembershipData; - await upsertMembership(membership); + const userData = await fetchMembershipUser(membership); + const guarded = await withActiveCredentialEventMutation(membership.user_id, async (client) => { + await upsertMembership(membership, userData, client); + }); + if (!guarded.applied) { + logger.warn({ userId: membership.user_id }, + 'Ignored stale WorkOS membership.updated event after confirmed deletion'); + } invalidateUnifiedUsersCache(); break; } @@ -983,10 +971,10 @@ export function createWorkOSWebhooksRouter(): Router { case 'user.created': { const user = event.data as unknown as UserData; - if (!(await upsertUser(user))) { - invalidateUnifiedUsersCache(); - break; - } + const guarded = await withCredentialCreationEventMutation(user.id, async (client) => { + await upsertUser(user, client); + }); + if (guarded.applied) { // Try to auto-link to Slack account by email const linkResult = await tryAutoLinkWebsiteUserToSlack(user.id, user.email); if (linkResult.linked) { @@ -999,18 +987,10 @@ export function createWorkOSWebhooksRouter(): Router { // Verified email is the trust gate: skip when WorkOS hasn't confirmed it yet // (the /api/me/* paths will retry after the user signs in and the email verifies). if (user.email_verified) { - try { - const linked = await autoLinkByVerifiedDomain(getWorkos(), user.id, user.email); - if (linked) { - logger.info( - { userId: user.id, email: user.email, orgId: linked.organizationId, role: linked.role }, - 'Auto-provisioned new user into verified-domain organization' - ); - } - } catch (linkErr) { - logger.warn({ err: linkErr, userId: user.id, email: user.email }, - 'Failed to auto-provision new user into verified-domain organization'); - } + logger.warn( + { userId: user.id }, + 'Verified-domain upstream auto-provision deferred: no durable exactly-once provider intent journal', + ); } // Fire-and-forget prospect triage + brand research for business emails. if (user.email) { @@ -1096,17 +1076,23 @@ export function createWorkOSWebhooksRouter(): Router { } } } + } + if (!guarded.applied) { + logger.warn({ userId: user.id }, 'Ignored stale WorkOS user.created event after confirmed deletion'); + } invalidateUnifiedUsersCache(); break; } case 'user.updated': { const user = event.data as unknown as UserData; - if (!(await upsertUser(user))) { - invalidateUnifiedUsersCache(); - break; + const guarded = await withActiveCredentialEventMutation(user.id, async (client) => { + await upsertUser(user, client); + await updateUserAcrossMemberships(user, client); + }); + if (!guarded.applied) { + logger.warn({ userId: user.id }, 'Ignored stale WorkOS user.updated event after confirmed deletion'); } - await updateUserAcrossMemberships(user); invalidateUnifiedUsersCache(); break; } @@ -1263,18 +1249,25 @@ export async function backfillOrganizationMemberships(): Promise<{ if (membership && membership.status === 'active') { seenMemberships.add(`${user.id}:${org.workos_organization_id}`); const role = membership.role?.slug || 'member'; - await upsertOrganizationMembership({ - user_id: user.id, - organization_id: org.workos_organization_id, - membership_id: membership.id, - email: user.email, - first_name: user.firstName, - last_name: user.lastName, - role, - seat_type: 'community_only', - has_explicit_seat_type: false, + const guarded = await withActiveCredentialEventMutation(user.id, async (client) => { + await upsertOrganizationMembership({ + user_id: user.id, + organization_id: org.workos_organization_id, + membership_id: membership.id, + email: user.email, + first_name: user.firstName, + last_name: user.lastName, + role, + seat_type: 'community_only', + has_explicit_seat_type: false, + }, client); }); - result.membershipsCreated++; + if (guarded.applied) { + result.membershipsCreated++; + } else { + logger.warn({ userId: user.id, orgId: org.workos_organization_id }, + 'Backfill: ignored stale membership after confirmed user deletion'); + } } } catch (memberError) { const msg = `Failed to process membership for user ${user.id}: ${memberError}`; @@ -1345,6 +1338,17 @@ export async function backfillOrganizationMemberships(): Promise<{ * any org), then lists users per org as a safety net. Removes local rows * for users confirmed deleted from WorkOS (verified via getUser). */ +export const BACKFILL_DELETION_MAX_ABSOLUTE = 100; +export const BACKFILL_DELETION_MIN_RATIO_COUNT = 10; +export const BACKFILL_DELETION_MAX_RATIO = 0.25; + +export function shouldAbortBackfillDeletion(localCount: number, candidateCount: number): boolean { + if (localCount <= 0 || candidateCount <= 0) return false; + return candidateCount > BACKFILL_DELETION_MAX_ABSOLUTE + || (candidateCount > BACKFILL_DELETION_MIN_RATIO_COUNT + && candidateCount / localCount > BACKFILL_DELETION_MAX_RATIO); +} + export async function backfillUsers(): Promise<{ usersProcessed: number; usersCreated: number; @@ -1468,36 +1472,46 @@ export async function backfillUsers(): Promise<{ logger.info({ count: candidates.length }, 'Backfill: verifying deletion candidates against WorkOS'); - // Process candidates sequentially to avoid pool connection deadlock. - // Each iteration may acquire a pool connection for the DELETE transaction, - // and the pool max is small (3). Using Promise.all with a batch size > pool - // max would cause all connections to be held simultaneously, blocking any - // new connect() calls and deadlocking. - for (const row of candidates) { - try { - // Confirm the user is actually gone from WorkOS before deleting - await getWorkos().userManagement.getUser(row.workos_user_id); - // User still exists in WorkOS — skip deletion - result.usersSkipped++; - } catch (getErr: any) { - if (getErr?.status === 404 || getErr?.code === 'entity_not_found') { - try { - const deletion = await deleteIdentityCredential(row.workos_user_id, 'sync_users_backfill'); - if (deletion.deleted) { - result.usersRemoved++; - logger.info({ userId: row.workos_user_id }, 'Backfill: removed user confirmed deleted from WorkOS'); - } else { + if (shouldAbortBackfillDeletion(localUsers.rows.length, candidates.length)) { + const message = `Deletion circuit breaker: refused ${candidates.length} candidates out of ${localUsers.rows.length} local users`; + result.errors.push(message); + result.usersSkipped += candidates.length; + logger.error({ + localUsers: localUsers.rows.length, + deletionCandidates: candidates.length, + }, 'Backfill: deletion circuit breaker refused an unsafe batch'); + } else { + // Process candidates sequentially to avoid pool connection deadlock. + // Each iteration may acquire a pool connection for the DELETE transaction, + // and the pool max is small (3). Using Promise.all with a batch size > pool + // max would cause all connections to be held simultaneously, blocking any + // new connect() calls and deadlocking. + for (const row of candidates) { + try { + // Confirm the user is actually gone from WorkOS before deleting + await getWorkos().userManagement.getUser(row.workos_user_id); + // User still exists in WorkOS — skip deletion + result.usersSkipped++; + } catch (getErr: any) { + if (getErr?.status === 404 || getErr?.code === 'entity_not_found') { + try { + const deletion = await deleteIdentityCredential(row.workos_user_id, 'sync_users_backfill'); + if (deletion.deleted) { + result.usersRemoved++; + logger.info({ userId: row.workos_user_id }, 'Backfill: removed user confirmed deleted from WorkOS'); + } else { + result.usersSkipped++; + } + } catch (err) { result.usersSkipped++; + logger.warn({ error: err, userId: row.workos_user_id }, 'Backfill: confirmed user deletion failed locally'); } - } catch (err) { + } else { + // WorkOS API error — don't delete, log full error server-side only + logger.warn({ error: getErr, userId: row.workos_user_id }, 'Backfill: WorkOS API error during user verification'); + result.errors.push(`Could not verify user ${row.workos_user_id}: WorkOS API error (status ${getErr?.status || 'unknown'})`); result.usersSkipped++; - logger.warn({ error: err, userId: row.workos_user_id }, 'Backfill: confirmed user deletion failed locally'); } - } else { - // WorkOS API error — don't delete, log full error server-side only - logger.warn({ error: getErr, userId: row.workos_user_id }, 'Backfill: WorkOS API error during user verification'); - result.errors.push(`Could not verify user ${row.workos_user_id}: WorkOS API error (status ${getErr?.status || 'unknown'})`); - result.usersSkipped++; } } } diff --git a/server/src/services/identity-credential-deletion.ts b/server/src/services/identity-credential-deletion.ts index d4486372649..21e684c87f2 100644 --- a/server/src/services/identity-credential-deletion.ts +++ b/server/src/services/identity-credential-deletion.ts @@ -30,6 +30,8 @@ export async function deleteIdentityCredential( ): Promise<IdentityCredentialDeletionResult> { const result = await deleteIdentityCredentialTransaction(workosUserId, deletionSource); + if (result.replay) return result; + invalidateSessionsForUsers(result.affectedUserIds); invalidateUnifiedUsersCache(); for (const workosUserId of result.affectedUserIds) { @@ -48,12 +50,12 @@ export async function deleteIdentityCredential( deletedWorkosUserId: quarantine.deleted_workos_user_id, recoveryState: quarantine.recovery_state, auditId: quarantine.audit_id, - }, 'Primary credential deletion requires manual recovery'); + }, 'Credential deletion left an identity without a primary'); try { notifySystemError({ source: `identity-primary-deletion-quarantine:${quarantine.identity_id}`, errorMessage: [ - `Primary credential ${quarantine.deleted_workos_user_id} was deleted without promotion.`, + `Credential ${quarantine.deleted_workos_user_id} was deleted without promotion.`, `Identity ${quarantine.identity_id} is quarantined.`, `recovery_state=${quarantine.recovery_state}.`, `audit_id=${quarantine.audit_id}.`, diff --git a/server/src/slack/sync.ts b/server/src/slack/sync.ts index 58cc55fe663..3af2586734b 100644 --- a/server/src/slack/sync.ts +++ b/server/src/slack/sync.ts @@ -550,11 +550,18 @@ export async function tryAutoLinkWebsiteUserToSlack( } // Link the accounts - await slackDb.mapUser({ + const mapped = await slackDb.mapUser({ slack_user_id: slackUser.slack_user_id, workos_user_id: workosUserId, mapping_source: 'email_auto', }); + if (!mapped) { + logger.warn( + { workosUserId, slackUserId: slackUser.slack_user_id }, + 'Skipped Slack auto-link because the local credential is not active', + ); + return { linked: false, reason: 'credential_inactive' }; + } logger.info( { workosUserId, slackUserId: slackUser.slack_user_id, email }, @@ -625,6 +632,13 @@ export async function checkAndAssignOrganizationByDomain( previousOrgName?: string; error?: string; } | null> { + logger.warn( + { workosUserId }, + 'Automatic domain organization assignment deferred: no durable provider intent journal', + ); + return null; + + /* c8 ignore start -- retained discovery for an eventual journaled workflow */ const pool = getPool(); try { @@ -721,6 +735,7 @@ export async function checkAndAssignOrganizationByDomain( error: error instanceof Error ? error.message : 'Unknown error', }; } + /* c8 ignore stop */ } /** @@ -819,6 +834,10 @@ export async function autoAddVerifiedDomainUsersAsMembers(): Promise<{ skipped: number; errors: number; }> { + logger.warn('Automatic verified-domain membership sweep deferred: no durable provider intent journal'); + return { added: 0, skipped: 0, errors: 0 }; + + /* c8 ignore start -- retained discovery for an eventual journaled workflow */ const pool = getPool(); const result = await pool.query<{ @@ -943,4 +962,5 @@ export async function autoAddVerifiedDomainUsersAsMembers(): Promise<{ } return { added: totalAdded, skipped: totalSkipped, errors: totalErrors }; + /* c8 ignore stop */ } diff --git a/server/tests/integration/admin-add-users.test.ts b/server/tests/integration/admin-add-users.test.ts index 9ca424fcb0a..946910b3d90 100644 --- a/server/tests/integration/admin-add-users.test.ts +++ b/server/tests/integration/admin-add-users.test.ts @@ -108,6 +108,11 @@ describe("admin add-users", () => { "DELETE FROM organization_memberships WHERE workos_organization_id IN ($1, $2)", [TARGET_ORG_ID, SOURCE_ORG_ID] ); + await pool.query( + "DELETE FROM identities WHERE id IN (SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1)", + [USER_ID] + ); + await pool.query("DELETE FROM users WHERE workos_user_id = $1", [USER_ID]); await pool.query( "DELETE FROM organizations WHERE workos_organization_id IN ($1, $2)", [TARGET_ORG_ID, SOURCE_ORG_ID] @@ -115,6 +120,14 @@ describe("admin add-users", () => { } async function seedSourceMembership() { + await pool.query( + `INSERT INTO users ( + workos_user_id, email, first_name, last_name, email_verified, + workos_created_at, workos_updated_at, created_at, updated_at, + primary_organization_id + ) VALUES ($1, 'user@example.test', 'Ada', 'Admin', TRUE, NOW(), NOW(), NOW(), NOW(), $2)`, + [USER_ID, SOURCE_ORG_ID] + ); await pool.query( `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, workos_membership_id, email, first_name, last_name, role, seat_type, created_at, updated_at, synced_at) diff --git a/server/tests/integration/authorization-deletion-cross-replica.test.ts b/server/tests/integration/authorization-deletion-cross-replica.test.ts new file mode 100644 index 00000000000..14ea78e8ffe --- /dev/null +++ b/server/tests/integration/authorization-deletion-cross-replica.test.ts @@ -0,0 +1,163 @@ +/** + * Exact cross-replica deletion-cache regression (#6827). + * + * Two independently loaded auth modules own distinct in-memory session caches + * and distinct PostgreSQL pools. Replica A deletes an epoch-0 credential and + * receives the local eviction. Replica B must reject its still-cached session + * on the immediate next hit by observing the durable deletion fingerprint. + */ + +import type { NextFunction, Request, Response } from 'express'; +import type { Pool } from 'pg'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const USER_ID = 'user_authz_cross_replica_deleted'; + +const mocks = vi.hoisted(() => ({ + authenticate: vi.fn(), + loadSealedSession: vi.fn(), +})); + +vi.hoisted(() => { + delete process.env.DEV_USER_EMAIL; + delete process.env.DEV_USER_ID; + process.env.WORKOS_API_KEY = process.env.WORKOS_API_KEY ?? 'sk_test'; + process.env.WORKOS_CLIENT_ID = process.env.WORKOS_CLIENT_ID ?? 'client_test'; + process.env.WORKOS_COOKIE_PASSWORD = + process.env.WORKOS_COOKIE_PASSWORD ?? 'placeholder-cookie-password-32-bytes-min'; +}); + +vi.mock('@workos-inc/node', () => ({ + WorkOS: vi.fn(function WorkOS() { + return { + userManagement: { loadSealedSession: mocks.loadSealedSession }, + apiKeys: { createValidation: vi.fn() }, + }; + }), +})); + +vi.mock('../../src/db/bans-db.js', () => ({ + bansDb: { + checkPlatformBan: vi.fn().mockResolvedValue({ banned: false, ban: null }), + checkPlatformBanForApiKey: vi.fn().mockResolvedValue({ banned: false, ban: null }), + }, +})); + +function requestFor(cookie: string): Request { + return { + headers: {}, + cookies: { 'wos-session': cookie }, + path: '/api/me', + originalUrl: '/api/me', + accepts: () => false, + } as unknown as Request; +} + +function response(): Response { + return { + cookie: vi.fn(), + status: vi.fn().mockReturnThis(), + json: vi.fn().mockReturnThis(), + redirect: vi.fn().mockReturnThis(), + } as unknown as Response; +} + +describe('durable deletion fingerprint across independent replicas', () => { + let poolA: Pool; + let closeA: () => Promise<void>; + let closeB: () => Promise<void>; + let deleteOnA: (userId: string, source: 'workos_webhook') => Promise<unknown>; + let fingerprintOnA: (userIds: string[]) => Promise<string>; + let requireAuthA: (req: Request, res: Response, next: NextFunction) => Promise<unknown>; + let requireAuthB: (req: Request, res: Response, next: NextFunction) => Promise<unknown>; + let invalidateA: (userIds: string[]) => void; + + beforeAll(async () => { + const connectionString = process.env.DATABASE_URL + || 'postgresql://adcp:localdev@localhost:5432/adcp_test'; + + vi.resetModules(); + const dbA = await import('../../src/db/client.js'); + poolA = dbA.initializeDatabase({ connectionString }); + closeA = dbA.closeDatabase; + const migrationA = await import('../../src/db/migrate.js'); + await migrationA.runMigrations(); + const identityA = await import('../../src/db/identity-db.js'); + const epochA = await import('../../src/db/authorization-epoch-db.js'); + const authA = await import('../../src/middleware/auth.js'); + deleteOnA = identityA.deleteIdentityCredentialTransaction; + fingerprintOnA = epochA.getAuthorizationFingerprint; + requireAuthA = authA.requireAuth; + invalidateA = authA.invalidateSessionsForUsers; + + vi.resetModules(); + const dbB = await import('../../src/db/client.js'); + dbB.initializeDatabase({ connectionString }); + closeB = dbB.closeDatabase; + const authB = await import('../../src/middleware/auth.js'); + requireAuthB = authB.requireAuth; + }, 60000); + + afterAll(async () => { + await poolA.query(`DELETE FROM registry_audit_log WHERE workos_user_id = $1`, [USER_ID]); + await poolA.query(`DELETE FROM users WHERE workos_user_id = $1`, [USER_ID]); + await closeB(); + await closeA(); + }); + + beforeEach(async () => { + vi.clearAllMocks(); + await poolA.query(`DELETE FROM registry_audit_log WHERE workos_user_id = $1`, [USER_ID]); + await poolA.query(`DELETE FROM users WHERE workos_user_id = $1`, [USER_ID]); + await poolA.query( + `INSERT INTO users ( + workos_user_id, email, first_name, last_name, email_verified, + workos_created_at, workos_updated_at, created_at, updated_at + ) VALUES ($1, 'cross-replica@authz.test', 'Cross', 'Replica', true, + NOW(), NOW(), NOW(), NOW())`, + [USER_ID], + ); + mocks.authenticate.mockResolvedValue({ + authenticated: true, + user: { + id: USER_ID, + email: 'cross-replica@authz.test', + firstName: 'Cross', + lastName: 'Replica', + emailVerified: true, + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + }, + accessToken: 'cross-replica-access-token', + }); + mocks.loadSealedSession.mockReturnValue({ + authenticate: mocks.authenticate, + refresh: vi.fn(), + }); + }); + + it('denies replica B immediately after replica A deletes an epoch-0 credential', async () => { + const nextA = vi.fn() as NextFunction; + const nextB = vi.fn() as NextFunction; + const cookieA = `sealed-replica-a-${Date.now()}`; + const cookieB = `sealed-replica-b-${Date.now()}`; + + expect(await fingerprintOnA([USER_ID])).toBe(''); + await requireAuthA(requestFor(cookieA), response(), nextA); + await requireAuthB(requestFor(cookieB), response(), nextB); + expect(mocks.authenticate).toHaveBeenCalledTimes(2); + + await deleteOnA(USER_ID, 'workos_webhook'); + invalidateA([USER_ID]); + expect(await fingerprintOnA([USER_ID])).toMatch( + /^user_authz_cross_replica_deleted:deleted:[0-9a-f-]{36}$/, + ); + + const replicaBResponse = response(); + await requireAuthB(requestFor(cookieB), replicaBResponse, nextB); + + expect(nextB).toHaveBeenCalledTimes(1); + expect(mocks.authenticate).toHaveBeenCalledTimes(3); + expect(replicaBResponse.status).toHaveBeenCalledWith(401); + }); +}); diff --git a/server/tests/integration/authorization-epoch.test.ts b/server/tests/integration/authorization-epoch.test.ts index ba98a9f4b7f..ff41f84a25d 100644 --- a/server/tests/integration/authorization-epoch.test.ts +++ b/server/tests/integration/authorization-epoch.test.ts @@ -20,6 +20,7 @@ import { } from '../../src/db/authorization-epoch-db.js'; import { mergeUsers } from '../../src/db/user-merge-db.js'; import { promoteSecondaryIfPrimaryDeleted } from '../../src/db/identity-db.js'; +import { deleteIdentityCredential } from '../../src/services/identity-credential-deletion.js'; const TEST_USER_PREFIX = 'user_authz_epoch_test_'; @@ -36,11 +37,13 @@ describe('Authorization epoch (migration 565)', () => { afterAll(async () => { await pool.query(`DELETE FROM users WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); + await pool.query(`DELETE FROM registry_audit_log WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); await closeDatabase(); }); beforeEach(async () => { await pool.query(`DELETE FROM users WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); + await pool.query(`DELETE FROM registry_audit_log WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); }); async function insertUser(suffix: string): Promise<string> { @@ -67,10 +70,10 @@ describe('Authorization epoch (migration 565)', () => { const userId = await insertUser('monotonic'); await bumpAuthorizationEpochs(pool, [userId]); - expect(await getAuthorizationFingerprint([userId])).toBe(`${userId}:1`); + expect(await getAuthorizationFingerprint([userId])).toBe(`${userId}:epoch:1`); await bumpAuthorizationEpochs(pool, [userId]); - expect(await getAuthorizationFingerprint([userId])).toBe(`${userId}:2`); + expect(await getAuthorizationFingerprint([userId])).toBe(`${userId}:epoch:2`); }); it('ignores credentials with no users row instead of failing the transaction', async () => { @@ -78,7 +81,7 @@ describe('Authorization epoch (migration 565)', () => { await bumpAuthorizationEpochs(pool, [userId, `${TEST_USER_PREFIX}absent`]); - expect(await getAuthorizationFingerprint([userId])).toBe(`${userId}:1`); + expect(await getAuthorizationFingerprint([userId])).toBe(`${userId}:epoch:1`); }); it('changes the fingerprint when a bumped credential is deleted', async () => { @@ -93,6 +96,17 @@ describe('Authorization epoch (migration 565)', () => { expect(after).toBe(''); }); + it('retains a deletion fingerprint for an epoch-0 credential after users-row cascade', async () => { + const userId = await insertUser('epoch_zero_confirmed_delete'); + expect(await getAuthorizationFingerprint([userId])).toBe(''); + + await expect(deleteIdentityCredential(userId, 'workos_webhook')) + .resolves.toMatchObject({ deleted: true }); + + expect(await getAuthorizationFingerprint([userId])) + .toMatch(new RegExp(`^${userId}:deleted:[0-9a-f-]+$`)); + }); + it('preserves both fingerprints when generic merging is refused', async () => { const primaryId = await insertUser('merge_primary'); const secondaryId = await insertUser('merge_secondary'); diff --git a/server/tests/integration/mcp-oauth-deletion-race.test.ts b/server/tests/integration/mcp-oauth-deletion-race.test.ts new file mode 100644 index 00000000000..8f1eba9454e --- /dev/null +++ b/server/tests/integration/mcp-oauth-deletion-race.test.ts @@ -0,0 +1,114 @@ +import type { Response } from 'express'; +import type { Pool } from 'pg'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const USER_ID = 'user_mcp_oauth_deletion_race'; +const PENDING_ID = 'pending_mcp_oauth_deletion_race'; + +const mocks = vi.hoisted(() => ({ + authenticateWithCodeForTokens: vi.fn(), +})); + +vi.mock('../../src/auth/workos-client.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/auth/workos-client.js')>()), + authenticateWithCodeForTokens: mocks.authenticateWithCodeForTokens, +})); + +const { initializeDatabase, closeDatabase } = await import('../../src/db/client.js'); +const { runMigrations } = await import('../../src/db/migrate.js'); +const mcpOAuthStateDb = await import('../../src/db/mcp-oauth-state-db.js'); +const { deleteIdentityCredential } = await import('../../src/services/identity-credential-deletion.js'); +const { handleMCPOAuthCallback } = await import('../../src/mcp/oauth-provider.js'); + +describe('MCP OAuth callback vs confirmed credential deletion', () => { + let pool: Pool; + + beforeAll(async () => { + pool = initializeDatabase({ + connectionString: process.env.DATABASE_URL + || 'postgresql://adcp:localdev@localhost:5432/adcp_test', + }); + await runMigrations(); + }, 60_000); + + afterAll(async () => { + await cleanup(); + await closeDatabase(); + }); + + beforeEach(async () => { + vi.clearAllMocks(); + await cleanup(); + await pool.query( + `INSERT INTO users ( + workos_user_id, email, first_name, last_name, email_verified, + workos_created_at, workos_updated_at, created_at, updated_at + ) VALUES ($1, 'mcp-race@example.test', 'MCP', 'Race', TRUE, + NOW(), NOW(), NOW(), NOW())`, + [USER_ID], + ); + await mcpOAuthStateDb.setPendingAuth(PENDING_ID, { + clientId: 'client_mcp_race', + redirectUri: 'https://client.example.test/callback', + codeChallenge: 'challenge', + scopes: ['openid'], + state: 'roundtrip-state', + }); + }); + + async function cleanup() { + if (!pool) return; + await pool.query(`DELETE FROM mcp_oauth_pending_auths WHERE id = $1`, [PENDING_ID]); + await pool.query( + `DELETE FROM mcp_oauth_auth_codes + WHERE data->>'clientId' = 'client_mcp_race'`, + ); + await pool.query(`DELETE FROM registry_audit_log WHERE workos_user_id = $1`, [USER_ID]); + await pool.query(`DELETE FROM users WHERE workos_user_id = $1`, [USER_ID]); + } + + it('issues no local code when deletion commits after exchange starts but before local finalize', async () => { + let exchangeEntered!: () => void; + let releaseExchange!: () => void; + const entered = new Promise<void>((resolve) => { exchangeEntered = resolve; }); + const release = new Promise<void>((resolve) => { releaseExchange = resolve; }); + mocks.authenticateWithCodeForTokens.mockImplementation(async () => { + exchangeEntered(); + await release; + return { + user: { + id: USER_ID, + email: 'mcp-race@example.test', + firstName: 'MCP', + lastName: 'Race', + emailVerified: true, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + accessToken: 'access-token', + refreshToken: 'refresh-token', + }; + }); + const res = { + status: vi.fn().mockReturnThis(), + json: vi.fn().mockReturnThis(), + redirect: vi.fn().mockReturnThis(), + } as unknown as Response; + + const callback = handleMCPOAuthCallback({} as never, res, 'workos-code', PENDING_ID); + await entered; + await deleteIdentityCredential(USER_ID, 'workos_webhook'); + releaseExchange(); + await callback; + + expect(res.redirect).toHaveBeenCalledTimes(1); + const redirect = new URL(vi.mocked(res.redirect).mock.calls[0][0] as string); + expect(redirect.searchParams.get('error')).toBe('access_denied'); + expect(redirect.searchParams.has('code')).toBe(false); + expect((await pool.query( + `SELECT 1 FROM mcp_oauth_auth_codes WHERE data->>'clientId' = 'client_mcp_race'`, + )).rows).toEqual([]); + expect((await pool.query(`SELECT 1 FROM users WHERE workos_user_id = $1`, [USER_ID])).rows) + .toEqual([]); + }); +}); diff --git a/server/tests/integration/membership-webhook.test.ts b/server/tests/integration/membership-webhook.test.ts index b5b4161691b..fe9505e71bf 100644 --- a/server/tests/integration/membership-webhook.test.ts +++ b/server/tests/integration/membership-webhook.test.ts @@ -37,11 +37,19 @@ describe('Membership webhook DB operations', () => { connectionString: process.env.DATABASE_URL || 'postgresql://adcp:localdev@localhost:5432/adcp_test', }); await runMigrations(); + await pool.query( + `INSERT INTO users (workos_user_id, email, created_at, updated_at) + VALUES ($1, 'one@membership.test', NOW(), NOW()), + ($2, 'two@membership.test', NOW(), NOW()) + ON CONFLICT (workos_user_id) DO NOTHING`, + [TEST_USER_1, TEST_USER_2], + ); }, 60000); afterAll(async () => { await pool.query('DELETE FROM invitation_seat_types WHERE workos_organization_id = $1', [TEST_ORG_ID]); await pool.query('DELETE FROM organization_memberships WHERE workos_organization_id = $1', [TEST_ORG_ID]); + await pool.query('DELETE FROM users WHERE workos_user_id = ANY($1)', [[TEST_USER_1, TEST_USER_2]]); await closeDatabase(); }); @@ -267,8 +275,8 @@ describe('Membership webhook DB operations', () => { expect(updateOrganizationMembership).not.toHaveBeenCalled(); }); - it('promotes to owner in WorkOS first when the org has no other admin/owner', async () => { - const { client, updateOrganizationMembership } = makeWorkos({ + it('defers ownerless promotion when the org has no other admin/owner', async () => { + const { client, updateOrganizationMembership, listOrganizationMemberships } = makeWorkos({ memberships: [{ userId: TEST_USER_1, role: { slug: 'member' } }], }); @@ -280,12 +288,13 @@ describe('Membership webhook DB operations', () => { incomingRole: 'member', }); - expect(result).toEqual({ role: 'owner', promoted: true }); - expect(updateOrganizationMembership).toHaveBeenCalledWith('om_target', { roleSlug: 'owner' }); + expect(result).toEqual({ role: 'member', promoted: false }); + expect(listOrganizationMemberships).not.toHaveBeenCalled(); + expect(updateOrganizationMembership).not.toHaveBeenCalled(); }); it('leaves role as member when another admin already exists in WorkOS', async () => { - const { client, updateOrganizationMembership } = makeWorkos({ + const { client, updateOrganizationMembership, listOrganizationMemberships } = makeWorkos({ memberships: [ { userId: TEST_USER_2, role: { slug: 'admin' } }, { userId: TEST_USER_1, role: { slug: 'member' } }, @@ -301,11 +310,12 @@ describe('Membership webhook DB operations', () => { }); expect(result).toEqual({ role: 'member', promoted: false }); + expect(listOrganizationMemberships).not.toHaveBeenCalled(); expect(updateOrganizationMembership).not.toHaveBeenCalled(); }); - it('falls back to member when the WorkOS update fails (no drift)', async () => { - const { client } = makeWorkos({ + it('does not attempt an owner update even when the client would fail', async () => { + const { client, updateOrganizationMembership } = makeWorkos({ memberships: [{ userId: TEST_USER_1, role: { slug: 'member' } }], updateFails: true, }); @@ -320,11 +330,12 @@ describe('Membership webhook DB operations', () => { expect(result.role).toBe('member'); expect(result.promoted).toBe(false); - expect(result.promotionError).toBeDefined(); + expect(result.promotionError).toBeUndefined(); + expect(updateOrganizationMembership).not.toHaveBeenCalled(); }); - it('falls back to member when WorkOS membership list fails (refuses to promote blind)', async () => { - const { client, updateOrganizationMembership } = makeWorkos({ + it('does not list provider memberships while promotion is deferred', async () => { + const { client, updateOrganizationMembership, listOrganizationMemberships } = makeWorkos({ memberships: [], listFails: true, }); @@ -339,7 +350,8 @@ describe('Membership webhook DB operations', () => { expect(result.role).toBe('member'); expect(result.promoted).toBe(false); - expect(result.promotionError).toBeDefined(); + expect(result.promotionError).toBeUndefined(); + expect(listOrganizationMemberships).not.toHaveBeenCalled(); expect(updateOrganizationMembership).not.toHaveBeenCalled(); }); }); @@ -547,24 +559,17 @@ describe('Membership webhook DB operations', () => { ); } - it('always creates membership as member; upsert path handles auto-promotion atomically', async () => { + it('defers automatic direct-domain provider authority without an intent journal', async () => { await seedOrgWithVerifiedDomain('autolink.com'); const workos = makeWorkOSMock(); const result = await autoLinkByVerifiedDomain(workos, AUTOLINK_USER, 'matt@autolink.com'); - expect(result).not.toBeNull(); - expect(result!.organizationId).toBe(TEST_AUTOLINK_ORG_ID); - expect(result!.organizationName).toBe('AutoLink Corp'); - expect(result!.role).toBe('member'); - expect(workos.userManagement.createOrganizationMembership).toHaveBeenCalledWith({ - userId: AUTOLINK_USER, - organizationId: TEST_AUTOLINK_ORG_ID, - roleSlug: 'member', - }); + expect(result).toBeNull(); + expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); }); - it('still creates as member when org already has an admin', async () => { + it('defers automatic provider authority even when the org already has an admin', async () => { await seedOrgWithVerifiedDomain('autolink.com'); await pool.query( `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role, seat_type, created_at, updated_at, synced_at) @@ -575,10 +580,8 @@ describe('Membership webhook DB operations', () => { const result = await autoLinkByVerifiedDomain(workos, AUTOLINK_USER, 'matt@autolink.com'); - expect(result).not.toBeNull(); - expect(workos.userManagement.createOrganizationMembership).toHaveBeenCalledWith( - expect.objectContaining({ roleSlug: 'member' }), - ); + expect(result).toBeNull(); + expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); }); it('returns null when no matching verified domain exists', async () => { @@ -623,13 +626,13 @@ describe('Membership webhook DB operations', () => { expect(result).toBeNull(); }); - it('handles membership_already_exists gracefully', async () => { + it('does not call WorkOS even when an already-exists response would have been safe to retry', async () => { await seedOrgWithVerifiedDomain('autolink.com'); const workos = makeWorkOSMock({ shouldFail: true, errorCode: 'organization_membership_already_exists' }); const result = await autoLinkByVerifiedDomain(workos, AUTOLINK_USER, 'matt@autolink.com'); - expect(result).not.toBeNull(); - expect(result!.organizationId).toBe(TEST_AUTOLINK_ORG_ID); + expect(result).toBeNull(); + expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); }); it('returns null on other WorkOS errors', async () => { @@ -676,7 +679,7 @@ describe('Membership webhook DB operations', () => { expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); }); - it('still creates membership when user has memberships in OTHER orgs (the Triton case)', async () => { + it('defers automatic authority when the user has memberships in other orgs', async () => { await seedOrgWithVerifiedDomain('autolink.com'); // Seed an unrelated personal org membership for the user. await pool.query( @@ -689,14 +692,8 @@ describe('Membership webhook DB operations', () => { const result = await autoLinkByVerifiedDomain(workos, AUTOLINK_USER, 'matt@autolink.com'); - expect(result).not.toBeNull(); - expect(result!.organizationId).toBe(TEST_AUTOLINK_ORG_ID); - expect(workos.userManagement.createOrganizationMembership).toHaveBeenCalledWith( - expect.objectContaining({ - userId: AUTOLINK_USER, - organizationId: TEST_AUTOLINK_ORG_ID, - }), - ); + expect(result).toBeNull(); + expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); // Cleanup the personal-org seed await pool.query( @@ -704,12 +701,12 @@ describe('Membership webhook DB operations', () => { ); }); - it('stages provisioning_source=verified_domain so the webhook can record it', async () => { + it('does not stage a membership when automatic provider authority is deferred', async () => { await seedOrgWithVerifiedDomain('autolink.com'); const workos = makeWorkOSMock(); const result = await autoLinkByVerifiedDomain(workos, AUTOLINK_USER, 'matt@autolink.com'); - expect(result).not.toBeNull(); + expect(result).toBeNull(); // The staging row should now exist for the org+email pair so the // organization_membership.created webhook can consume it. @@ -718,9 +715,7 @@ describe('Membership webhook DB operations', () => { WHERE workos_organization_id = $1 AND lower(email) = lower($2)`, [TEST_AUTOLINK_ORG_ID, 'matt@autolink.com'], ); - expect(staged.rows[0]).toBeDefined(); - expect(staged.rows[0].seat_type).toBe('community_only'); - expect(staged.rows[0].source).toBe('verified_domain'); + expect(staged.rows).toEqual([]); }); // ───────────────────────────────────────────────────────────────── @@ -776,19 +771,14 @@ describe('Membership webhook DB operations', () => { await clearBrandHierarchy(); }); - it('links @child.com user to parent when org has opted into hierarchical auto-provisioning', async () => { + it('defers hierarchical provider authority even when the org opted in', async () => { await seedBrandHierarchy(); const workos = makeWorkOSMock(); const result = await autoLinkByVerifiedDomain(workos, AUTOLINK_USER, `mike@${CHILD_DOMAIN}`); - expect(result).not.toBeNull(); - expect(result!.organizationId).toBe(TEST_AUTOLINK_ORG_ID); - expect(workos.userManagement.createOrganizationMembership).toHaveBeenCalledWith({ - userId: AUTOLINK_USER, - organizationId: TEST_AUTOLINK_ORG_ID, - roleSlug: 'member', - }); + expect(result).toBeNull(); + expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); }); it('does NOT auto-link via hierarchy by default (auto_provision_brand_hierarchy_children = false)', async () => { @@ -830,7 +820,7 @@ describe('Membership webhook DB operations', () => { expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); }); - it('honors direct auto-provisioning opt-out independently for direct matches', async () => { + it('defers hierarchy authority independently of the direct-domain flag', async () => { // Direct=false (opt-out), hierarchy=true. A user matching the // child via hierarchy still gets in. await seedBrandHierarchy(); @@ -841,9 +831,9 @@ describe('Membership webhook DB operations', () => { ); const workos = makeWorkOSMock(); - // Hierarchical match still works. const inheritedResult = await autoLinkByVerifiedDomain(workos, AUTOLINK_USER, `mike@${CHILD_DOMAIN}`); - expect(inheritedResult).not.toBeNull(); + expect(inheritedResult).toBeNull(); + expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); }); it('short-circuits when the user is already in the resolved parent org', async () => { @@ -893,7 +883,7 @@ describe('Membership webhook DB operations', () => { await pool.query('DELETE FROM users WHERE workos_user_id = $1', [AUTOLINK_USER]); }); - it('cohort: still auto-links a NEW user (created after flag flip) via hierarchy', async () => { + it('cohort: defers a new user created after the hierarchy flag flip', async () => { await seedBrandHierarchy({ hierarchyOptIn: false }); // start opted out // Flip flag at T0 (sets enabled_at = NOW()). await pool.query( @@ -912,13 +902,13 @@ describe('Membership webhook DB operations', () => { const workos = makeWorkOSMock(); const result = await autoLinkByVerifiedDomain(workos, AUTOLINK_USER, `mike@${CHILD_DOMAIN}`); - expect(result).not.toBeNull(); - expect(result!.organizationId).toBe(TEST_AUTOLINK_ORG_ID); + expect(result).toBeNull(); + expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); await pool.query('DELETE FROM users WHERE workos_user_id = $1', [AUTOLINK_USER]); }); - it('cohort: auto-links when no users row exists yet (just-created via webhook)', async () => { + it('cohort: does not treat a missing user row as permission to auto-link', async () => { // Webhook race: autoLink fires before user.created webhook lands the // local users row. Don't block on a missing row — treat as new joiner. await seedBrandHierarchy(); // hierarchyOptIn defaults to true @@ -929,11 +919,11 @@ describe('Membership webhook DB operations', () => { const result = await autoLinkByVerifiedDomain(workos, AUTOLINK_USER, `mike@${CHILD_DOMAIN}`); - expect(result).not.toBeNull(); - expect(result!.organizationId).toBe(TEST_AUTOLINK_ORG_ID); + expect(result).toBeNull(); + expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); }); - it('prefers a direct verified-domain match over a hierarchical one when both exist', async () => { + it('does not choose a direct or hierarchical target while automatic writes are deferred', async () => { // Direct match wins: even if the brand registry says child→parent, // an explicit organization_domains row on the child takes priority. await seedBrandHierarchy(); @@ -955,8 +945,8 @@ describe('Membership webhook DB operations', () => { const workos = makeWorkOSMock(); const result = await autoLinkByVerifiedDomain(workos, AUTOLINK_USER, `mike@${CHILD_DOMAIN}`); - expect(result).not.toBeNull(); - expect(result!.organizationId).toBe(DIRECT_ORG); + expect(result).toBeNull(); + expect(workos.userManagement.createOrganizationMembership).not.toHaveBeenCalled(); // Cleanup await pool.query('DELETE FROM organization_domains WHERE workos_organization_id = $1', [DIRECT_ORG]); diff --git a/server/tests/integration/workos-user-deleted-primary.test.ts b/server/tests/integration/workos-user-deleted-primary.test.ts index 17e615fe4df..756248608e2 100644 --- a/server/tests/integration/workos-user-deleted-primary.test.ts +++ b/server/tests/integration/workos-user-deleted-primary.test.ts @@ -15,11 +15,19 @@ import { initializeDatabase, closeDatabase } from '../../src/db/client.js'; import { runMigrations } from '../../src/db/migrate.js'; import { getIdentityRecoveryQuarantine, + IDENTITY_DELETION_SNAPSHOT_MAX_DETAILS_BYTES, + IDENTITY_DELETION_SNAPSHOT_POLICY, IDENTITY_RECOVERY_STATE, promoteSecondaryIfPrimaryDeleted, + USER_CASCADE_SNAPSHOT_INVENTORY, + USER_SET_NULL_SNAPSHOT_INVENTORY, } from '../../src/db/identity-db.js'; import { deleteIdentityCredential } from '../../src/services/identity-credential-deletion.js'; -import { bumpAuthorizationEpochs, getAuthorizationFingerprint } from '../../src/db/authorization-epoch-db.js'; +import { + bumpAuthorizationEpochs, + getAuthorizationFingerprint, + readCredentialAuthorizationLifecycle, +} from '../../src/db/authorization-epoch-db.js'; import { notifySystemError } from '../../src/addie/error-notifier.js'; import { isSlackUserAAOAdmin } from '../../src/addie/mcp/admin-tools.js'; import { @@ -31,6 +39,7 @@ import type { Pool } from 'pg'; const TEST_USER_PREFIX = 'user_wh_deleted_test_'; const TEST_SLACK_PREFIX = 'slack_wh_deleted_test_'; const TEST_ORG = 'org_wh_deleted_test_pinnacle'; +const TEST_ORG_PREFIX = 'org_wh_deleted_test_'; describe('user.deleted: automatic promotion containment (#6827)', () => { let pool: Pool; @@ -55,6 +64,7 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { }); async function cleanup() { + await pool.query(`DELETE FROM aao_admin_access_events WHERE target_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); await pool.query( `DELETE FROM registry_audit_log WHERE action IN ('identity_primary_deletion_quarantined', 'identity_credential_deleted') @@ -62,11 +72,23 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { [`${TEST_USER_PREFIX}%`], ); await pool.query(`DELETE FROM organization_memberships WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); - await pool.query(`DELETE FROM working_group_memberships WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); - await pool.query(`DELETE FROM working_group_leaders WHERE user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); + await pool.query( + `DELETE FROM working_group_memberships + WHERE workos_user_id LIKE $1 OR workos_user_id LIKE $2`, + [`${TEST_USER_PREFIX}%`, `${TEST_SLACK_PREFIX}%`], + ); + await pool.query( + `DELETE FROM working_group_leaders WHERE user_id LIKE $1 OR user_id LIKE $2`, + [`${TEST_USER_PREFIX}%`, `${TEST_SLACK_PREFIX}%`], + ); + await pool.query( + `DELETE FROM working_group_topic_subscriptions + WHERE workos_user_id LIKE $1 OR workos_user_id LIKE $2`, + [`${TEST_USER_PREFIX}%`, `${TEST_SLACK_PREFIX}%`], + ); await pool.query(`DELETE FROM slack_user_mappings WHERE slack_user_id LIKE $1`, [`${TEST_SLACK_PREFIX}%`]); await pool.query(`DELETE FROM users WHERE workos_user_id LIKE $1`, [`${TEST_USER_PREFIX}%`]); - await pool.query(`DELETE FROM organizations WHERE workos_organization_id = $1`, [TEST_ORG]); + await pool.query(`DELETE FROM organizations WHERE workos_organization_id LIKE $1`, [`${TEST_ORG_PREFIX}%`]); vi.mocked(notifySystemError).mockClear(); } @@ -175,8 +197,11 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { const bystander = await insertUser('revoke_bystander', 'maya-revoke@pinnacle.example'); const identityId = await bindAsSecondary(primary, secondary, 1); await bindAsSecondary(primary, bystander, 2); + const grantOrganizations = [TEST_ORG, `${TEST_ORG_PREFIX}revoked`, `${TEST_ORG_PREFIX}expired`, `${TEST_ORG_PREFIX}future`]; await pool.query( - `INSERT INTO organizations (workos_organization_id, name) VALUES ($1, 'Pinnacle Agency')`, [TEST_ORG], + `INSERT INTO organizations (workos_organization_id, name) + SELECT id, 'Deletion snapshot ' || id FROM unnest($1::text[]) AS id`, + [grantOrganizations], ); await pool.query( `INSERT INTO organization_memberships (workos_user_id, workos_organization_id, email, role) @@ -209,7 +234,37 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { [slackUserId, primary], ); await bumpAuthorizationEpochs(pool, [primary, secondary, bystander]); - return { primary, secondary, bystander, identityId, slackUserId }; + await pool.query( + `INSERT INTO organization_credential_grants ( + workos_organization_id, workos_user_id, role, + granted_by_workos_user_id, reason, effective_from, effective_until, + revoked_at, revoked_by_workos_user_id + ) VALUES + ($1, $5, 'owner', 'approver_active', 'active deletion grant', NOW() - INTERVAL '1 day', NULL, NULL, NULL), + ($2, $5, 'admin', 'approver_revoked', 'revoked deletion grant', NOW() - INTERVAL '3 days', NULL, NOW() - INTERVAL '1 day', 'revoker_1'), + ($3, $5, 'member', 'approver_expired', 'expired deletion grant', NOW() - INTERVAL '3 days', NOW() - INTERVAL '1 day', NULL, NULL), + ($4, $5, 'admin', 'approver_future', 'future deletion grant', NOW() + INTERVAL '1 day', NOW() + INTERVAL '3 days', NULL, NULL)`, + [...grantOrganizations, primary], + ); + const aaoAdminGroup = await pool.query<{ id: string }>( + `SELECT id FROM working_groups WHERE slug = 'aao-admin'`, + ); + await pool.query( + `INSERT INTO aao_admin_access_events ( + event_type, actor_user_id, target_user_id, mechanism, + actor_authorization_mechanism, reason + ) VALUES ('granted', 'admin_seed', $1, 'aao_admin_working_group', + 'static_admin_api_key', 'seed active authority before provider deletion')`, + [primary], + ); + return { + primary, + secondary, + bystander, + identityId, + slackUserId, + aaoAdminGroupId: aaoAdminGroup.rows[0].id, + }; } async function survivorAuthority(ids: string[]) { @@ -219,6 +274,7 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { ['organization_memberships', 'workos_user_id'], ['working_group_memberships', 'workos_user_id'], ['working_group_leaders', 'user_id'], + ['working_group_topic_subscriptions', 'workos_user_id'], ['slack_user_mappings', 'workos_user_id'], ]; const state: Record<string, unknown> = {}; @@ -262,6 +318,12 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { WHERE wgl.user_id = ANY($1) ORDER BY wgl.user_id, wgl.working_group_id`, [ids], ], + working_group_topic_subscriptions: [ + `SELECT to_jsonb(wgts) AS row FROM working_group_topic_subscriptions wgts + WHERE wgts.workos_user_id = ANY($1) + ORDER BY wgts.workos_user_id, wgts.working_group_id`, + [ids], + ], slack_user_mappings: [ `SELECT to_jsonb(sm) AS row FROM slack_user_mappings sm WHERE sm.workos_user_id = ANY($1) @@ -278,11 +340,30 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { const query = await pool.query(sql, params); return [key, key === 'identity' ? query.rows[0].row : query.rows.map(({ row }) => row)]; })); - return Object.fromEntries(entries); + const snapshotInventory = async ( + specs: readonly { readonly table: string; readonly columns: readonly string[] }[], + ) => Object.fromEntries(await Promise.all(specs.map(async ({ table, columns }) => { + const predicates = columns.map((column) => `child."${column}" = ANY($1)`).join(' OR '); + const rows = await pool.query( + `SELECT to_jsonb(child) AS row FROM "${table}" child + WHERE ${predicates} ORDER BY to_jsonb(child)::text`, + [ids], + ); + return [table, rows.rows.map(({ row }) => row)]; + }))); + const baseGraph = Object.fromEntries(entries); + const cascadeRows = await snapshotInventory(USER_CASCADE_SNAPSHOT_INVENTORY); + cascadeRows.identity_workos_users = baseGraph.identity_workos_users; + cascadeRows.authorization_epochs = baseGraph.authorization_epochs; + return { + ...baseGraph, + user_cascade_rows: cascadeRows, + user_set_null_attributions: await snapshotInventory(USER_SET_NULL_SNAPSHOT_INVENTORY), + }; } it('serializes webhook-vs-backfill deletion, revokes caches, and records the exact recovery graph once', async () => { - const { primary, secondary, bystander, identityId, slackUserId } = await seedDeletionAuthority(); + const { primary, secondary, bystander, identityId, slackUserId, aaoAdminGroupId } = await seedDeletionAuthority(); const survivors = [secondary, bystander]; const allCredentials = [primary, secondary, bystander].sort(); const authorityBefore = await survivorAuthority(survivors); @@ -303,6 +384,14 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { ]); expect(results.filter(({ deleted }) => deleted)).toHaveLength(1); + const conflictingConcurrentReplay = results.find(({ replay }) => replay !== null); + expect(conflictingConcurrentReplay?.replay).toMatchObject({ + kind: 'different_source', + requested_deletion_source: expect.stringMatching(/^(workos_webhook|sync_users_backfill)$/), + original_deletion_source: expect.stringMatching(/^(workos_webhook|sync_users_backfill)$/), + }); + expect(conflictingConcurrentReplay?.replay?.requested_deletion_source) + .not.toBe(conflictingConcurrentReplay?.replay?.original_deletion_source); expect(results.flatMap(({ affectedUserIds }) => affectedUserIds)).toEqual( expect.arrayContaining([primary, secondary, bystander]), ); @@ -318,13 +407,14 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { )).rows).toEqual([]); const auditRows = await pool.query( - `SELECT id, details FROM registry_audit_log - WHERE action = 'identity_primary_deletion_quarantined' - AND resource_type = 'identity_recovery' - AND resource_id = $1`, - [identityId], + `SELECT id, action, details FROM registry_audit_log + WHERE workos_user_id = $1 + AND action IN ('identity_primary_deletion_quarantined', 'identity_credential_deleted') + ORDER BY created_at, id`, + [primary], ); expect(auditRows.rows).toHaveLength(1); + expect(auditRows.rows[0].action).toBe('identity_primary_deletion_quarantined'); const durable = await getIdentityRecoveryQuarantine(identityId); expect(durable).toEqual({ audit_id: auditRows.rows[0].id, @@ -337,13 +427,27 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { workos_user_id: primary, }, recovery_state: IDENTITY_RECOVERY_STATE, + snapshot_policy: IDENTITY_DELETION_SNAPSHOT_POLICY, before_graph: beforeGraph, + aao_admin_revocations: [{ + event_type: 'revoked', + target_user_id: primary, + working_group_id: aaoAdminGroupId, + mechanism: 'aao_admin_working_group', + actor: { + type: 'workos_provider', + source: expect.stringMatching(/^(workos_webhook|sync_users_backfill)$/), + workos_user_id: primary, + }, + evidence_ledger: 'registry_audit_log', + dedicated_ledger_limitation: 'provider_deletion_actor_not_representable', + }], }); expect(notifySystemError).toHaveBeenCalledTimes(1); expect(notifySystemError).toHaveBeenCalledWith({ source: `identity-primary-deletion-quarantine:${identityId}`, errorMessage: [ - `Primary credential ${primary} was deleted without promotion.`, + `Credential ${primary} was deleted without promotion.`, `Identity ${identityId} is quarantined.`, `recovery_state=${IDENTITY_RECOVERY_STATE}.`, `audit_id=${auditRows.rows[0].id}.`, @@ -362,16 +466,83 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { [slackUserId], )).rows).toEqual([{ workos_user_id: null, mapping_status: 'unmapped' }]); expect(await isSlackUserAAOAdmin(slackUserId)).toBe(false); + expect(beforeGraph.user_cascade_rows.organization_credential_grants).toHaveLength(4); + expect(beforeGraph.user_cascade_rows.organization_credential_grants.map((row: any) => ({ + role: row.role, + granted_by: row.granted_by_workos_user_id, + reason: row.reason, + effective_from: row.effective_from, + effective_until: row.effective_until, + revoked_at: row.revoked_at, + revoked_by: row.revoked_by_workos_user_id, + }))).toEqual(expect.arrayContaining([ + expect.objectContaining({ role: 'owner', granted_by: 'approver_active', reason: 'active deletion grant', revoked_at: null }), + expect.objectContaining({ role: 'admin', granted_by: 'approver_revoked', reason: 'revoked deletion grant', revoked_by: 'revoker_1' }), + expect.objectContaining({ role: 'member', granted_by: 'approver_expired', reason: 'expired deletion grant' }), + expect.objectContaining({ role: 'admin', granted_by: 'approver_future', reason: 'future deletion grant' }), + ])); + expect((await pool.query( + `SELECT event_type FROM aao_admin_access_events WHERE target_user_id = $1 ORDER BY created_at`, + [primary], + )).rows).toEqual([{ event_type: 'granted' }]); const fingerprintAfter = await getAuthorizationFingerprint(survivors); - await deleteIdentityCredential(primary, 'workos_webhook'); + const allAuditRowsBeforeReplay = (await pool.query( + `SELECT * FROM registry_audit_log WHERE workos_user_id = $1 ORDER BY created_at, id`, + [primary], + )).rows; + const epochRowsBeforeReplay = (await pool.query( + `SELECT * FROM authorization_epochs WHERE workos_user_id = ANY($1) ORDER BY workos_user_id`, + [allCredentials], + )).rows; + const replayCache = new Map([[TEST_ORG, [{ + id: primary, + email: 'must-remain-cached@test.example', + firstName: 'Replay', + lastName: 'Sentinel', + }]]]); + setUnifiedUsersCache(replayCache); + const originalSource = durable!.deletion_source; + const sameSourceReplay = await deleteIdentityCredential(primary, originalSource); + expect(sameSourceReplay).toMatchObject({ + deleted: false, + affectedUserIds: allCredentials, + affectedSlackUserIds: [slackUserId], + quarantine: durable, + replay: { + audit_id: auditRows.rows[0].id, + kind: 'same_source', + original_deletion_source: originalSource, + requested_deletion_source: originalSource, + }, + }); + const conflictingSource = originalSource === 'workos_webhook' + ? 'sync_users_backfill' + : 'workos_webhook'; + const conflictingReplay = await deleteIdentityCredential(primary, conflictingSource); + expect(conflictingReplay).toMatchObject({ + deleted: false, + affectedUserIds: allCredentials, + affectedSlackUserIds: [slackUserId], + quarantine: durable, + replay: { + audit_id: auditRows.rows[0].id, + kind: 'different_source', + original_deletion_source: originalSource, + requested_deletion_source: conflictingSource, + }, + }); expect(await getAuthorizationFingerprint(survivors)).toBe(fingerprintAfter); expect(await survivorAuthority(survivors)).toEqual(authorityBefore); expect((await pool.query( - `SELECT 1 FROM registry_audit_log - WHERE action = 'identity_primary_deletion_quarantined' AND resource_id = $1`, - [identityId], - )).rows).toHaveLength(1); + `SELECT * FROM registry_audit_log WHERE workos_user_id = $1 ORDER BY created_at, id`, + [primary], + )).rows).toEqual(allAuditRowsBeforeReplay); + expect((await pool.query( + `SELECT * FROM authorization_epochs WHERE workos_user_id = ANY($1) ORDER BY workos_user_id`, + [allCredentials], + )).rows).toEqual(epochRowsBeforeReplay); + expect(getUnifiedUsersCache()).toBe(replayCache); expect(notifySystemError).toHaveBeenCalledTimes(1); }); @@ -387,9 +558,221 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { WHERE identity_id = $1 AND is_primary = TRUE`, [identityId], )).rows).toEqual([{ workos_user_id: primary }]); + const auditRowsAfterDeletion = (await pool.query( + `SELECT * FROM registry_audit_log WHERE workos_user_id = $1 ORDER BY created_at, id`, + [secondary], + )).rows; + const epochRowsAfterDeletion = (await pool.query( + `SELECT * FROM authorization_epochs + WHERE workos_user_id = ANY($1) ORDER BY workos_user_id`, + [[primary, secondary]], + )).rows; + expect(auditRowsAfterDeletion).toHaveLength(1); + const replay = await deleteIdentityCredential(secondary, 'workos_webhook'); + expect(replay).toMatchObject({ + deleted: false, + quarantine: null, + replay: { + audit_id: auditRowsAfterDeletion[0].id, + kind: 'same_source', + original_deletion_source: 'workos_webhook', + requested_deletion_source: 'workos_webhook', + }, + }); + expect((await pool.query( + `SELECT * FROM registry_audit_log WHERE workos_user_id = $1 ORDER BY created_at, id`, + [secondary], + )).rows).toEqual(auditRowsAfterDeletion); + expect((await pool.query( + `SELECT * FROM authorization_epochs + WHERE workos_user_id = ANY($1) ORDER BY workos_user_id`, + [[primary, secondary]], + )).rows).toEqual(epochRowsAfterDeletion); expect(notifySystemError).not.toHaveBeenCalled(); }); + it('snapshots and revokes Slack-keyed authority and topic subscriptions for the deleted credential', async () => { + const { primary, identityId, slackUserId, aaoAdminGroupId } = await seedDeletionAuthority(); + await pool.query( + `INSERT INTO working_group_memberships (working_group_id, workos_user_id, status) + VALUES ($1, $2, 'active')`, + [aaoAdminGroupId, slackUserId], + ); + await pool.query( + `INSERT INTO working_group_leaders (working_group_id, user_id) + VALUES ($1, $2)`, + [aaoAdminGroupId, slackUserId], + ); + await pool.query( + `INSERT INTO working_group_topic_subscriptions (working_group_id, workos_user_id, topic_slugs) + VALUES ($1, $2, ARRAY['deleted-topic'])`, + [aaoAdminGroupId, slackUserId], + ); + + await deleteIdentityCredential(primary, 'workos_webhook'); + + expect((await pool.query( + `SELECT status FROM working_group_memberships + WHERE working_group_id = $1 AND workos_user_id = $2`, + [aaoAdminGroupId, slackUserId], + )).rows).toEqual([{ status: 'inactive' }]); + expect((await pool.query( + `SELECT 1 FROM working_group_leaders WHERE working_group_id = $1 AND user_id = $2`, + [aaoAdminGroupId, slackUserId], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM working_group_topic_subscriptions + WHERE working_group_id = $1 AND workos_user_id = $2`, + [aaoAdminGroupId, slackUserId], + )).rows).toEqual([]); + expect(await isSlackUserAAOAdmin(slackUserId)).toBe(false); + + const quarantine = await getIdentityRecoveryQuarantine(identityId); + expect(quarantine?.before_graph.working_group_memberships).toEqual( + expect.arrayContaining([expect.objectContaining({ + workos_user_id: slackUserId, + working_group_id: aaoAdminGroupId, + status: 'active', + })]), + ); + expect(quarantine?.before_graph.working_group_leaders).toEqual( + expect.arrayContaining([expect.objectContaining({ user_id: slackUserId })]), + ); + expect(quarantine?.before_graph.working_group_topic_subscriptions).toEqual( + expect.arrayContaining([expect.objectContaining({ + workos_user_id: slackUserId, + topic_slugs: ['deleted-topic'], + })]), + ); + expect(quarantine?.aao_admin_revocations.filter( + (row) => row.target_user_id === slackUserId, + )).toHaveLength(1); + }); + + it('quarantines a corrupt zero-primary identity when deleting a secondary leaves survivors', async () => { + const { primary, secondary, bystander, identityId } = await seedDeletionAuthority(); + const credentials = [primary, secondary, bystander].sort(); + await pool.query( + `UPDATE identity_workos_users SET is_primary = FALSE WHERE identity_id = $1`, + [identityId], + ); + await expect(readCredentialAuthorizationLifecycle(secondary)).resolves.toEqual({ + status: 'terminal', + reason: 'missing_primary', + }); + const beforeGraph = await fullBeforeGraph(identityId, credentials); + + const deletion = await deleteIdentityCredential(secondary, 'workos_webhook'); + expect(deletion.deleted).toBe(true); + const auditRows = await pool.query<{ id: string }>( + `SELECT id FROM registry_audit_log + WHERE action = 'identity_primary_deletion_quarantined' AND resource_id = $1`, + [identityId], + ); + expect(auditRows.rows).toHaveLength(1); + expect(await getIdentityRecoveryQuarantine(identityId)).toEqual({ + audit_id: auditRows.rows[0].id, + identity_id: identityId, + deleted_workos_user_id: secondary, + deletion_source: 'workos_webhook', + actor: { type: 'workos_provider', source: 'workos_webhook', workos_user_id: secondary }, + recovery_state: IDENTITY_RECOVERY_STATE, + snapshot_policy: IDENTITY_DELETION_SNAPSHOT_POLICY, + before_graph: beforeGraph, + aao_admin_revocations: [], + }); + expect(notifySystemError).toHaveBeenCalledWith({ + source: `identity-primary-deletion-quarantine:${identityId}`, + errorMessage: [ + `Credential ${secondary} was deleted without promotion.`, + `Identity ${identityId} is quarantined.`, + `recovery_state=${IDENTITY_RECOVERY_STATE}.`, + `audit_id=${auditRows.rows[0].id}.`, + ].join(' '), + }); + expect((await pool.query( + `SELECT workos_user_id FROM identity_workos_users + WHERE identity_id = $1 AND is_primary = TRUE`, + [identityId], + )).rows).toEqual([]); + }); + + it.each(['quarantine-primary', 'non-primary', 'singleton', 'unbound'] as const)( + 'rolls back every side effect when DELETE RETURNING yields no row on the %s path', + async (path) => { + let target: string; + let credentials: string[]; + if (path === 'quarantine-primary' || path === 'non-primary') { + const seeded = await seedDeletionAuthority(); + target = path === 'quarantine-primary' ? seeded.primary : seeded.secondary; + credentials = [seeded.primary, seeded.secondary, seeded.bystander]; + } else { + target = await insertUser(`return_null_${path}`, `${path}@return-null.test`); + credentials = [target]; + await pool.query( + `INSERT INTO organizations (workos_organization_id, name) + VALUES ($1, 'RETURN NULL test')`, + [TEST_ORG], + ); + await pool.query( + `INSERT INTO organization_memberships ( + workos_user_id, workos_organization_id, email, role + ) VALUES ($1, $2, $3, 'admin')`, + [target, TEST_ORG, `${path}@return-null.test`], + ); + if (path === 'unbound') { + const binding = await pool.query<{ identity_id: string }>( + `DELETE FROM identity_workos_users WHERE workos_user_id = $1 RETURNING identity_id`, + [target], + ); + await pool.query(`DELETE FROM identities WHERE id = $1`, [binding.rows[0].identity_id]); + } + await bumpAuthorizationEpochs(pool, [target]); + } + const slackUserId = `${TEST_SLACK_PREFIX}return_null_${path}`; + await pool.query( + `INSERT INTO slack_user_mappings ( + slack_user_id, slack_email, workos_user_id, mapping_status, mapping_source, mapped_at + ) VALUES ($1, $2, $3, 'mapped', 'manual_admin', NOW()) + ON CONFLICT (slack_user_id) DO UPDATE + SET workos_user_id = EXCLUDED.workos_user_id, + mapping_status = 'mapped', mapping_source = 'manual_admin', mapped_at = NOW()`, + [slackUserId, `${path}@return-null.test`, target], + ); + const before = await survivorAuthority(credentials); + const fingerprintBefore = await getAuthorizationFingerprint(credentials); + await pool.query(` + CREATE FUNCTION suppress_confirmed_user_delete() RETURNS trigger AS $$ + BEGIN + IF OLD.workos_user_id = '${target}' THEN + RETURN NULL; + END IF; + RETURN OLD; + END; + $$ LANGUAGE plpgsql; + CREATE TRIGGER suppress_confirmed_user_delete + BEFORE DELETE ON users + FOR EACH ROW EXECUTE FUNCTION suppress_confirmed_user_delete(); + `); + try { + await expect(deleteIdentityCredential(target, 'workos_webhook')) + .rejects.toThrow('Confirmed credential deletion did not delete exactly one locked user'); + expect(await survivorAuthority(credentials)).toEqual(before); + expect(await getAuthorizationFingerprint(credentials)).toBe(fingerprintBefore); + expect((await pool.query( + `SELECT 1 FROM registry_audit_log + WHERE workos_user_id = $1 + AND action IN ('identity_credential_deleted', 'identity_primary_deletion_quarantined')`, + [target], + )).rows).toEqual([]); + expect(notifySystemError).not.toHaveBeenCalled(); + } finally { + await pool.query(`DROP TRIGGER suppress_confirmed_user_delete ON users`); + await pool.query(`DROP FUNCTION suppress_confirmed_user_delete()`); + } + }, + ); + it('rolls back epoch changes and deletion together when the database refuses deletion', async () => { const { primary, secondary, bystander } = await seedDeletionAuthority(); const credentials = [primary, secondary, bystander]; @@ -424,7 +807,8 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { await pool.query(` CREATE OR REPLACE FUNCTION reject_identity_quarantine_audit() RETURNS trigger AS $$ BEGIN - IF NEW.action = 'identity_primary_deletion_quarantined' THEN + IF NEW.workos_user_id = '${primary}' + AND NEW.action = 'identity_primary_deletion_quarantined' THEN RAISE EXCEPTION 'injected quarantine audit failure'; END IF; RETURN NEW; @@ -446,6 +830,214 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { await pool.query(`DROP FUNCTION reject_identity_quarantine_audit()`); } }); + + it.each(['insert', 'update'] as const)( + 'rolls back deletion when an authorization epoch %s trigger suppresses rows', + async (operation) => { + const { primary, secondary, bystander, identityId } = await seedDeletionAuthority(); + const credentials = [primary, secondary, bystander]; + if (operation === 'insert') { + await pool.query( + `DELETE FROM authorization_epochs WHERE workos_user_id = ANY($1)`, + [credentials], + ); + } + const before = await survivorAuthority(credentials); + const fingerprintBefore = await getAuthorizationFingerprint(credentials); + const triggerOperation = operation.toUpperCase(); + await pool.query(` + CREATE FUNCTION suppress_authorization_epoch_${operation}() RETURNS trigger AS $$ + BEGIN + IF NEW.workos_user_id = '${primary}' THEN + RETURN NULL; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + CREATE TRIGGER suppress_authorization_epoch_${operation} + BEFORE ${triggerOperation} ON authorization_epochs + FOR EACH ROW EXECUTE FUNCTION suppress_authorization_epoch_${operation}(); + `); + try { + await expect(deleteIdentityCredential(primary, 'workos_webhook')) + .rejects.toThrow('did not bump every live authorization epoch'); + expect(await survivorAuthority(credentials)).toEqual(before); + expect(await getAuthorizationFingerprint(credentials)).toBe(fingerprintBefore); + expect(await getIdentityRecoveryQuarantine(identityId)).toBeNull(); + expect((await pool.query( + `SELECT 1 FROM registry_audit_log + WHERE workos_user_id = $1 + AND action IN ('identity_credential_deleted', 'identity_primary_deletion_quarantined')`, + [primary], + )).rows).toEqual([]); + expect(notifySystemError).not.toHaveBeenCalled(); + } finally { + await pool.query( + `DROP TRIGGER suppress_authorization_epoch_${operation} ON authorization_epochs`, + ); + await pool.query(`DROP FUNCTION suppress_authorization_epoch_${operation}()`); + } + }, + ); + + it('rolls back deletion when the deletion-audit INSERT returns no row', async () => { + const { primary, secondary, bystander, identityId } = await seedDeletionAuthority(); + const credentials = [primary, secondary, bystander]; + const before = await survivorAuthority(credentials); + const fingerprintBefore = await getAuthorizationFingerprint(credentials); + await pool.query(` + CREATE FUNCTION suppress_identity_deletion_audit() RETURNS trigger AS $$ + BEGIN + IF NEW.workos_user_id = '${primary}' + AND NEW.action IN ('identity_credential_deleted', 'identity_primary_deletion_quarantined') THEN + RETURN NULL; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + CREATE TRIGGER suppress_identity_deletion_audit + BEFORE INSERT ON registry_audit_log + FOR EACH ROW EXECUTE FUNCTION suppress_identity_deletion_audit(); + `); + try { + await expect(deleteIdentityCredential(primary, 'workos_webhook')) + .rejects.toThrow('audit was not durably recorded'); + expect(await survivorAuthority(credentials)).toEqual(before); + expect(await getAuthorizationFingerprint(credentials)).toBe(fingerprintBefore); + expect(await getIdentityRecoveryQuarantine(identityId)).toBeNull(); + expect((await pool.query( + `SELECT 1 FROM registry_audit_log + WHERE workos_user_id = $1 + AND action IN ('identity_credential_deleted', 'identity_primary_deletion_quarantined')`, + [primary], + )).rows).toEqual([]); + expect(notifySystemError).not.toHaveBeenCalled(); + } finally { + await pool.query(`DROP TRIGGER suppress_identity_deletion_audit ON registry_audit_log`); + await pool.query(`DROP FUNCTION suppress_identity_deletion_audit()`); + } + }); + + it('retains row provenance without copying binary PII or email-link bearer secrets', async () => { + const userId = await insertUser('binary_snapshot', 'binary-snapshot@test.example'); + const rawLinkToken = 'raw-email-link-token-must-not-enter-durable-audit'; + await pool.query( + `INSERT INTO user_avatar_uploads (workos_user_id, image_data, content_type) + VALUES ($1, $2, 'image/png')`, + [userId, Buffer.alloc(64, 0xab)], + ); + await pool.query( + `INSERT INTO member_portraits (user_id, image_url, portrait_data, palette, status) + VALUES ($1, 'https://example.test/portrait.png', $2, 'amber', 'approved')`, + [userId, Buffer.alloc(96, 0xcd)], + ); + await pool.query( + `INSERT INTO email_link_tokens ( + token, primary_workos_user_id, target_email, status, expires_at + ) VALUES ($1, $2, 'linked-target@test.example', 'pending', NOW() + INTERVAL '1 hour')`, + [rawLinkToken, userId], + ); + + await expect(deleteIdentityCredential(userId, 'workos_webhook')) + .resolves.toMatchObject({ deleted: true }); + const audit = await pool.query<{ details: any }>( + `SELECT details FROM registry_audit_log + WHERE workos_user_id = $1 AND action = 'identity_credential_deleted'`, + [userId], + ); + expect(audit.rows).toHaveLength(1); + const avatar = audit.rows[0].details.before_graph.user_cascade_rows.user_avatar_uploads[0]; + expect(avatar).toMatchObject({ + workos_user_id: userId, + content_type: 'image/png', + image_data_redacted: true, + image_data_byte_length: 64, + }); + expect(avatar).not.toHaveProperty('image_data'); + const portrait = audit.rows[0].details.before_graph.user_cascade_rows.member_portraits[0]; + expect(portrait).toMatchObject({ + user_id: userId, + image_url: 'https://example.test/portrait.png', + portrait_data_redacted: true, + portrait_data_byte_length: 96, + }); + expect(portrait).not.toHaveProperty('portrait_data'); + const emailLinkToken = + audit.rows[0].details.before_graph.user_cascade_rows.email_link_tokens[0]; + expect(emailLinkToken).toMatchObject({ + primary_workos_user_id: userId, + target_email: 'linked-target@test.example', + status: 'pending', + token_redacted: true, + }); + expect(emailLinkToken).not.toHaveProperty('token'); + expect(JSON.stringify(audit.rows[0].details)).not.toContain(rawLinkToken); + expect(audit.rows[0].details.snapshot_policy).toEqual(IDENTITY_DELETION_SNAPSHOT_POLICY); + expect(Buffer.byteLength(JSON.stringify(audit.rows[0].details), 'utf8')) + .toBeLessThanOrEqual(IDENTITY_DELETION_SNAPSHOT_MAX_DETAILS_BYTES); + }); + + it('fails closed before deletion when the complete durable snapshot exceeds its byte bound', async () => { + const userId = await insertUser('snapshot_bound', 'snapshot-bound@test.example'); + await pool.query( + `INSERT INTO email_link_tokens ( + token, primary_workos_user_id, target_email, status, expires_at, merge_summary + ) VALUES ( + 'snapshot-bound-token', $1, 'bounded-target@test.example', 'pending', + NOW() + INTERVAL '1 hour', jsonb_build_object('payload', repeat('x', $2)) + )`, + [userId, IDENTITY_DELETION_SNAPSHOT_MAX_DETAILS_BYTES], + ); + + await expect(deleteIdentityCredential(userId, 'workos_webhook')) + .rejects.toThrow('snapshot exceeds the fail-closed byte limit'); + expect((await pool.query( + `SELECT 1 FROM users WHERE workos_user_id = $1`, + [userId], + )).rows).toHaveLength(1); + expect((await pool.query( + `SELECT 1 FROM email_link_tokens WHERE primary_workos_user_id = $1`, + [userId], + )).rows).toHaveLength(1); + expect((await pool.query( + `SELECT 1 FROM registry_audit_log WHERE workos_user_id = $1`, + [userId], + )).rows).toEqual([]); + }); + + it('rolls back deletion, audit, and authority when an AAO-admin deactivation returns no row', async () => { + const { primary, secondary, bystander, identityId } = await seedDeletionAuthority(); + const credentials = [primary, secondary, bystander]; + const before = await survivorAuthority(credentials); + const fingerprintBefore = await getAuthorizationFingerprint(credentials); + await pool.query(` + CREATE FUNCTION suppress_aao_admin_deactivation() RETURNS trigger AS $$ + BEGIN + IF OLD.workos_user_id = '${primary}' AND NEW.status = 'inactive' THEN + RETURN NULL; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + CREATE TRIGGER suppress_aao_admin_deactivation + BEFORE UPDATE ON working_group_memberships + FOR EACH ROW EXECUTE FUNCTION suppress_aao_admin_deactivation(); + `); + try { + await expect(deleteIdentityCredential(primary, 'workos_webhook')) + .rejects.toThrow('did not deactivate every working-group membership'); + expect(await survivorAuthority(credentials)).toEqual(before); + expect(await getAuthorizationFingerprint(credentials)).toBe(fingerprintBefore); + expect(await getIdentityRecoveryQuarantine(identityId)).toBeNull(); + expect((await pool.query( + `SELECT event_type FROM aao_admin_access_events WHERE target_user_id = $1`, [primary], + )).rows).toEqual([{ event_type: 'granted' }]); + expect(notifySystemError).not.toHaveBeenCalled(); + } finally { + await pool.query(`DROP TRIGGER suppress_aao_admin_deactivation ON working_group_memberships`); + await pool.query(`DROP FUNCTION suppress_aao_admin_deactivation()`); + } + }); it('deletes an account with accumulated community data', async () => { const userId = await insertUser('community', 'community@test.example'); await pool.query( @@ -477,4 +1069,32 @@ describe('user.deleted: automatic promotion containment (#6827)', () => { expect(restrictive.rows).toEqual([]); }); + + it('keeps the durable snapshot inventory synchronized with every users FK deletion action', async () => { + const readInventory = async (deleteType: 'c' | 'n') => { + const rows = await pool.query<{ table_name: string; column_name: string }>( + `SELECT con.conrelid::regclass::text AS table_name, attr.attname AS column_name + FROM pg_constraint con + JOIN LATERAL unnest(con.conkey) AS key(attnum) ON TRUE + JOIN pg_attribute attr + ON attr.attrelid = con.conrelid AND attr.attnum = key.attnum + WHERE con.contype = 'f' + AND con.confrelid = 'users'::regclass + AND con.confdeltype = $1 + ORDER BY table_name, column_name`, + [deleteType], + ); + const grouped = new Map<string, string[]>(); + for (const row of rows.rows) { + grouped.set(row.table_name, [...(grouped.get(row.table_name) ?? []), row.column_name]); + } + return [...grouped].map(([table, columns]) => ({ table, columns: columns.sort() })); + }; + const normalize = (specs: readonly { readonly table: string; readonly columns: readonly string[] }[]) => + specs.map(({ table, columns }) => ({ table, columns: [...columns].sort() })) + .sort((a, b) => a.table.localeCompare(b.table)); + + expect(await readInventory('c')).toEqual(normalize(USER_CASCADE_SNAPSHOT_INVENTORY)); + expect(await readInventory('n')).toEqual(normalize(USER_SET_NULL_SNAPSHOT_INVENTORY)); + }); }); diff --git a/server/tests/integration/workos-user-deletion-race.test.ts b/server/tests/integration/workos-user-deletion-race.test.ts index 3a0927735bc..823b2da15ea 100644 --- a/server/tests/integration/workos-user-deletion-race.test.ts +++ b/server/tests/integration/workos-user-deletion-race.test.ts @@ -9,6 +9,8 @@ const mocks = vi.hoisted(() => ({ listUsers: vi.fn(), invalidateSessionsForUsers: vi.fn(), notifySystemError: vi.fn(), + tryAutoLinkWebsiteUserToSlack: vi.fn(), + coordinateDeletionCallers: false, deletionEntrants: [] as Array<{ workosUserId: string; source: string }>, deletionWaiters: [] as Array<() => void>, })); @@ -34,6 +36,11 @@ vi.mock('../../src/addie/error-notifier.js', async (importOriginal) => ({ notifySystemError: mocks.notifySystemError, })); +vi.mock('../../src/slack/sync.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/slack/sync.js')>()), + tryAutoLinkWebsiteUserToSlack: mocks.tryAutoLinkWebsiteUserToSlack, +})); + vi.mock('../../src/services/identity-credential-deletion.js', async (importOriginal) => { const actual = await importOriginal<typeof import('../../src/services/identity-credential-deletion.js')>(); return { @@ -42,7 +49,7 @@ vi.mock('../../src/services/identity-credential-deletion.js', async (importOrigi workosUserId: string, source: 'workos_webhook' | 'sync_users_backfill', ) => { - if (workosUserId === 'user_wh_backfill_race_primary') { + if (mocks.coordinateDeletionCallers && workosUserId === 'user_wh_backfill_race_primary') { mocks.deletionEntrants.push({ workosUserId, source }); await new Promise<void>((resolve) => { mocks.deletionWaiters.push(resolve); @@ -57,14 +64,34 @@ vi.mock('../../src/services/identity-credential-deletion.js', async (importOrigi }); process.env.WORKOS_WEBHOOK_SECRET = 'test_workos_webhook_secret'; +process.env.WORKOS_PROVIDER_PREFETCH_TIMEOUT_MS = '100'; const { initializeDatabase, closeDatabase } = await import('../../src/db/client.js'); const { runMigrations } = await import('../../src/db/migrate.js'); const { getAuthorizationFingerprint } = await import('../../src/db/authorization-epoch-db.js'); -const { getIdentityRecoveryQuarantine, IDENTITY_RECOVERY_STATE } = await import('../../src/db/identity-db.js'); -const { createWorkOSWebhooksRouter, backfillUsers } = await import('../../src/routes/workos-webhooks.js'); +const { + getIdentityRecoveryQuarantine, + IDENTITY_DELETION_SNAPSHOT_POLICY, + IDENTITY_RECOVERY_STATE, + withActiveCredentialEventMutation, + withCredentialCreationEventMutation, + upsertWorkosUserInCredentialEvent, +} = await import('../../src/db/identity-db.js'); +const { + createWorkOSWebhooksRouter, + backfillUsers, + shouldAbortBackfillDeletion, +} = await import('../../src/routes/workos-webhooks.js'); const { getUnifiedUsersCache, setUnifiedUsersCache } = await import('../../src/cache/unified-users.js'); const { isSlackUserAAOAdmin } = await import('../../src/addie/mcp/admin-tools.js'); +const { deleteIdentityCredential } = await import('../../src/services/identity-credential-deletion.js'); +const { SlackDatabase } = await import('../../src/db/slack-db.js'); +const { WorkingGroupDatabase } = await import('../../src/db/working-group-db.js'); +const { MeetingsDatabase } = await import('../../src/db/meetings-db.js'); +const { + setMembershipRole, + upsertOrganizationMembership, +} = await import('../../src/db/membership-db.js'); const PREFIX = 'user_wh_backfill_race_'; const PRIMARY = `${PREFIX}primary`; @@ -72,6 +99,7 @@ const SECONDARY = `${PREFIX}secondary`; const BYSTANDER = `${PREFIX}bystander`; const ORG = 'org_wh_backfill_race_pinnacle'; const SLACK = 'slack_wh_backfill_race_primary'; +const CHAPTER_SLUG = 'wh-backfill-race-chapter'; describe('WorkOS webhook vs sync-users deletion', () => { let pool: Pool; @@ -94,6 +122,8 @@ describe('WorkOS webhook vs sync-users deletion', () => { vi.clearAllMocks(); mocks.deletionEntrants.length = 0; mocks.deletionWaiters.length = 0; + mocks.coordinateDeletionCallers = false; + mocks.tryAutoLinkWebsiteUserToSlack.mockResolvedValue({ linked: false, reason: 'no_slack_user' }); }); async function cleanup() { @@ -105,11 +135,24 @@ describe('WorkOS webhook vs sync-users deletion', () => { AND workos_user_id LIKE $1`, [`${PREFIX}%`], ); - await pool.query(`DELETE FROM working_group_memberships WHERE workos_user_id LIKE $1`, [`${PREFIX}%`]); - await pool.query(`DELETE FROM working_group_leaders WHERE user_id LIKE $1`, [`${PREFIX}%`]); + await pool.query( + `DELETE FROM working_group_memberships WHERE workos_user_id LIKE $1 OR workos_user_id = $2`, + [`${PREFIX}%`, SLACK], + ); + await pool.query( + `DELETE FROM working_group_leaders WHERE user_id LIKE $1 OR user_id = $2`, + [`${PREFIX}%`, SLACK], + ); + await pool.query( + `DELETE FROM working_group_topic_subscriptions + WHERE workos_user_id LIKE $1 OR workos_user_id = $2`, + [`${PREFIX}%`, SLACK], + ); await pool.query(`DELETE FROM slack_user_mappings WHERE slack_user_id = $1`, [SLACK]); + await pool.query(`DELETE FROM invitation_seat_types WHERE workos_invitation_id LIKE 'invite_wh_%'`); await pool.query(`DELETE FROM organization_memberships WHERE workos_user_id LIKE $1`, [`${PREFIX}%`]); await pool.query(`DELETE FROM users WHERE workos_user_id LIKE $1`, [`${PREFIX}%`]); + await pool.query(`DELETE FROM working_groups WHERE slug = $1`, [CHAPTER_SLUG]); await pool.query(`DELETE FROM organizations WHERE workos_organization_id = $1`, [ORG]); } @@ -174,6 +217,237 @@ describe('WorkOS webhook vs sync-users deletion', () => { return identityId; } + function signedWebhook(app: express.Express, event: string, data: Record<string, unknown>) { + return request(app) + .post('/api/webhooks/workos') + .set('Content-Type', 'application/json') + .set('WorkOS-Signature', 'valid-test-signature') + .send({ id: `event_${event}_${Date.now()}`, event, data, created_at: new Date().toISOString() }); + } + + async function within<T>(promise: Promise<T>, timeoutMs = 5_000): Promise<T> { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise<never>((_, reject) => { + timeout = setTimeout(() => reject(new Error(`race did not settle within ${timeoutMs}ms`)), timeoutMs); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } + } + + it('fails closed before a suspicious backfill deletion batch can write tombstones', () => { + expect(shouldAbortBackfillDeletion(1_000, 101)).toBe(true); + expect(shouldAbortBackfillDeletion(40, 11)).toBe(true); + expect(shouldAbortBackfillDeletion(40, 10)).toBe(false); + expect(shouldAbortBackfillDeletion(1_000, 25)).toBe(false); + }); + + it('bounds concurrent guarded writers below deletion-reserved pool capacity', async () => { + let activeCallbacks = 0; + let maxActiveCallbacks = 0; + + const results = await within(Promise.all( + Array.from({ length: 10 }, async (_, index) => withCredentialCreationEventMutation( + `${PREFIX}pool_capacity_${index}`, + async (client) => { + await upsertWorkosUserInCredentialEvent(client, { + id: `${PREFIX}pool_capacity_${index}`, + email: `pool-${index}@race.test`, + firstName: 'Pool', + lastName: 'Capacity', + emailVerified: true, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, 'provider_authoritative'); + activeCallbacks++; + maxActiveCallbacks = Math.max(maxActiveCallbacks, activeCallbacks); + try { + await client.query(`SELECT pg_sleep(0.025)`); + } finally { + activeCallbacks--; + } + }, + )), + ), 3_000); + + expect(results).toHaveLength(10); + expect(results.every((result) => result.applied)).toBe(true); + expect(maxActiveCallbacks).toBeGreaterThan(1); + // One connection stays reserved so confirmed deletion can always enter. + expect(maxActiveCallbacks).toBeLessThanOrEqual(7); + }); + + it('never replays a callback that has started after a retryable database error', async () => { + let callbackCalls = 0; + const retryableAfterMutation = Object.assign(new Error('injected callback deadlock'), { code: '40P01' }); + + await expect(withCredentialCreationEventMutation( + `${PREFIX}no_partial_replay`, + async (client) => { + await upsertWorkosUserInCredentialEvent(client, { + id: `${PREFIX}no_partial_replay`, + email: 'no-partial-replay@race.test', + firstName: 'No', + lastName: 'Replay', + emailVerified: true, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, 'provider_authoritative'); + callbackCalls++; + throw retryableAfterMutation; + }, + )).rejects.toBe(retryableAfterMutation); + + expect(callbackCalls).toBe(1); + expect((await pool.query( + `SELECT 1 FROM users WHERE workos_user_id = $1`, + [`${PREFIX}no_partial_replay`], + )).rows).toEqual([]); + }); + + it('preserves the distinct provider-authoritative and login-preserving name policies', async () => { + const userId = `${PREFIX}name_policy`; + const timestamp = new Date().toISOString(); + const created = await withCredentialCreationEventMutation(userId, (client) => + upsertWorkosUserInCredentialEvent(client, { + id: userId, + email: 'name-policy@race.test', + firstName: 'Original', + lastName: 'Local', + emailVerified: true, + createdAt: timestamp, + updatedAt: timestamp, + }, 'provider_authoritative'), + ); + expect(created.applied).toBe(true); + + const login = await withActiveCredentialEventMutation(userId, (client) => + upsertWorkosUserInCredentialEvent(client, { + id: userId, + email: 'name-policy@race.test', + firstName: 'Login', + lastName: 'Provider', + emailVerified: true, + createdAt: timestamp, + updatedAt: timestamp, + }, 'preserve_existing'), + ); + expect(login.applied).toBe(true); + expect((await pool.query( + `SELECT first_name, last_name FROM users WHERE workos_user_id = $1`, + [userId], + )).rows).toEqual([{ first_name: 'Original', last_name: 'Local' }]); + + mocks.constructEvent.mockResolvedValue(undefined); + const app = express(); + app.use('/api/webhooks', createWorkOSWebhooksRouter()); + const webhook = await signedWebhook(app, 'user.updated', { + id: userId, + email: 'name-policy@race.test', + first_name: 'Webhook', + last_name: 'Authoritative', + email_verified: true, + created_at: timestamp, + updated_at: timestamp, + }); + expect(webhook.status).toBe(200); + expect((await pool.query( + `SELECT first_name, last_name FROM users WHERE workos_user_id = $1`, + [userId], + )).rows).toEqual([{ first_name: 'Webhook', last_name: 'Authoritative' }]); + }); + + it('durably tombstones a signed deletion received before first local creation', async () => { + const deletedBeforeCreate = `${PREFIX}deleted_before_create`; + mocks.constructEvent.mockResolvedValue(undefined); + const app = express(); + app.use('/api/webhooks', createWorkOSWebhooksRouter()); + + const deletion = await signedWebhook(app, 'user.deleted', { id: deletedBeforeCreate }); + expect(deletion.status).toBe(200); + const audits = await pool.query<{ id: string; action: string; details: Record<string, unknown> }>( + `SELECT id, action, details FROM registry_audit_log + WHERE workos_user_id = $1 + ORDER BY created_at, id`, + [deletedBeforeCreate], + ); + expect(audits.rows).toHaveLength(1); + expect(audits.rows[0].action).toBe('identity_credential_deleted'); + expect(audits.rows[0].details).toMatchObject({ + deleted_workos_user_id: deletedBeforeCreate, + deletion_source: 'workos_webhook', + before_graph: { + users: [], + identity_workos_users: [], + }, + }); + + const replayCache = new Map([[ORG, [{ + id: deletedBeforeCreate, + email: 'must-remain-cached@race.test', + firstName: 'Replay', + lastName: 'Sentinel', + }]]]); + setUnifiedUsersCache(replayCache); + mocks.invalidateSessionsForUsers.mockClear(); + mocks.notifySystemError.mockClear(); + const sameSourceReplay = await signedWebhook(app, 'user.deleted', { id: deletedBeforeCreate }); + expect(sameSourceReplay.status).toBe(200); + const differentSourceReplay = await deleteIdentityCredential( + deletedBeforeCreate, + 'sync_users_backfill', + ); + expect(differentSourceReplay).toMatchObject({ + deleted: false, + affectedUserIds: [deletedBeforeCreate], + affectedSlackUserIds: [], + quarantine: null, + replay: { + audit_id: audits.rows[0].id, + kind: 'different_source', + original_deletion_source: 'workos_webhook', + requested_deletion_source: 'sync_users_backfill', + }, + }); + expect((await pool.query( + `SELECT id, action, details FROM registry_audit_log + WHERE workos_user_id = $1 + ORDER BY created_at, id`, + [deletedBeforeCreate], + )).rows).toEqual(audits.rows); + expect((await pool.query( + `SELECT 1 FROM authorization_epochs WHERE workos_user_id = $1`, + [deletedBeforeCreate], + )).rows).toEqual([]); + expect(getUnifiedUsersCache()).toBe(replayCache); + expect(mocks.invalidateSessionsForUsers).not.toHaveBeenCalled(); + expect(mocks.notifySystemError).not.toHaveBeenCalled(); + + const staleCreate = await signedWebhook(app, 'user.created', { + id: deletedBeforeCreate, + email: 'deleted-before-create@race.test', + first_name: 'Deleted', + last_name: 'BeforeCreate', + email_verified: true, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }); + expect(staleCreate.status).toBe(200); + expect((await pool.query( + `SELECT 1 FROM users WHERE workos_user_id = $1`, + [deletedBeforeCreate], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM identity_workos_users WHERE workos_user_id = $1`, + [deletedBeforeCreate], + )).rows).toEqual([]); + expect(mocks.tryAutoLinkWebsiteUserToSlack).not.toHaveBeenCalled(); + }); + it('serializes both real callers and invalidates cached authority exactly through the helper', async () => { const identityId = await seedIdentity(); expect((await pool.query<{ workos_user_id: string }>( @@ -222,6 +496,7 @@ describe('WorkOS webhook vs sync-users deletion', () => { return { id: workosUserId }; }); mocks.constructEvent.mockResolvedValue(undefined); + mocks.coordinateDeletionCallers = true; // Keep the winning transaction open briefly so the other real caller // reaches the identity lock and exercises PostgreSQL serialization. @@ -279,6 +554,7 @@ describe('WorkOS webhook vs sync-users deletion', () => { deleted_workos_user_id: PRIMARY, deletion_source: expect.stringMatching(/^(workos_webhook|sync_users_backfill)$/), recovery_state: IDENTITY_RECOVERY_STATE, + snapshot_policy: IDENTITY_DELETION_SNAPSHOT_POLICY, }); const quarantineAudits = (await pool.query<{ id: string }>( `SELECT id FROM registry_audit_log @@ -297,7 +573,7 @@ describe('WorkOS webhook vs sync-users deletion', () => { expect(mocks.notifySystemError).toHaveBeenCalledWith({ source: `identity-primary-deletion-quarantine:${identityId}`, errorMessage: [ - `Primary credential ${PRIMARY} was deleted without promotion.`, + `Credential ${PRIMARY} was deleted without promotion.`, `Identity ${identityId} is quarantined.`, `recovery_state=${IDENTITY_RECOVERY_STATE}.`, `audit_id=${quarantineAudits[0].id}.`, @@ -365,4 +641,388 @@ describe('WorkOS webhook vs sync-users deletion', () => { )).rows).toEqual([]); expect(await isSlackUserAAOAdmin(SLACK)).toBe(false); }); + + it('lets deletion finish while post-commit work waits and fences stale Slack/chapter writes', async () => { + await seedIdentity(); + const chapter = await pool.query<{ id: string }>( + `INSERT INTO working_groups (name, slug) VALUES ('Race chapter', $1) RETURNING id`, + [CHAPTER_SLUG], + ); + await pool.query( + `UPDATE slack_user_mappings + SET workos_user_id = NULL, mapping_status = 'unmapped', mapping_source = NULL + WHERE slack_user_id = $1`, + [SLACK], + ); + mocks.constructEvent.mockResolvedValue(undefined); + let releaseSideEffects!: () => void; + let sideEffectsEntered!: () => void; + const entered = new Promise<void>((resolve) => { sideEffectsEntered = resolve; }); + const release = new Promise<void>((resolve) => { releaseSideEffects = resolve; }); + mocks.tryAutoLinkWebsiteUserToSlack.mockImplementation(async () => { + sideEffectsEntered(); + await release; + const mapping = await new SlackDatabase().mapUser({ + slack_user_id: SLACK, + workos_user_id: PRIMARY, + mapping_source: 'email_auto', + }); + if (mapping) { + await new WorkingGroupDatabase().addMembership({ + working_group_id: chapter.rows[0].id, + workos_user_id: PRIMARY, + }); + } + return { linked: Boolean(mapping), slack_user_id: SLACK, workos_user_id: PRIMARY, chapters_joined: 0 }; + }); + + const app = express(); + app.use('/api/webhooks', createWorkOSWebhooksRouter()); + const staleUserEvent = Promise.resolve(signedWebhook(app, 'user.created', { + id: PRIMARY, + email: 'jordan-race@gmail.com', + first_name: 'Jordan', + last_name: 'Race', + email_verified: true, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + })); + await entered; + let deletionSettled = false; + const deletion = deleteIdentityCredential(PRIMARY, 'workos_webhook') + .finally(() => { deletionSettled = true; }); + await new Promise((resolve) => setTimeout(resolve, 50)); + const deletionWasBlocked = !deletionSettled; + releaseSideEffects(); + const [eventResponse, deletionResult] = await within(Promise.all([staleUserEvent, deletion])); + + expect(deletionWasBlocked).toBe(false); + expect(eventResponse.status).toBe(200); + expect(deletionResult.deleted).toBe(true); + expect((await pool.query(`SELECT 1 FROM users WHERE workos_user_id = $1`, [PRIMARY])).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM organization_memberships WHERE workos_user_id = $1`, [PRIMARY], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT status FROM working_group_memberships + WHERE workos_user_id = $1 AND working_group_id = $2`, + [PRIMARY, chapter.rows[0].id], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT workos_user_id, mapping_status FROM slack_user_mappings WHERE slack_user_id = $1`, [SLACK], + )).rows).toEqual([{ workos_user_id: null, mapping_status: 'unmapped' }]); + const replay = await signedWebhook(app, 'user.created', { + id: PRIMARY, + email: 'jordan-race@gmail.com', + first_name: 'Jordan', + last_name: 'Race', + email_verified: true, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }); + expect(replay.status).toBe(200); + expect(mocks.tryAutoLinkWebsiteUserToSlack).toHaveBeenCalledTimes(1); + expect((await pool.query(`SELECT 1 FROM users WHERE workos_user_id = $1`, [PRIMARY])).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM working_group_memberships + WHERE workos_user_id = $1 AND working_group_id = $2 AND status = 'active'`, + [PRIMARY, chapter.rows[0].id], + )).rows).toEqual([]); + }); + + it('commits signed membership locally before post-commit work and lets deletion revoke it immediately', async () => { + await seedIdentity(); + await pool.query( + `DELETE FROM organization_memberships WHERE workos_user_id = $1 AND workos_organization_id = $2`, + [PRIMARY, ORG], + ); + await pool.query( + `INSERT INTO invitation_seat_types ( + workos_invitation_id, workos_organization_id, email, seat_type + ) VALUES ('invite_wh_membership_race', $1, 'jordan-race@pinnacle.example', 'community_only')`, + [ORG], + ); + mocks.constructEvent.mockResolvedValue(undefined); + mocks.getUser.mockResolvedValue({ + id: PRIMARY, + email: 'jordan-race@pinnacle.example', + firstName: 'Jordan', + lastName: 'Race', + emailVerified: true, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }); + let releasePostWrite!: () => void; + let postWriteEntered!: () => void; + const entered = new Promise<void>((resolve) => { postWriteEntered = resolve; }); + const release = new Promise<void>((resolve) => { releasePostWrite = resolve; }); + mocks.tryAutoLinkWebsiteUserToSlack.mockImplementation(async () => { + postWriteEntered(); + await release; + return { linked: false, reason: 'already_linked' }; + }); + + const app = express(); + app.use('/api/webhooks', createWorkOSWebhooksRouter()); + const staleMembershipEvent = Promise.resolve(signedWebhook(app, 'organization_membership.created', { + id: 'om_race_restore', + user_id: PRIMARY, + organization_id: ORG, + status: 'active', + role: { slug: 'admin' }, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + })); + await entered; + const membershipWhileLocked = (await pool.query( + `SELECT role FROM organization_memberships + WHERE workos_user_id = $1 AND workos_organization_id = $2`, + [PRIMARY, ORG], + )).rows; + let deletionSettled = false; + const deletion = deleteIdentityCredential(PRIMARY, 'workos_webhook') + .finally(() => { deletionSettled = true; }); + await new Promise((resolve) => setTimeout(resolve, 50)); + const deletionWasBlocked = !deletionSettled; + releasePostWrite(); + const [eventResponse, deletionResult] = await within(Promise.all([staleMembershipEvent, deletion])); + + expect(deletionWasBlocked).toBe(false); + expect(membershipWhileLocked).toEqual([{ role: 'admin' }]); + expect(eventResponse.status).toBe(200); + expect(deletionResult.deleted).toBe(true); + expect((await pool.query( + `SELECT 1 FROM organization_memberships WHERE workos_user_id = $1`, [PRIMARY], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT workos_user_id, mapping_status FROM slack_user_mappings WHERE slack_user_id = $1`, [SLACK], + )).rows).toEqual([{ workos_user_id: null, mapping_status: 'unmapped' }]); + expect((await pool.query( + `SELECT 1 FROM working_group_memberships WHERE workos_user_id = $1 AND status = 'active'`, [PRIMARY], + )).rows).toEqual([]); + const replay = await signedWebhook(app, 'organization_membership.created', { + id: 'om_race_restore_replay', + user_id: PRIMARY, + organization_id: ORG, + status: 'active', + role: { slug: 'admin' }, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }); + expect(replay.status).toBe(200); + expect(mocks.tryAutoLinkWebsiteUserToSlack).toHaveBeenCalledTimes(1); + expect((await pool.query( + `SELECT 1 FROM organization_memberships WHERE workos_user_id = $1`, [PRIMARY], + )).rows).toEqual([]); + }); + + it('serializes a distinct Slack/WG/leader/topic authority writer with deletion and leaves no alias authority', async () => { + await seedIdentity(); + const chapter = await pool.query<{ id: string }>( + `INSERT INTO working_groups (name, slug) VALUES ('Authority writer race', $1) RETURNING id`, + [CHAPTER_SLUG], + ); + await pool.query( + `UPDATE slack_user_mappings + SET workos_user_id = NULL, mapping_status = 'unmapped', mapping_source = NULL + WHERE slack_user_id = $1`, + [SLACK], + ); + let writerEntered!: () => void; + let releaseWriter!: () => void; + const entered = new Promise<void>((resolve) => { writerEntered = resolve; }); + const release = new Promise<void>((resolve) => { releaseWriter = resolve; }); + + const writer = withActiveCredentialEventMutation(PRIMARY, async (client) => { + await new SlackDatabase().mapUser({ + slack_user_id: SLACK, + workos_user_id: PRIMARY, + mapping_source: 'email_auto', + }, client); + await new WorkingGroupDatabase().addLeader(chapter.rows[0].id, PRIMARY, client); + await client.query( + `INSERT INTO working_group_memberships (working_group_id, workos_user_id, status) + VALUES ($1, $2, 'active')`, + [chapter.rows[0].id, SLACK], + ); + await client.query( + `INSERT INTO working_group_leaders (working_group_id, user_id) + VALUES ($1, $2)`, + [chapter.rows[0].id, SLACK], + ); + await client.query( + `INSERT INTO working_group_topic_subscriptions (working_group_id, workos_user_id, topic_slugs) + VALUES ($1, $2, ARRAY['race-topic'])`, + [chapter.rows[0].id, SLACK], + ); + writerEntered(); + await release; + }); + await entered; + + let deletionSettled = false; + const deletion = deleteIdentityCredential(PRIMARY, 'workos_webhook') + .finally(() => { deletionSettled = true; }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(deletionSettled).toBe(false); + releaseWriter(); + const [writerResult, deletionResult] = await within(Promise.all([writer, deletion])); + + expect(writerResult.applied).toBe(true); + expect(deletionResult.deleted).toBe(true); + expect((await pool.query( + `SELECT workos_user_id, mapping_status FROM slack_user_mappings WHERE slack_user_id = $1`, + [SLACK], + )).rows).toEqual([{ workos_user_id: null, mapping_status: 'unmapped' }]); + expect((await pool.query( + `SELECT 1 FROM working_group_memberships + WHERE working_group_id = $1 AND workos_user_id = ANY($2) AND status = 'active'`, + [chapter.rows[0].id, [PRIMARY, SLACK]], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM working_group_leaders + WHERE working_group_id = $1 AND user_id = ANY($2)`, + [chapter.rows[0].id, [PRIMARY, SLACK]], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM working_group_topic_subscriptions + WHERE working_group_id = $1 AND workos_user_id = ANY($2)`, + [chapter.rows[0].id, [PRIMARY, SLACK]], + )).rows).toEqual([]); + expect(await isSlackUserAAOAdmin(SLACK)).toBe(false); + }); + + it('rejects every authority-writer fence after the credential is durably deleted', async () => { + await seedIdentity(); + const chapter = await pool.query<{ id: string }>( + `INSERT INTO working_groups (name, slug) VALUES ('Deleted writer fence', $1) RETURNING id`, + [CHAPTER_SLUG], + ); + expect((await deleteIdentityCredential(PRIMARY, 'workos_webhook')).deleted).toBe(true); + + const workingGroups = new WorkingGroupDatabase(); + await expect(workingGroups.grantAAOAdminMembership({ + targetUserId: PRIMARY, + actorUserId: BYSTANDER, + actorAuthorizationMechanism: 'aao_admin_working_group', + reason: 'must remain deleted', + })).rejects.toThrow('Cannot grant AAO admin authority to an inactive credential'); + await expect(workingGroups.addMembership({ + working_group_id: chapter.rows[0].id, + workos_user_id: PRIMARY, + })).rejects.toThrow('Cannot add working-group authority to an inactive credential'); + await expect(workingGroups.addLeader( + chapter.rows[0].id, + PRIMARY, + )).rejects.toThrow('Cannot assign leadership to an inactive credential'); + await expect(upsertOrganizationMembership({ + user_id: PRIMARY, + organization_id: ORG, + membership_id: 'om_deleted_writer_fence', + email: 'deleted-writer@race.test', + first_name: 'Deleted', + last_name: 'Writer', + role: 'admin', + seat_type: 'community_only', + has_explicit_seat_type: true, + })).rejects.toThrow('Cannot write organization authority for an inactive credential'); + await expect(setMembershipRole( + PRIMARY, + ORG, + 'admin', + )).rejects.toThrow('Cannot update organization authority for an inactive credential'); + await expect(new MeetingsDatabase().updateTopicSubscription({ + working_group_id: chapter.rows[0].id, + workos_user_id: PRIMARY, + topic_slugs: ['must-remain-deleted'], + })).rejects.toThrow('Cannot update topic subscriptions for an inactive credential'); + + expect((await pool.query( + `SELECT 1 FROM working_group_memberships + WHERE working_group_id = $1 AND workos_user_id = $2 AND status = 'active'`, + [chapter.rows[0].id, PRIMARY], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM working_group_leaders WHERE working_group_id = $1 AND user_id = $2`, + [chapter.rows[0].id, PRIMARY], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM working_group_topic_subscriptions + WHERE working_group_id = $1 AND workos_user_id = $2`, + [chapter.rows[0].id, PRIMARY], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM organization_memberships WHERE workos_user_id = $1`, + [PRIMARY], + )).rows).toEqual([]); + expect((await pool.query( + `SELECT 1 FROM aao_admin_access_events + WHERE target_user_id = $1 AND event_type = 'granted'`, + [PRIMARY], + )).rows).toEqual([]); + }); + + it('does not hold the credential lock while membership provider prefetch is hung', async () => { + await seedIdentity(); + mocks.constructEvent.mockResolvedValue(undefined); + let providerEntered!: () => void; + const entered = new Promise<void>((resolve) => { providerEntered = resolve; }); + mocks.getUser.mockImplementation(async () => { + providerEntered(); + return new Promise(() => undefined); + }); + const app = express(); + app.use('/api/webhooks', createWorkOSWebhooksRouter()); + const started = Date.now(); + const event = Promise.resolve(signedWebhook(app, 'organization_membership.created', { + id: 'om_hung_provider', + user_id: PRIMARY, + organization_id: ORG, + status: 'active', + role: { slug: 'admin' }, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + })); + await entered; + + const deletion = await within(deleteIdentityCredential(PRIMARY, 'workos_webhook'), 1_000); + const eventResponse = await within(event, 1_000); + + expect(deletion.deleted).toBe(true); + expect(eventResponse.status).toBe(500); + expect(Date.now() - started).toBeLessThan(1_000); + expect((await pool.query(`SELECT 1 FROM users WHERE workos_user_id = $1`, [PRIMARY])).rows) + .toEqual([]); + }); + + it('fails a signed stale writer closed after bounded credential-lock retries', async () => { + await seedIdentity(); + mocks.constructEvent.mockResolvedValue(undefined); + const lockHolder = await pool.connect(); + await lockHolder.query('BEGIN'); + await lockHolder.query(`SELECT pg_advisory_xact_lock(hashtextextended($1, 6827))`, [PRIMARY]); + try { + const app = express(); + app.use('/api/webhooks', createWorkOSWebhooksRouter()); + const started = Date.now(); + const response = await within(signedWebhook(app, 'user.updated', { + id: PRIMARY, + email: 'stale-update@pinnacle.example', + first_name: 'Stale', + last_name: 'Update', + email_verified: true, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }), 5_000); + expect(response.status).toBe(500); + expect(Date.now() - started).toBeLessThan(4_000); + expect((await pool.query<{ email: string }>( + `SELECT email FROM users WHERE workos_user_id = $1`, [PRIMARY], + )).rows).toEqual([{ email: 'jordan-race@pinnacle.example' }]); + expect(mocks.tryAutoLinkWebsiteUserToSlack).not.toHaveBeenCalled(); + } finally { + await lockHolder.query('ROLLBACK'); + lockHolder.release(); + } + }); }); diff --git a/server/tests/unit/aao-admin-membership-audit.test.ts b/server/tests/unit/aao-admin-membership-audit.test.ts index 479f2e23494..7c0f3db5903 100644 --- a/server/tests/unit/aao-admin-membership-audit.test.ts +++ b/server/tests/unit/aao-admin-membership-audit.test.ts @@ -1,12 +1,13 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ + poolQuery: vi.fn(), clientQuery: vi.fn(), release: vi.fn(), })); vi.mock('../../src/db/client.js', () => ({ - query: vi.fn(), + query: mocks.poolQuery, getPool: () => ({ connect: vi.fn().mockResolvedValue({ query: mocks.clientQuery, release: mocks.release }), }), @@ -16,6 +17,27 @@ vi.mock('../../src/addie/services/journey-computation.js', () => ({ computeJourneyStage: vi.fn(), })); +vi.mock('../../src/db/identity-db.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/db/identity-db.js')>()), + withActiveCredentialEventMutation: async ( + _workosUserId: string, + mutation: (client: { query: typeof mocks.clientQuery }) => Promise<unknown>, + ) => { + const client = { query: mocks.clientQuery }; + await client.query('BEGIN'); + try { + const value = await mutation(client); + await client.query('COMMIT'); + mocks.release(); + return { applied: true, value }; + } catch (error) { + await client.query('ROLLBACK'); + mocks.release(); + throw error; + } + }, +})); + import { WorkingGroupDatabase } from '../../src/db/working-group-db.js'; function queryResult(rows: unknown[] = [], rowCount = rows.length) { @@ -25,6 +47,12 @@ function queryResult(rows: unknown[] = [], rowCount = rows.length) { describe('AAO site-admin membership audit transaction', () => { beforeEach(() => { vi.clearAllMocks(); + mocks.poolQuery.mockImplementation((sql: string) => { + if (sql.includes('FROM slack_user_mappings')) { + return Promise.resolve(queryResult([{ workos_user_id: 'user_canonical' }])); + } + throw new Error(`Unexpected pool SQL: ${sql}`); + }); mocks.clientQuery.mockImplementation((sql: string) => { if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') return Promise.resolve(queryResult()); if (sql.includes('SELECT id FROM working_groups')) return Promise.resolve(queryResult([{ id: 'wg_aao_admin' }])); @@ -74,6 +102,7 @@ describe('AAO site-admin membership audit transaction', () => { }); it('rolls the membership write back when the audit insert fails', async () => { + mocks.poolQuery.mockResolvedValue(queryResult()); mocks.clientQuery.mockImplementation((sql: string) => { if (sql.includes('INSERT INTO aao_admin_access_events')) return Promise.reject(new Error('audit storage unavailable')); if (sql === 'BEGIN' || sql === 'ROLLBACK') return Promise.resolve(queryResult()); @@ -95,6 +124,7 @@ describe('AAO site-admin membership audit transaction', () => { }); it('rolls an active-membership delete back when its revoke audit insert fails', async () => { + mocks.poolQuery.mockResolvedValue(queryResult()); mocks.clientQuery.mockImplementation((sql: string) => { if (sql.includes('INSERT INTO aao_admin_access_events')) return Promise.reject(new Error('audit storage unavailable')); if (sql === 'BEGIN' || sql === 'ROLLBACK') return Promise.resolve(queryResult()); diff --git a/server/tests/unit/aao-admin-status.test.ts b/server/tests/unit/aao-admin-status.test.ts index 44757d9e91a..f4ec874511c 100644 --- a/server/tests/unit/aao-admin-status.test.ts +++ b/server/tests/unit/aao-admin-status.test.ts @@ -45,6 +45,13 @@ describe('site-admin access decisions', () => { else process.env.ADMIN_EMAILS = originalAdminEmails; }); + it('does not cache positive membership decisions across requests', async () => { + mocks.isMember.mockResolvedValueOnce(true).mockResolvedValueOnce(false); + await expect(isWebUserAAOAdmin('user_admin')).resolves.toBe(true); + await expect(isWebUserAAOAdmin('user_admin')).resolves.toBe(false); + expect(mocks.isMember).toHaveBeenCalledTimes(2); + }); + it('makes a grant immediately visible to an independent replica after its prior denial', async () => { mocks.isMember.mockResolvedValueOnce(false).mockResolvedValueOnce(true); diff --git a/server/tests/unit/addie-admin-global-auth.test.ts b/server/tests/unit/addie-admin-global-auth.test.ts index 5ee44aab4d8..b3b0bf57234 100644 --- a/server/tests/unit/addie-admin-global-auth.test.ts +++ b/server/tests/unit/addie-admin-global-auth.test.ts @@ -17,6 +17,7 @@ const mocks = vi.hoisted(() => ({ getModelExecutionReadiness: vi.fn(), getRouterShadowSummary: vi.fn(), serveHtmlWithConfig: vi.fn(), + readCredentialAuthorizationLifecycle: vi.fn(), })); vi.hoisted(() => { @@ -53,6 +54,11 @@ vi.mock('../../src/db/client.js', async (importOriginal) => ({ getPool: () => ({ query: mocks.poolQuery }), })); +vi.mock('../../src/db/authorization-epoch-db.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/db/authorization-epoch-db.js')>()), + readCredentialAuthorizationLifecycle: mocks.readCredentialAuthorizationLifecycle, +})); + vi.mock('../../src/db/working-group-db.js', () => ({ WorkingGroupDatabase: class WorkingGroupDatabase { getWorkingGroupIdBySlug = mocks.getAdminWorkingGroupIdBySlug; @@ -124,6 +130,20 @@ describe('Addie real global-admin boundary', () => { mocks.resolveEffectiveMembership.mockResolvedValue({ is_member: true }); mocks.checkPlatformBanForApiKey.mockResolvedValue({ banned: false }); mocks.checkPlatformBan.mockResolvedValue({ banned: false }); + mocks.readCredentialAuthorizationLifecycle.mockImplementation((workosUserId: string) => + Promise.resolve({ + status: 'active', + snapshot: { + workos_user_id: workosUserId, + email: 'sso-admin@example.test', + first_name: 'SSO', + last_name: 'Admin', + identity_id: '00000000-0000-4000-8000-000000000001', + primary_workos_user_id: workosUserId, + fingerprint: '', + }, + }), + ); mocks.getAdminWorkingGroupIdBySlug.mockResolvedValue('wg_aao_admin'); mocks.isAdminGroupMember.mockResolvedValue(true); mocks.poolQuery.mockImplementation((sql: string) => { @@ -257,6 +277,55 @@ describe('Addie real global-admin boundary', () => { expect(mocks.getWebConversations).toHaveBeenCalledOnce(); }); + it.each([true, false])('keeps credential admin authority %s after lifecycle routing to a linked primary', async (credentialIsAdmin) => { + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue({ + status: 'active', + snapshot: { + workos_user_id: 'user_sso_admin', + email: 'sso-admin@example.test', + first_name: 'SSO', + last_name: 'Admin', + identity_id: '00000000-0000-4000-8000-000000000001', + primary_workos_user_id: 'user_linked_primary', + fingerprint: '', + }, + }); + mocks.isAdminGroupMember.mockImplementation(async (_groupId, userId) => + userId === 'user_sso_admin' ? credentialIsAdmin : !credentialIsAdmin, + ); + + // The second request uses the session cache and must retain the same + // authenticated credential for the parent's fresh authority lookup. + for (let attempt = 0; attempt < 2; attempt++) { + const response = await request(app) + .get('/api/admin/addie/conversations') + .set('Cookie', `wos-session=linked-authority-${credentialIsAdmin}`); + expect(response.status).toBe(credentialIsAdmin ? 200 : 403); + } + expect(mocks.isAdminGroupMember.mock.calls).toEqual([ + ['wg_aao_admin', 'user_sso_admin'], + ['wg_aao_admin', 'user_sso_admin'], + ]); + expect(mocks.getWebConversations).toHaveBeenCalledTimes(credentialIsAdmin ? 2 : 0); + }); + + it.each(['deleted_or_quarantined', 'missing_primary'] as const)( + 'refuses a warmed administrator session after the credential becomes %s', + async (reason) => { + const cookie = `wos-session=terminal-admin-${reason}`; + const warm = await request(app).get('/api/admin/addie/conversations').set('Cookie', cookie); + expect(warm.status).toBe(200); + mocks.getWebConversations.mockClear(); + mocks.isAdminGroupMember.mockClear(); + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue({ status: 'terminal', reason }); + + const denied = await request(app).get('/api/admin/addie/conversations').set('Cookie', cookie); + expect(denied.status).toBe(401); + expect(mocks.isAdminGroupMember).not.toHaveBeenCalled(); + expect(mocks.getWebConversations).not.toHaveBeenCalled(); + }, + ); + it('exposes model execution readiness only through the global-admin boundary', async () => { const denied = await request(app) .get('/api/admin/addie/threads/model-execution-provenance-readiness?hours=48&minimum_thread_message_samples=200') diff --git a/server/tests/unit/authorization-epoch-session.test.ts b/server/tests/unit/authorization-epoch-session.test.ts index 0ad33ee6e2a..b167fdad474 100644 --- a/server/tests/unit/authorization-epoch-session.test.ts +++ b/server/tests/unit/authorization-epoch-session.test.ts @@ -16,11 +16,14 @@ const mocks = vi.hoisted(() => ({ loadSealedSession: vi.fn(), checkPlatformBan: vi.fn(), checkPlatformBanForApiKey: vi.fn(), - getAuthorizationFingerprint: vi.fn(), + readCredentialAuthorizationLifecycle: vi.fn(), poolQuery: vi.fn(), + verifyWorkOSJWT: vi.fn(), })); vi.hoisted(() => { + delete process.env.DEV_USER_EMAIL; + delete process.env.DEV_USER_ID; process.env.WORKOS_API_KEY = process.env.WORKOS_API_KEY ?? 'sk_test'; process.env.WORKOS_CLIENT_ID = process.env.WORKOS_CLIENT_ID ?? 'client_test'; process.env.WORKOS_COOKIE_PASSWORD = @@ -46,7 +49,7 @@ vi.mock('../../src/db/bans-db.js', () => ({ })); vi.mock('../../src/db/authorization-epoch-db.js', () => ({ - getAuthorizationFingerprint: mocks.getAuthorizationFingerprint, + readCredentialAuthorizationLifecycle: mocks.readCredentialAuthorizationLifecycle, bumpAuthorizationEpochs: vi.fn(), })); @@ -56,7 +59,12 @@ vi.mock('../../src/db/client.js', () => ({ isDatabaseInitialized: () => true, })); -import { requireAuth } from '../../src/middleware/auth.js'; +vi.mock('../../src/auth/workos-jwt.js', () => ({ + looksLikeJWT: vi.fn(() => true), + verifyWorkOSJWT: mocks.verifyWorkOSJWT, +})); + +import { optionalAuth, requireAuth, validateWorkOSBearerJWT } from '../../src/middleware/auth.js'; const SESSION_USER = { id: 'user_epoch_primary', @@ -87,6 +95,27 @@ function makeResponse(): Response { } as unknown as Response; } +function makeBearerRequest(token: string): Request { + return { + headers: { authorization: `Bearer ${token}` }, + } as unknown as Request; +} + +function activeLifecycle(fingerprint: string) { + return { + status: 'active' as const, + snapshot: { + workos_user_id: SESSION_USER.id, + email: SESSION_USER.email, + first_name: SESSION_USER.firstName, + last_name: SESSION_USER.lastName, + identity_id: 'identity_epoch', + primary_workos_user_id: SESSION_USER.id, + fingerprint, + }, + }; +} + describe('persisted authorization epoch gates the session cache', () => { beforeEach(() => { vi.clearAllMocks(); @@ -113,7 +142,15 @@ describe('persisted authorization epoch gates the session cache', () => { authenticate: mocks.authenticate, refresh: vi.fn(), }); - mocks.getAuthorizationFingerprint.mockResolvedValue('user_epoch_primary:1'); + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue( + activeLifecycle('user_epoch_primary:1'), + ); + mocks.verifyWorkOSJWT.mockResolvedValue({ + sub: SESSION_USER.id, + email: SESSION_USER.email, + expiresAt: Math.floor(Date.now() / 1000) + 3600, + isM2M: false, + }); }); it('serves the cached session while the fingerprint is unchanged', async () => { @@ -136,7 +173,9 @@ describe('persisted authorization epoch gates the session cache', () => { expect(mocks.authenticate).toHaveBeenCalledTimes(1); // A binding mutation bumped the credential's epoch on another instance. - mocks.getAuthorizationFingerprint.mockResolvedValue('user_epoch_primary:2'); + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue( + activeLifecycle('user_epoch_primary:2'), + ); await requireAuth(makeRequest(cookie), makeResponse(), next); @@ -151,11 +190,134 @@ describe('persisted authorization epoch gates the session cache', () => { await requireAuth(makeRequest(cookie), makeResponse(), next); expect(mocks.authenticate).toHaveBeenCalledTimes(1); - mocks.getAuthorizationFingerprint.mockRejectedValueOnce(new Error('db down')); + mocks.readCredentialAuthorizationLifecycle.mockResolvedValueOnce({ status: 'unavailable' }); await requireAuth(makeRequest(cookie), makeResponse(), next); expect(next).toHaveBeenCalledTimes(2); expect(mocks.authenticate).toHaveBeenCalledTimes(2); }); + + it('denies an epoch-0 credential on replica B immediately after replica A records deletion', async () => { + const cookie = `sealed-epoch-zero-deleted-${Date.now()}`; + const next = vi.fn() as NextFunction; + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue(activeLifecycle('')); + + await requireAuth(makeRequest(cookie), makeResponse(), next); + expect(next).toHaveBeenCalledTimes(1); + expect(mocks.authenticate).toHaveBeenCalledTimes(1); + + // Replica A committed a non-cascading deletion tombstone. Replica B still + // has its own cold epoch-0 session entry, but the next hit sees the marker. + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue({ + status: 'terminal', reason: 'deleted_or_quarantined', + }); + const response = makeResponse(); + await requireAuth(makeRequest(cookie), response, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(mocks.authenticate).toHaveBeenCalledTimes(2); + expect(response.status).toHaveBeenCalledWith(401); + }); + + it('does not serve replica-B cached authority when deletion-marker lookup is unavailable', async () => { + const cookie = `sealed-deletion-marker-outage-${Date.now()}`; + const next = vi.fn() as NextFunction; + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue(activeLifecycle('')); + + await requireAuth(makeRequest(cookie), makeResponse(), next); + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue({ status: 'unavailable' }); + const response = makeResponse(); + await requireAuth(makeRequest(cookie), response, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(mocks.authenticate).toHaveBeenCalledTimes(2); + expect(response.status).toHaveBeenCalledWith(503); + }); + + it('invalidates replica B through persistence when only replica A receives local eviction', async () => { + vi.resetModules(); + const replicaA = await import('../../src/middleware/auth.js'); + vi.resetModules(); + const replicaB = await import('../../src/middleware/auth.js'); + const cookieA = `sealed-replica-a-${Date.now()}`; + const cookieB = `sealed-replica-b-${Date.now()}`; + const nextA = vi.fn() as NextFunction; + const nextB = vi.fn() as NextFunction; + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue(activeLifecycle('')); + + await replicaA.requireAuth(makeRequest(cookieA), makeResponse(), nextA); + await replicaB.requireAuth(makeRequest(cookieB), makeResponse(), nextB); + expect(mocks.authenticate).toHaveBeenCalledTimes(2); + + // The deletion process only has access to replica A's in-memory cache. + // Replica B retains its independent entry and must observe the durable + // deletion marker on its immediate next request, without waiting for TTL. + replicaA.invalidateSessionsForUsers([SESSION_USER.id]); + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue({ + status: 'terminal', reason: 'deleted_or_quarantined', + }); + const responseB = makeResponse(); + await replicaB.requireAuth(makeRequest(cookieB), responseB, nextB); + + expect(nextB).toHaveBeenCalledTimes(1); + expect(mocks.authenticate).toHaveBeenCalledTimes(3); + expect(responseB.status).toHaveBeenCalledWith(401); + }); + + it('denies an epoch-0 cached bearer JWT on the immediate hit after deletion', async () => { + const token = `header.${Date.now()}.signature`; + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue(activeLifecycle('')); + + await expect(validateWorkOSBearerJWT(makeBearerRequest(token))).resolves.toMatchObject({ + user: { id: SESSION_USER.id }, + }); + expect(mocks.verifyWorkOSJWT).toHaveBeenCalledTimes(1); + + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue({ + status: 'terminal', reason: 'deleted_or_quarantined', + }); + await expect(validateWorkOSBearerJWT(makeBearerRequest(token))).resolves.toBeNull(); + + expect(mocks.verifyWorkOSJWT).toHaveBeenCalledTimes(2); + }); + + it('cannot poison requireAuth from optionalAuth with a deleted sealed credential', async () => { + const cookie = `sealed-optional-poison-${Date.now()}`; + const optionalRequest = makeRequest(cookie); + const optionalNext = vi.fn() as NextFunction; + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue({ + status: 'terminal', reason: 'deleted_or_quarantined', + }); + + await optionalAuth(optionalRequest, makeResponse(), optionalNext); + expect(optionalNext).toHaveBeenCalledTimes(1); + expect(optionalRequest.user).toBeUndefined(); + + const requiredResponse = makeResponse(); + const requiredNext = vi.fn() as NextFunction; + await requireAuth(makeRequest(cookie), requiredResponse, requiredNext); + + expect(mocks.authenticate).toHaveBeenCalledTimes(2); + expect(requiredNext).not.toHaveBeenCalled(); + expect(requiredResponse.status).toHaveBeenCalledWith(401); + }); + + it('cannot serve optionalAuth from a requireAuth cache after deletion', async () => { + const cookie = `sealed-required-poison-${Date.now()}`; + const requiredNext = vi.fn() as NextFunction; + await requireAuth(makeRequest(cookie), makeResponse(), requiredNext); + expect(requiredNext).toHaveBeenCalledTimes(1); + + mocks.readCredentialAuthorizationLifecycle.mockResolvedValue({ + status: 'terminal', reason: 'deleted_or_quarantined', + }); + const optionalRequest = makeRequest(cookie); + const optionalNext = vi.fn() as NextFunction; + await optionalAuth(optionalRequest, makeResponse(), optionalNext); + + expect(mocks.authenticate).toHaveBeenCalledTimes(2); + expect(optionalNext).toHaveBeenCalledTimes(1); + expect(optionalRequest.user).toBeUndefined(); + }); }); diff --git a/server/tests/unit/login-name-preservation.test.ts b/server/tests/unit/login-name-preservation.test.ts index 81de4fa45c5..5c3e77c3c28 100644 --- a/server/tests/unit/login-name-preservation.test.ts +++ b/server/tests/unit/login-name-preservation.test.ts @@ -18,6 +18,9 @@ import * as path from 'path'; const SRC_DIR = path.resolve(__dirname, '../../src'); const HTTP_FILE = path.join(SRC_DIR, 'http.ts'); +const IDENTITY_DB_FILE = path.join(SRC_DIR, 'db/identity-db.ts'); +const WEBHOOK_FILE = path.join(SRC_DIR, 'routes/workos-webhooks.ts'); +const MCP_OAUTH_FILE = path.join(SRC_DIR, 'mcp/oauth-provider.ts'); const MEMBERSHIP_DB_FILE = path.join(SRC_DIR, 'db/membership-db.ts'); const ADMIN_ORGS_FILE = path.join(SRC_DIR, 'routes/admin/organizations.ts'); @@ -55,7 +58,14 @@ function findUnconditionalNameOverwrites(filePath: string): string[] { describe('auth callback name preservation', () => { const httpContent = fs.readFileSync(HTTP_FILE, 'utf-8'); - const authCallbackSection = extractAuthCallbackUpsert(httpContent); + const identityDbContent = fs.readFileSync(IDENTITY_DB_FILE, 'utf-8'); + const authCallbackSection = extractCredentialEventUpsert(identityDbContent); + + it('routes the auth callback through the credential-event upsert', () => { + const callback = extractAuthCallback(httpContent); + expect(callback).toContain('upsertWorkosUserInCredentialEvent'); + expect(callback).toContain("'preserve_existing'"); + }); it('should not unconditionally overwrite first_name on login', () => { expect(authCallbackSection).not.toContain('first_name = EXCLUDED.first_name'); @@ -69,6 +79,19 @@ describe('auth callback name preservation', () => { expect(authCallbackSection).toMatch(/COALESCE.*users\.first_name.*EXCLUDED\.first_name/s); expect(authCallbackSection).toMatch(/COALESCE.*users\.last_name.*EXCLUDED\.last_name/s); }); + + it('keeps login and provider-event name precedence explicit at every shared-helper caller', () => { + const webhookContent = fs.readFileSync(WEBHOOK_FILE, 'utf-8'); + const mcpOauthContent = fs.readFileSync(MCP_OAUTH_FILE, 'utf-8'); + expect(webhookContent).toMatch( + /upsertWorkosUserInCredentialEvent\([\s\S]*?'provider_authoritative'/, + ); + expect(mcpOauthContent).toMatch( + /upsertWorkosUserInCredentialEvent\([\s\S]*?'preserve_existing'/, + ); + expect(identityDbContent).toContain("'provider_authoritative'"); + expect(identityDbContent).toContain("'preserve_existing'"); + }); }); describe('/api/me name source', () => { @@ -109,12 +132,22 @@ describe('no unconditional name overwrites anywhere in runtime code', () => { // --- helpers --- -function extractAuthCallbackUpsert(content: string): string { +function extractAuthCallback(content: string): string { const callbackStart = content.indexOf("'/auth/callback'"); if (callbackStart === -1) throw new Error('Could not find /auth/callback handler'); - const insertStart = content.indexOf('INSERT INTO users', callbackStart); - if (insertStart === -1) throw new Error('Could not find INSERT INTO users in auth callback'); + const callbackEnd = content.indexOf("// GET /auth/logout", callbackStart); + if (callbackEnd === -1) throw new Error('Could not find end of /auth/callback handler'); + + return content.slice(callbackStart, callbackEnd); +} + +function extractCredentialEventUpsert(content: string): string { + const helperStart = content.indexOf('export async function upsertWorkosUserInCredentialEvent'); + if (helperStart === -1) throw new Error('Could not find credential-event user upsert'); + + const insertStart = content.indexOf('INSERT INTO users', helperStart); + if (insertStart === -1) throw new Error('Could not find INSERT INTO users in credential-event upsert'); const onConflict = content.indexOf('ON CONFLICT', insertStart); if (onConflict === -1) throw new Error('Could not find ON CONFLICT in auth callback upsert'); diff --git a/server/tests/unit/native-auth-http-wiring.test.ts b/server/tests/unit/native-auth-http-wiring.test.ts index 29642c6e0e3..456d4e8150a 100644 --- a/server/tests/unit/native-auth-http-wiring.test.ts +++ b/server/tests/unit/native-auth-http-wiring.test.ts @@ -72,6 +72,27 @@ vi.mock('../../src/db/client.js', () => ({ query: mocks.query, })); +vi.mock('../../src/db/identity-db.js', async (importOriginal) => { + const actual = await importOriginal<typeof import('../../src/db/identity-db.js')>(); + return { + ...actual, + withCredentialCreationEventMutation: vi.fn(async ( + _userId: string, + mutation: (client: { query: typeof mocks.query }) => Promise<unknown>, + ) => ({ + applied: true, + value: await mutation({ query: mocks.query }), + })), + upsertWorkosUserInCredentialEvent: vi.fn(async ( + client: { query: typeof mocks.query }, + user: { id: string }, + ) => client.query( + 'INSERT INTO users (workos_user_id) VALUES ($1) ON CONFLICT DO NOTHING', + [user.id], + )), + }; +}); + vi.mock('../../src/db/migrate.js', () => ({ runMigrations: vi.fn().mockResolvedValue(undefined), })); @@ -355,7 +376,10 @@ describe('native OAuth HTTPServer wiring', () => { slack_user_id: 'U123', workos_user_id: null, } as never); - const mapUser = vi.spyOn(SlackDatabase.prototype, 'mapUser').mockResolvedValueOnce(undefined); + const mapUser = vi.spyOn(SlackDatabase.prototype, 'mapUser').mockResolvedValueOnce({ + slack_user_id: 'U123', + workos_user_id: 'user_1', + } as never); server = new HTTPServer(); const response = await request(appFor(server)).get('/auth/callback').query({ @@ -404,7 +428,10 @@ describe('native OAuth HTTPServer wiring', () => { slack_user_id: 'U123', workos_user_id: null, } as never); - vi.spyOn(SlackDatabase.prototype, 'mapUser').mockResolvedValueOnce(undefined); + vi.spyOn(SlackDatabase.prototype, 'mapUser').mockResolvedValueOnce({ + slack_user_id: 'U123', + workos_user_id: 'user_1', + } as never); mocks.sendAccountLinkedMessage.mockRejectedValueOnce(new Error('synthetic Slack rejection')); server = new HTTPServer(); diff --git a/server/tests/unit/network-health-security.test.ts b/server/tests/unit/network-health-security.test.ts index f4d82ed4634..3c134c8f98a 100644 --- a/server/tests/unit/network-health-security.test.ts +++ b/server/tests/unit/network-health-security.test.ts @@ -20,6 +20,7 @@ const mocks = vi.hoisted(() => ({ upsertAlertRule: vi.fn(), resolveAlert: vi.fn(), serveHtmlWithConfig: vi.fn(), + readCredentialAuthorizationLifecycle: vi.fn(), })); vi.hoisted(() => { @@ -56,6 +57,11 @@ vi.mock('../../src/db/client.js', async (importOriginal) => ({ getPool: () => ({ query: mocks.poolQuery }), })); +vi.mock('../../src/db/authorization-epoch-db.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/db/authorization-epoch-db.js')>()), + readCredentialAuthorizationLifecycle: mocks.readCredentialAuthorizationLifecycle, +})); + vi.mock('../../src/addie/mcp/admin-tools.js', () => ({ isWebUserAAOAdmin: mocks.isWebUserAAOAdmin, })); @@ -196,6 +202,20 @@ describe('network-health global authorization boundary', () => { mocks.resolveEffectiveMembership.mockResolvedValue({ is_member: true }); mocks.checkPlatformBanForApiKey.mockResolvedValue({ banned: false }); mocks.checkPlatformBan.mockResolvedValue({ banned: false }); + mocks.readCredentialAuthorizationLifecycle.mockImplementation((workosUserId: string) => + Promise.resolve({ + status: 'active', + snapshot: { + workos_user_id: workosUserId, + email: 'user@example.test', + first_name: 'Regular', + last_name: 'User', + identity_id: '00000000-0000-4000-8000-000000000001', + primary_workos_user_id: workosUserId, + fingerprint: '', + }, + }), + ); mocks.isWebUserAAOAdmin.mockResolvedValue(false); mocks.poolQuery.mockImplementation((sql: string) => { if (sql.includes('FROM users')) { diff --git a/server/tests/unit/working-group-admin-global-auth.test.ts b/server/tests/unit/working-group-admin-global-auth.test.ts index 0a6b790198d..7c2f4eff7b4 100644 --- a/server/tests/unit/working-group-admin-global-auth.test.ts +++ b/server/tests/unit/working-group-admin-global-auth.test.ts @@ -23,6 +23,7 @@ const mocks = vi.hoisted(() => ({ removeMembership: vi.fn(), invalidateMemberContextCache: vi.fn(), invalidateWebAdminStatusCache: vi.fn(), + readCredentialAuthorizationLifecycle: vi.fn(), })); vi.hoisted(() => { @@ -59,6 +60,11 @@ vi.mock('../../src/db/client.js', async (importOriginal) => ({ getPool: () => ({ query: mocks.poolQuery }), })); +vi.mock('../../src/db/authorization-epoch-db.js', async (importOriginal) => ({ + ...(await importOriginal<typeof import('../../src/db/authorization-epoch-db.js')>()), + readCredentialAuthorizationLifecycle: mocks.readCredentialAuthorizationLifecycle, +})); + vi.mock('../../src/addie/mcp/admin-tools.js', async () => { const { isWebUserAAOAdmin } = await import( '../../src/addie/admin-status-lookup.js' @@ -131,6 +137,20 @@ describe('working-group real global-admin boundary', () => { mocks.resolveEffectiveMembership.mockResolvedValue({ is_member: true }); mocks.checkPlatformBanForApiKey.mockResolvedValue({ banned: false }); mocks.checkPlatformBan.mockResolvedValue({ banned: false }); + mocks.readCredentialAuthorizationLifecycle.mockImplementation((workosUserId: string) => + Promise.resolve({ + status: 'active', + snapshot: { + workos_user_id: workosUserId, + email: 'sso-admin@example.test', + first_name: 'SSO', + last_name: 'Admin', + identity_id: '00000000-0000-4000-8000-000000000001', + primary_workos_user_id: workosUserId, + fingerprint: '', + }, + }), + ); mocks.getWorkingGroupBySlug.mockImplementation((slug: string) => Promise.resolve(slug === 'aao-admin' ? { id: 'wg_aao_admin', slug: 'aao-admin' } diff --git a/server/tests/unit/workos-webhook-inactive-membership.test.ts b/server/tests/unit/workos-webhook-inactive-membership.test.ts index 766db18f53f..bcdc5e20ba9 100644 --- a/server/tests/unit/workos-webhook-inactive-membership.test.ts +++ b/server/tests/unit/workos-webhook-inactive-membership.test.ts @@ -15,7 +15,7 @@ describe('non-active organization_membership webhook handling', () => { )?.[0] ?? ''; it('routes non-active membership updates through local cache deletion only', () => { - expect(nonActiveBlock).toContain('deleteInactiveMembershipCache(membership)'); + expect(nonActiveBlock).toContain('deleteInactiveMembershipCache(membership, client)'); expect(nonActiveBlock).not.toContain('deleteMembership(membership)'); }); diff --git a/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts b/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts index 2a183583d4f..d8efec499f7 100644 --- a/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts +++ b/server/tests/unit/workos-webhook-user-deleted-wiring.test.ts @@ -11,6 +11,8 @@ const WEBHOOK_FILE = path.resolve(__dirname, '../../src/routes/workos-webhooks.t const webhookSource = fs.readFileSync(WEBHOOK_FILE, 'utf-8'); const DELETION_SERVICE_FILE = path.resolve(__dirname, '../../src/services/identity-credential-deletion.ts'); const deletionServiceSource = fs.readFileSync(DELETION_SERVICE_FILE, 'utf-8'); +const IDENTITY_DB_FILE = path.resolve(__dirname, '../../src/db/identity-db.ts'); +const identityDbSource = fs.readFileSync(IDENTITY_DB_FILE, 'utf-8'); describe('user.deleted containment wiring (#6827)', () => { const userDeletedBlockMatch = webhookSource.match( @@ -46,7 +48,49 @@ describe('user.deleted containment wiring (#6827)', () => { "deleteIdentityCredential(row.workos_user_id, 'sync_users_backfill')", ); expect(backfillBlock).toContain('upsertWorkosUserUnlessConfirmedDeleted(user)'); + expect(backfillBlock).toContain('shouldAbortBackfillDeletion(localUsers.rows.length, candidates.length)'); + expect(backfillBlock).toContain('deletion circuit breaker refused an unsafe batch'); expect(backfillBlock).not.toMatch(/DELETE FROM organization_memberships/); expect(backfillBlock).not.toMatch(/DELETE FROM users/); }); + + it('guards every authority-bearing provider writer with the deletion lock seam', () => { + for (const eventType of [ + 'organization_membership.created', + 'organization_membership.updated', + 'user.created', + 'user.updated', + ]) { + const eventStart = webhookSource.indexOf(`case '${eventType}'`); + const eventEnd = webhookSource.indexOf('\n case ', eventStart + 1); + const eventBlock = webhookSource.slice(eventStart, eventEnd); + expect(eventStart).toBeGreaterThan(-1); + expect(eventBlock).toContain(eventType === 'user.created' + ? 'withCredentialCreationEventMutation(' + : 'withActiveCredentialEventMutation('); + } + }); + + it('keeps guarded user and membership helpers on the supplied transaction client', () => { + expect(webhookSource).toContain( + 'credentialClient ?? pool, user.id, user.first_name, user.last_name', + ); + expect(webhookSource).toContain( + 'canAddSeat(membership.organization_id, seatType, client)', + ); + }); + + it('uses one credential-first lock order and fails closed after bounded retries', () => { + expect(identityDbSource).toContain('await lockCredentialThenIdentityMutation(client, workosUserId)'); + const sharedLock = identityDbSource.indexOf('async function lockCredentialThenIdentityMutation'); + const credentialLock = identityDbSource.indexOf('await lockCredentialMutations(client, [workosUserId])', sharedLock); + const bindingLock = identityDbSource.indexOf('FOR UPDATE', credentialLock); + const identityLock = identityDbSource.indexOf('FOR UPDATE OF i', bindingLock); + expect(credentialLock).toBeGreaterThan(sharedLock); + expect(bindingLock).toBeGreaterThan(credentialLock); + expect(identityLock).toBeGreaterThan(bindingLock); + expect(identityDbSource).toContain('CREDENTIAL_MUTATION_MAX_ATTEMPTS = 3'); + expect(identityDbSource).toContain('if (await hasConfirmedDeletionTombstone(client, workosUserId))'); + expect(identityDbSource).not.toContain('Credential event mutation retry bound exhausted; applying without lock'); + }); }); From 9fa7707c828be7d75dd1c41b12a1549121f1010e Mon Sep 17 00:00:00 2001 From: Brian O'Kelley <bokelley@scope3.com> Date: Mon, 14 Sep 2026 16:29:13 +0000 Subject: [PATCH 6/6] fix(db): use the narrowed membership transaction client --- server/src/db/membership-db.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/db/membership-db.ts b/server/src/db/membership-db.ts index 5f4cf9e37c5..049878031dc 100644 --- a/server/src/db/membership-db.ts +++ b/server/src/db/membership-db.ts @@ -135,7 +135,7 @@ export async function upsertOrganizationMembership( } return guarded.value; } - const db = externalClient ?? getPool(); + const db = externalClient; const result = await db.query<{ role: string }>( `INSERT INTO organization_memberships (