From 8a7f6083d4e1376a22eab3ca33551e5139a21544 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 13:10:20 -0400 Subject: [PATCH 01/42] feat(migrate): add custom sources and migrate sources `clerk migrate import --source` now also takes the path to a source you wrote, loaded at runtime and validated before use. An import run records a custom source's content hash, so an edited source is a different one when a re-run decides whether to continue. `clerk migrate sources` lists the built-in sources with what each brings across, or shows one in full. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/completion/__complete.ts | 6 +- .../cli-core/src/commands/migrate/README.md | 111 +++++++- .../src/commands/migrate/index.test.ts | 15 +- .../cli-core/src/commands/migrate/index.ts | 29 +- .../commands/migrate/lib/transform.test.ts | 22 ++ .../cli-core/src/commands/migrate/run.test.ts | 136 ++++++++++ packages/cli-core/src/commands/migrate/run.ts | 38 ++- .../src/commands/migrate/runs.test.ts | 10 + .../cli-core/src/commands/migrate/runs.ts | 17 +- .../src/commands/migrate/sources/list.test.ts | 172 ++++++++++++ .../src/commands/migrate/sources/list.ts | 202 ++++++++++++++ .../migrate/sources/load-custom.test.ts | 250 ++++++++++++++++++ .../commands/migrate/sources/load-custom.ts | 184 +++++++++++++ .../src/commands/migrate/sources/registry.ts | 66 ++++- .../commands/migrate/sources/sources.test.ts | 15 +- .../src/test/integration/completion.test.ts | 17 ++ 16 files changed, 1250 insertions(+), 40 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/sources/list.test.ts create mode 100644 packages/cli-core/src/commands/migrate/sources/list.ts create mode 100644 packages/cli-core/src/commands/migrate/sources/load-custom.test.ts create mode 100644 packages/cli-core/src/commands/migrate/sources/load-custom.ts diff --git a/packages/cli-core/src/commands/completion/__complete.ts b/packages/cli-core/src/commands/completion/__complete.ts index d709f04f6..ad53d1ba7 100644 --- a/packages/cli-core/src/commands/completion/__complete.ts +++ b/packages/cli-core/src/commands/completion/__complete.ts @@ -36,7 +36,10 @@ const HTTP_METHOD_COMPLETIONS: Completion[] = [ { name: "DELETE", description: "Delete resource" }, ]; -/** The built-in migrate sources. Kept off `.choices()` so an unknown key gets the source error. */ +/** + * The built-in migrate sources. `--source` also takes a path to a source you + * wrote, so it cannot use `.choices()`. + */ const SOURCE_COMPLETIONS: Completion[] = sources.map((entry) => ({ name: entry.key, description: entry.label, @@ -86,6 +89,7 @@ const KNOWN_OPTION_VALUES: Record = { * to root, e.g. "open dashboard". */ const KNOWN_POSITIONAL_COMPLETIONS: Record = { + "migrate sources": SOURCE_COMPLETIONS, "open dashboard": KNOWN_DASHBOARD_PATHS.map((path) => ({ name: path, description: "Dashboard subpath", diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 5ce1d74c2..ef4c66301 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -19,6 +19,7 @@ clerk migrate export [source] [-o ] [--json] clerk migrate import [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] clerk migrate runs [run-id] [--json] clerk migrate undo [--dry-run] [--yes] [--include-signed-in] [--json] +clerk migrate sources [source] [--json] clerk migrate help ``` @@ -682,7 +683,7 @@ clerk migrate import # a human is asked | Flag | Description | | --------------------------------------- | ------------------------------------------------------------------------------------------------- | | `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | -| `--source ` | Where the file came from: one of the [sources](#sources) | +| `--source ` | Where the file came from: a [source](#sources), or one you wrote | | `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | | `--allow-partial` | Import the users that pass, and record the rest as skipped | | `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | @@ -753,8 +754,8 @@ which skip the checks too. #### Re-running Running the same import again continues where it left off. The match is the -file's sha256, the source, and the instance ID; the latest matching import run -decides what happens: +file's sha256, the source (and a custom source's content hash), and the +instance ID; the latest matching import run decides what happens: | Latest match | Re-running does | | ----------------------------------------- | ---------------------------------------------------------------------- | @@ -1103,6 +1104,26 @@ With `writeFailed` or `stopped`, `run` is the undo run as it stands on disk, once one was written: its counts come from its lines, and `state` says how it stopped, as in `clerk migrate runs`. +### `clerk migrate sources` + +```sh +clerk migrate sources # every source, with what it carries +clerk migrate sources betterauth # one source in full +clerk migrate sources ./my-source.ts # a source you wrote +clerk migrate sources --json +``` + +| Flag | Description | +| ---------- | ---------------------------------------------------------------- | +| `[source]` | A built-in key, or the path to a source you wrote, to show fully | +| `--json` | The same data, on stdout | + +`sources` alone prints the table under [Sources](#sources). `sources ` shows one source in +full: its export command, what it carries with a note for each, where each +field lands (`encrypted_password → password`), its fixed defaults, and any +caveats. An unknown key exits 2 and lists the valid ones. There is no +intro/outro gutter: this reads a static registry rather than running anything. + ### `clerk migrate help` `clerk migrate help` and `clerk migrate --help` print the help for the @@ -1125,16 +1146,88 @@ brings across. Adding a platform is one file in `sources/` plus one line in | `supabase` | Supabase `auth.users` export | yes | no | partial | | `workos` | WorkOS User Management API | no | no | yes | -An unknown `--source` exits 2 with the list of valid keys. - There is no column for social sign-ins, because no source copies them and none -needs to. Enable the same providers in Clerk, and a user who signs in with one -is linked to their imported account by verified email. See +needs to. Every source shows the same note instead: enable the same providers in +Clerk, and a user who signs in with one is linked to their imported account by +verified email. See [account linking](https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking). The import's checks count the providers in a Supabase, Auth0 or Auth.js file and offer to enable any that are off. Auth0's come from each `user_id`'s prefix (`github|…`), so a user who linked a second provider counts under the first. +### `--source` + +`clerk migrate import` takes `--source `: + +- A value starting with `./`, `../` or `/`, or ending in `.ts`, `.js` or + `.mjs`, is loaded as a [custom source](#custom-sources). +- Anything else must be a built-in key. An unknown key exits 2 with the list of + valid keys. + +A file `clerk migrate export` wrote names its own source, so it needs none. + +### Custom sources + +Migrating from a platform with no built-in, without recompiling the CLI: + +```sh +clerk migrate import users.json --source ./my-platform.ts +``` + +The file lives in **your** project, not in the CLI, and is imported at runtime. +It exports the same shape the built-ins use — plain data, no imports, since +there is nothing in a compiled binary for your file to import from: + +```ts +export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform's admin console.", + transformer: { + account_ref: "userId", // required: becomes the Clerk user's external_id + contact_email: "email", + given: "firstName", + family: "lastName", + pw_bcrypt: "password", + }, + carries: { + passwords: { level: "yes", note: "bcrypt hashes from the pw_bcrypt column." }, + mfa: { level: "no", note: "Not exported." }, + metadata: { level: "no", note: "Not exported." }, + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { + if (!user.firstName) delete user.firstName; + }, +}; +``` + +TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, +`satisfies` and `as const` all work in a file the compiled binary imports. +Plain `.js` works too. + +An import run records a custom source's key and a hash of the file, so an +edited source counts as a different source. + +#### Validation + +The file is code the CLI executes, so its shape is checked before use and +rejected with the specific problem rather than crashing mid-pipeline: + +| Problem | Message | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Path does not exist | `No source file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | +| A hook is not a function | `postTransform must be a function when present` | + +The `userId` check is the load-bearing one: without it the import would run to +completion and create every user with no `external_id`, which is what makes a +migration re-runnable. + ### Metadata Metadata a user can edit on the source platform — Auth0's `user_metadata`, @@ -1167,6 +1260,7 @@ Better Auth's scrypt uses the hex salt string as the salt and a 64-byte key, which is what `scrypt_werkzeug` verifies once N, r and p are written inline. It also normalizes a password to NFKC before hashing, and Clerk does not, so a password whose NFKC form differs will not verify and that user resets it. +`clerk migrate sources betterauth` lists that caveat. A password Clerk cannot verify is **dropped, not rejected**: dropping it doesn't reject the user, who imports without it and can sign in another way or reset @@ -1233,7 +1327,8 @@ An export with no password hashes needs no parameters at all. What a source maps _onto_. Every user is validated against this schema before any request is made, so a field a source produces that is not listed here is dropped — Zod strips unknown keys — and never reaches Clerk. The checks -warn about each one (`Clerk won't store: …`). +warn about each one (`Clerk won't store: …`). Writing a custom source means +targeting these names exactly. The schema lives in `validator.ts`; adding a platform means adding a source, not editing it. diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index f798e640d..d3e5857c8 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -32,12 +32,13 @@ describe("registerMigrate", () => { expect(migrate?.description()).toContain("Migrate users"); }); - test("registers import, export, undo and runs", () => { + test("registers import, export, undo, runs and sources", () => { expect(findCommand(["migrate"])?.commands.map((cmd) => cmd.name())).toEqual([ "import", "export", "undo", "runs", + "sources", ]); }); @@ -158,6 +159,12 @@ describe("registerMigrate", () => { expect(runsCommand?.options.map((option) => option.long)).toEqual(["--json", "--runs-dir"]); }); + test("registers sources with an optional source and --json", () => { + const sourcesCommand = findCommand(["migrate", "sources"]); + expect(sourcesCommand?.registeredArguments[0]?.required).toBe(false); + expect(sourcesCommand?.options.map((option) => option.long)).toEqual(["--json"]); + }); + test("registers undo with a required run ID and its flags", () => { const undoCommand = findCommand(["migrate", "undo"]); expect(undoCommand?.registeredArguments[0]?.required).toBe(true); @@ -173,9 +180,9 @@ describe("registerMigrate", () => { ]); }); - // No `.choices()`: an unknown key reaches resolveSource, whose error lists - // the valid ones. Completion offers them through `KNOWN_OPTION_VALUES`. - test("--source accepts any value", () => { + // It also takes a path, so it cannot use `.choices()`: completion offers the + // built-in keys through `KNOWN_OPTION_VALUES` instead. + test("--source accepts any value, so a path to a source you wrote gets through", () => { const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === "--source"); expect(option?.argChoices).toBeUndefined(); }); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index e0c1728c1..a4bb4afed 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -8,6 +8,7 @@ import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; import { run } from "./run.ts"; import { runs } from "./runs.ts"; import { undo } from "./undo.ts"; +import { list as sources } from "./sources/list.ts"; export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = process.env): void { if (!isExperimentEnabled("migrate", env)) { @@ -51,6 +52,7 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce command: "clerk migrate undo 20260929-141502-a1b2", description: "Delete the users an import created", }, + { command: "clerk migrate sources", description: "What each source brings across" }, ]); // `-y` is read several layers down — by the credential-retry loop and the @@ -81,8 +83,8 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .description("Import users from an exported JSON or CSV file") .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") .option( - "--source ", - "Where the file came from. Not needed for a file from `clerk migrate export`", + "--source ", + "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`", ) .option("--dry-run", "Check the file against the instance, report, and write nothing") .option("--allow-partial", "Import the users that pass the checks, and skip the rest") @@ -129,6 +131,10 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce command: "clerk migrate import users.json --source clerk --allow-partial --yes", description: "Import a Clerk Dashboard export, skipping users that would be rejected", }, + { + command: "clerk migrate import users.json --source ./my-source.ts --yes", + description: "Import with a source you wrote", + }, ]) .action(async (input, _opts, cmd) => run({ @@ -184,4 +190,23 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .action(async (runId, _opts, cmd) => runs(runId, cmd.optsWithGlobals() as Parameters[1]), ); + + // A compiled binary has no source tree to grep, so the available mappings + // need a command rather than only appearing in the interactive picker. + migrateCommand + .command("sources") + .description("List the sources an import can read, or show one in full") + .argument("[source]", "A built-in source, or the path to a source you wrote") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate sources", description: "What each source brings across" }, + { + command: "clerk migrate sources betterauth", + description: "Where each field lands, how to export, and caveats", + }, + { command: "clerk migrate sources ./my-source.ts", description: "Check a source you wrote" }, + ]) + .action(async (source, _opts, cmd) => + sources(source, cmd.optsWithGlobals() as Parameters[1]), + ); } diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 561ad655e..766ae00e7 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import clerkSource from "../sources/clerk.ts"; import type { SourceEntry } from "../types.ts"; +import { __resetCustomSourcesForTesting, registerCustomSource } from "../sources/registry.ts"; import { consolidateClerkIdentifiers, flattenObjectSelectively, @@ -298,6 +299,27 @@ describe("loadUsersFromFile", () => { } }); + test("a custom preTransform's rows win over a CSV file", async () => { + registerCustomSource({ + ...clerkSource, + key: "rows-from-pretransform", + preTransform: (filePath) => ({ + filePath, + data: [{ id: "from-pretransform", primary_email_address: "p@x.dev" }], + }), + }); + try { + fs.writeFileSync( + path.join(workDir, "ignored.csv"), + "id,primary_email_address\nfile,f@x.dev\n", + ); + const { users } = await loadUsersFromFile("ignored.csv", "rows-from-pretransform"); + expect(users.map((user) => user.userId)).toEqual(["from-pretransform"]); + } finally { + __resetCustomSourcesForTesting(); + } + }); + test("rejects a JSON file that is not an array of users", async () => { fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 395360b4e..0d0577293 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -30,6 +30,7 @@ import { startRun, } from "./lib/run-store.ts"; import { explainErrors, run, validateRunOptions } from "./run.ts"; +import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; // After run.ts: imported first, signals.ts loads version.ts ahead of its Bun // macro here, and the file fails to load. import { _resetInterruptState, abortInFlight, beginInterrupt } from "../../lib/signals.ts"; @@ -2094,6 +2095,141 @@ describe("run", () => { }); }); + describe("--source ", () => { + const CUSTOM = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email", given: "firstName", pw: "password" }, + carries: { + passwords: { level: "yes", note: "bcrypt." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { if (!user.firstName) delete user.firstName; }, + };`; + + let customFile: string; + let customCounter = 0; + + beforeEach(() => { + // A fresh filename each time: dynamic import() caches by URL, so reusing + // one would silently return a previous test's module. + customFile = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, customFile), CUSTOM); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { account_ref: "mp_1", contact_email: "a@x.dev", given: "Ada", pw: BCRYPT }, + { account_ref: "mp_2", contact_email: "b@x.dev", given: "", pw: BCRYPT }, + ]), + ); + }); + + afterEach(() => { + __resetCustomSourcesForTesting(); + }); + + const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); + + // The registered key isn't something --source accepts; the path is. + test("the printed command names the source's path, not its key", async () => { + const error = (await run({ + input: "export.json", + source: customFile, + secretKey: "sk_test_x", + json: true, + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.examples?.[0]?.command).toBe( + `clerk migrate import export.json --source ${customFile} --secret-key --json --yes`, + ); + }); + + test("imports through a user-authored source", async () => { + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_1", + "mp_2", + ]); + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("source from"); + }); + + test("applies the custom source's defaults and postTransform", async () => { + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ first_name: "Ada", password_hasher: "bcrypt" }); + // postTransform dropped the empty given name rather than sending "". + expect("first_name" in (bodies[1] ?? {})).toBe(false); + }); + + // An edited source is a different source, so the run records which one. + test("records the custom source's content hash on the run", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + + const [record] = listRuns(runsDir()); + expect(record?.source).toBe("myplatform"); + expect(record?.sourceHash).toMatch(/^[0-9a-f]{64}$/); + }); + + test("an unknown built-in key is a usage error listing the valid ones", async () => { + await expect( + run({ input: "export.json", source: "okta", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/Unknown source "okta". Valid sources: clerk, auth0/); + expect(created()).toHaveLength(0); + }); + + test("fails before any request when the file is not there", async () => { + await expect( + run({ + input: "export.json", + source: "./nope.ts", + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/No source file at/); + expect(requests).toHaveLength(0); + }); + + test("fails before any request when the file is malformed", async () => { + const bad = `./bad-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, bad), + `export default { key: "x", label: "X", transformer: {} };`, + ); + + const error = (await run({ + input: "export.json", + source: bad, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + expect(error.message).toMatch(/no source field maps to `userId`/); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(requests).toHaveLength(0); + }); + + test("still requires a file", async () => { + await expect(run({ source: customFile, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /needs the file to import/, + ); + }); + }); + describe("per-platform imports", () => { /** One realistic record per platform, in that platform's export shape. */ const PLATFORMS: [string, unknown, string][] = [ diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 27d853609..ad7ae1c17 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -111,8 +111,12 @@ import { link } from "../link/index.ts"; export type MigrateRunOptions = { /** An export file, or the ID of the export run that wrote one. */ input?: string; - /** A built-in source key. */ + /** A built-in source key, or the path to a source you wrote. */ source?: string; + /** Content hash of a custom `--source`, set once it is loaded. */ + sourceHash?: string; + /** `--source` as typed, for printed commands: a custom source's path. Not a flag. */ + sourceArg?: string; /** The file to read, once `input` is resolved. Not a flag. */ file?: string; requirePassword?: boolean; @@ -140,7 +144,7 @@ export type MigrateRunOptions = { * Validates what a run needs before anything is read or sent. * * `--source` has already been resolved to a registered key by the time this - * runs. + * runs, custom sources included. * * @returns The source key and file path, both guaranteed present. */ @@ -150,7 +154,7 @@ export function validateRunOptions(options: MigrateRunOptions): { } { if (!options.source) { throwUsageError( - `Missing --source. Valid values: ${sourceKeys().join(", ")}.`, + `Missing --source. Valid values: ${sourceKeys().join(", ")}, or the path to a source you wrote.`, undefined, ERROR_CODE.USAGE_ERROR, [ @@ -375,14 +379,23 @@ function throwChangedExport(runId: string, file: string): never { } /** - * Resolves `--source` to a registered key. + * Resolves `--source` to a registered key, loading a custom source from its + * path so the rest of the run treats it exactly like a built-in. * * @throws UsageError for an unknown key. */ async function applySource(options: MigrateRunOptions): Promise { if (!options.source) return options; const resolved = await resolveSource(options.source); - return { ...options, source: resolved.key }; + if (resolved.path && !options.json) { + log.info(`Loaded the \`${resolved.key}\` source from ${options.source}.`); + } + return { + ...options, + source: resolved.key, + sourceArg: options.source, + ...(resolved.hash ? { sourceHash: resolved.hash } : {}), + }; } /** @@ -423,8 +436,8 @@ type ResumeCase = | { kind: "complete"; record: RunRecord }; /** - * Finds the latest import run of this file (by sha256), this source and this - * instance, and decides what a re-run does: + * Finds the latest import run of this file (by sha256), this source (and a + * custom source's hash) and this instance, and decides what a re-run does: * * - none, or undone: a new run * - interrupted: continue it, skipping the users it created @@ -441,6 +454,7 @@ function findResume( match: { sha256: string; source: string; + sourceHash?: string; instanceId: string; /** The key's stand-in ID, for a run recorded while Clerk could not name the instance. */ keyInstanceId: string; @@ -452,8 +466,9 @@ function findResume( const ofFile = importsOfFile(runs, match); const latest = ofFile.find( (record) => - record.target.instanceId === match.instanceId || - record.target.instanceId === match.keyInstanceId, + record.sourceHash === match.sourceHash && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId), ); if (!latest) { // Clerk did not name the instance (GET /v1/instance failed), so a run of @@ -668,7 +683,8 @@ function commandFor( ) { const input = fromExport ?? options.input ?? options.file ?? ""; const parts = ["clerk migrate import", quoteArg(input)]; - if (!fromExport && options.source) parts.push("--source", quoteArg(options.source)); + const source = options.sourceArg ?? options.source; + if (!fromExport && source) parts.push("--source", quoteArg(source)); if (options.newRun) parts.push("--new-run"); parts.push(...importFlags(options)); // Names, not `<…>`: pasted as is, a shell reads `` as a redirect. @@ -877,6 +893,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const match = { sha256, source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), instanceId: target.instanceId, keyInstanceId: keyInstanceId(secretKey), }; @@ -1248,6 +1265,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { kind: "import", target, source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), file: { path: filePath, sha256 }, ...(input.fromExport ? { fromExport: input.fromExport } : {}), ...(firebaseHash ? { firebaseHash } : {}), diff --git a/packages/cli-core/src/commands/migrate/runs.test.ts b/packages/cli-core/src/commands/migrate/runs.test.ts index eb2c4b6cb..655537637 100644 --- a/packages/cli-core/src/commands/migrate/runs.test.ts +++ b/packages/cli-core/src/commands/migrate/runs.test.ts @@ -406,6 +406,16 @@ describe("runs ", () => { expect(next[0]).toContain("to retry the users it didn't finish"); }); + // The record has the custom source's hash, not its path. + test("a partial import from a source you wrote names the source file to pass", async () => { + const record = partialImport({ source: "acme", sourceHash: "abc123" }); + + await runs(record.id, { runsDir, json: true }); + + const { next } = JSON.parse(captured.out) as { next: string[] }; + expect(next[0]).toContain(`clerk migrate import ${quoteArg(usersFile())} --source SOURCE_FILE`); + }); + test("an unknown ID is a usage error", async () => { await expect(runs("nope", { runsDir })).rejects.toThrow(/No run `nope`/); }); diff --git a/packages/cli-core/src/commands/migrate/runs.ts b/packages/cli-core/src/commands/migrate/runs.ts index 70660eaa1..3e14c0934 100644 --- a/packages/cli-core/src/commands/migrate/runs.ts +++ b/packages/cli-core/src/commands/migrate/runs.ts @@ -124,16 +124,19 @@ function continueCommand(record: RunRecord, carry: string[]): string | undefined const input = record.fromExport ?? record.file?.path; if (!input) return undefined; const parts = ["clerk migrate import", quoteArg(input)]; - if (!record.fromExport && record.source) parts.push("--source", quoteArg(record.source)); + // A source you wrote is a file the record knows only by its hash. A name, + // not `<…>`: pasted as is, a shell reads `` as a redirect. + if (record.sourceHash) parts.push("--source", "SOURCE_FILE"); + else if (!record.fromExport && record.source) parts.push("--source", quoteArg(record.source)); return withFlags(parts.join(" "), [...importFlags(record.flags), ...carry, "--yes"]); } /** * Whether importing the record's file again continues this run, rather than - * starting another or refusing: no newer import of that file and source may - * be for the same instance, the file is unchanged, and no undo of it exists. - * An unfinished undo makes an import refuse; a finished one that never marked - * the run makes it start a new one. + * starting another or refusing: no newer import of that file and source (and + * version of a source you wrote) may be for the same instance, the file is + * unchanged, and no undo of it exists. An unfinished undo makes an import + * refuse; a finished one that never marked the run makes it start a new one. */ function continuesThisRun(runsDir: string, record: RunRecord): boolean { const file = record.file; @@ -146,8 +149,8 @@ function continuesThisRun(runsDir: string, record: RunRecord): boolean { id === record.target.instanceId || isStandInInstanceId(id) || isStandInInstanceId(record.target.instanceId); - const newest = importsOfFile(runs, { sha256: file.sha256, source: record.source }).find((other) => - mayBeSameInstance(other.target.instanceId), + const newest = importsOfFile(runs, { sha256: file.sha256, source: record.source }).find( + (other) => other.sourceHash === record.sourceHash && mayBeSameInstance(other.target.instanceId), ); if (newest?.id !== record.id) return false; if (undosOf(runs, record.id).length > 0) return false; diff --git a/packages/cli-core/src/commands/migrate/sources/list.test.ts b/packages/cli-core/src/commands/migrate/sources/list.test.ts new file mode 100644 index 000000000..a28a89cd1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.test.ts @@ -0,0 +1,172 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { list, wrapText } from "./list.ts"; +import { ACCOUNT_LINKING_URL, sources } from "./registry.ts"; + +const captured = useCaptureLog(); + +/** `log.info` highlights backticked spans and the levels are coloured. */ +const plain = () => Bun.stripANSI(captured.err).replace(/\s+/g, " "); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-sources-"))); + process.chdir(workDir); + fs.writeFileSync( + path.join(workDir, "custom.ts"), + `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", mail: "email" }, + carries: { + passwords: { level: "partial", note: "Only for accounts made after 2020." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "yes", note: "All of it." }, + }, + };`, + ); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("sources", () => { + test.each([...sources])("lists $key with what it carries", async (entry) => { + await list(undefined); + const row = plain().match(new RegExp(`${entry.key} (\\w+) (\\w+) (\\w+)`)); + expect(row?.slice(1)).toEqual([ + entry.carries.passwords.level, + entry.carries.mfa.level, + entry.carries.metadata.level, + ]); + }); + + // No social column: every source shares one note. + test("prints the account-linking note once", async () => { + await list(undefined); + expect(captured.err.split(ACCOUNT_LINKING_URL)).toHaveLength(2); + }); + + // A compiled binary has no source tree to grep, so the way to extend it has + // to be discoverable from the list itself. + test("says how to add one", async () => { + await list(undefined); + expect(captured.err).toContain("--source ./my-source.ts"); + }); + + test("--json lists every built-in with its carries on stdout", async () => { + await list(undefined, { json: true }); + + const parsed = JSON.parse(captured.out) as { + sources: { key: string; carries: unknown }[]; + account_linking: string; + }; + expect(parsed.sources.map((entry) => entry.key)).toEqual(sources.map((entry) => entry.key)); + expect(parsed.sources[0]?.carries).toEqual(sources[0]?.carries); + expect(parsed.account_linking).toContain(ACCOUNT_LINKING_URL); + }); +}); + +describe("sources ", () => { + test("shows where each field lands, how to export, and what comes across", async () => { + await list("supabase"); + + expect(plain()).toContain("Export with `clerk migrate export supabase`"); + expect(plain()).toContain("encrypted_password → password"); + expect(plain()).toContain("Passwords yes"); + expect(captured.err).toContain(ACCOUNT_LINKING_URL); + }); + + // Supabase detects its hasher per user, so a source with a fixed one shows it. + test("shows a source's fixed field values", async () => { + await list("firebase"); + expect(plain()).toContain('passwordHasher is always "scrypt_firebase"'); + }); + + test("shows a source you wrote, by its path", async () => { + await list("./custom.ts"); + + expect(plain()).toContain("myplatform My Platform (custom — ./custom.ts)"); + expect(plain()).toContain("mail → email"); + expect(plain()).toContain("Only for accounts made after 2020."); + }); + + test("--json returns the detail on stdout", async () => { + await list("auth0", { json: true }); + + expect(JSON.parse(captured.out)).toMatchObject({ + key: "auth0", + export_command: "clerk migrate export auth0", + fields: { user_id: "userId" }, + carries: { passwords: { level: "partial" } }, + }); + }); + + test("an unknown source is a usage error naming the valid ones", async () => { + const error = (await list("okta").catch((caught: unknown) => caught)) as CliError; + expect(error).toBeInstanceOf(CliError); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Valid sources: clerk, auth0"); + }); +}); + +describe("human-mode frame", () => { + let originalMode: string | undefined; + + beforeAll(() => { + originalMode = process.env.CLERK_MODE; + process.env.CLERK_MODE = "human"; + }); + + afterAll(() => { + if (originalMode === undefined) delete process.env.CLERK_MODE; + else process.env.CLERK_MODE = originalMode; + }); + + // Reading a static registry is not a run: there is no progress to bracket, + // and the gutter's `│` would sit in front of every wrapped line. + test("prints no intro/outro gutter", async () => { + await list(undefined); + + expect(captured.err).not.toContain("┌"); + expect(captured.err).not.toContain("└"); + }); + + test("--json stays on stdout only", async () => { + await list(undefined, { json: true }); + + expect(() => JSON.parse(captured.out)).not.toThrow(); + expect(captured.err).toBe(""); + }); +}); + +describe("wrapText", () => { + test("breaks on whitespace within the width", () => { + expect(wrapText("one two three four", 9)).toEqual(["one two", "three", "four"]); + }); + + // `log.info` pairs backticks per line, so a span split across two lines + // leaves one unmatched backtick on each and colours the wrong half of both. + test("never breaks inside a backticked span", () => { + const lines = wrapText("Assumes an export of `SELECT id, name FROM users`.", 30); + + expect(lines).toContain("`SELECT id, name FROM users`."); + for (const line of lines) { + expect((line.match(/`/g) ?? []).length % 2).toBe(0); + } + }); + + test("gives an over-long word its own line rather than dropping it", () => { + expect(wrapText("short supercalifragilistic", 8)).toEqual(["short", "supercalifragilistic"]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts new file mode 100644 index 000000000..6955e37db --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -0,0 +1,202 @@ +/** + * `clerk migrate sources [source]` — which platforms an import can read, and + * what each one brings across. + * + * A compiled binary's users have no source tree to read the mappings in, so + * this is where they are shown. `sources` alone lists every source with its + * passwords, MFA and metadata at a glance. `sources ` shows one in + * full: what each field becomes, how to export from it, and its caveats. + */ + +import { bold, cyan, dim, green, red, yellow } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { exportPlatforms } from "../export/registry.ts"; +import type { Carry, CarryLevel, SourceEntry } from "../types.ts"; +import { ACCOUNT_LINKING_NOTE, resolveSource, sources as builtIns } from "./registry.ts"; + +export type SourcesOptions = { + json?: boolean; +}; + +const KINDS = [ + ["passwords", "Passwords"], + ["mfa", "MFA"], + ["metadata", "Metadata"], +] as const; + +/** + * Capped, not just measured: a description that rewrapped differently on every + * terminal makes two runs of the same command look like different output. 80 is + * the same width `--help` lays itself out at. + */ +const MAX_WIDTH = 80; + +function outputWidth(): number { + return Math.min(process.stderr.columns || MAX_WIDTH, MAX_WIDTH); +} + +/** + * A run of non-space characters, except that a backticked span counts as one + * character run even when it contains spaces. Keeps `SELECT a, b FROM users` + * whole: `log.info` pairs backticks per line, so a span broken across two lines + * leaves an unmatched backtick on each and colours the wrong half of both. + */ +const WORD = /(?:`[^`]*`|\S)+/g; + +/** + * Wraps on whitespace. Safe to measure raw because the backtick spans + * `log.info` highlights keep their backticks — the colour it adds is invisible + * to width, and nothing here is coloured before wrapping. + */ +export function wrapText(text: string, width: number): string[] { + const lines: string[] = []; + let line = ""; + + for (const word of text.match(WORD) ?? []) { + if (!line) line = word; + else if (line.length + 1 + word.length <= width) line += ` ${word}`; + else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + + return lines; +} + +function levelMark(level: CarryLevel): string { + if (level === "yes") return green("yes"); + if (level === "partial") return yellow("partial"); + return red("no"); +} + +/** The command that writes a file this source reads, when there is one. */ +function exportCommand(entry: SourceEntry): string | undefined { + const platform = exportPlatforms.find((candidate) => candidate.sourceKey === entry.key); + return platform ? `clerk migrate export ${platform.key}` : undefined; +} + +function toJson(entry: SourceEntry, custom?: string) { + return { + key: entry.key, + label: entry.label, + description: entry.description, + ...(custom ? { path: custom } : {}), + carries: entry.carries, + export_command: exportCommand(entry) ?? null, + fields: entry.transformer, + caveats: entry.caveats ?? [], + }; +} + +function printList(width: number): void { + for (const line of wrapText( + "A source maps one platform's export onto the fields Clerk imports. A file from " + + "`clerk migrate export` names its own; anything else takes `--source `.", + width, + )) { + log.info(line); + } + log.blank(); + + const keyWidth = Math.max(...builtIns.map((entry) => entry.key.length)) + 2; + log.info( + bold( + `${"SOURCE".padEnd(keyWidth)}${"PASSWORDS".padEnd(11)}${"MFA".padEnd(9)}${"METADATA".padEnd(10)}`, + ), + ); + for (const entry of builtIns) { + const cell = (carry: Carry, pad: number) => + levelMark(carry.level) + " ".repeat(Math.max(1, pad - carry.level.length)); + log.info( + `${cyan(entry.key.padEnd(keyWidth))}${cell(entry.carries.passwords, 11)}` + + `${cell(entry.carries.mfa, 9)}${levelMark(entry.carries.metadata.level)}`, + ); + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); + log.blank(); + log.info("Run `clerk migrate sources ` for what each field becomes."); + log.info("Migrating from something else? Write a source and pass --source ./my-source.ts."); +} + +function printDetail(entry: SourceEntry, width: number, custom?: string): void { + log.info(`${cyan(bold(entry.key))} ${entry.label}${custom ? dim(` (custom — ${custom})`) : ""}`); + for (const line of wrapText(entry.description, width)) log.info(line); + + const command = exportCommand(entry); + if (command) { + log.blank(); + log.info(`${bold("Export with")} \`${command}\``); + } + + log.blank(); + log.info(bold("What comes across")); + for (const [kind, label] of KINDS) { + const carry = entry.carries[kind]; + log.info(` ${label.padEnd(10)}${levelMark(carry.level)}`); + for (const line of wrapText(carry.note, width - 4)) log.info(` ${dim(line)}`); + } + + const fields = Object.entries(entry.transformer); + const fromWidth = Math.max(...fields.map(([from]) => from.length)) + 2; + log.blank(); + log.info(bold("Where each field lands")); + for (const [from, to] of fields) log.info(` ${from.padEnd(fromWidth)}→ ${to}`); + if (entry.defaults) { + for (const [field, value] of Object.entries(entry.defaults)) { + log.info(dim(` ${field} is always ${JSON.stringify(value)}`)); + } + } + + if (entry.caveats?.length) { + log.blank(); + log.info(bold("Caveats")); + for (const caveat of entry.caveats) { + const [first, ...rest] = wrapText(caveat, width - 4); + log.info(` - ${first ?? ""}`); + for (const line of rest) log.info(` ${line}`); + } + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); +} + +export async function list(key: string | undefined, options: SourcesOptions = {}): Promise { + const width = outputWidth(); + + if (key === undefined) { + if (options.json) { + log.data( + JSON.stringify( + { + sources: builtIns.map((entry) => toJson(entry)), + account_linking: ACCOUNT_LINKING_NOTE, + }, + null, + 2, + ), + ); + return; + } + printList(width); + return; + } + + // No gutter: this reads a static registry, it does not run anything. + const resolved = await resolveSource(key); + if (options.json) { + log.data( + JSON.stringify( + { ...toJson(resolved.entry, resolved.path), account_linking: ACCOUNT_LINKING_NOTE }, + null, + 2, + ), + ); + return; + } + printDetail(resolved.entry, width, resolved.path ? key : undefined); +} diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts new file mode 100644 index 000000000..513457375 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts @@ -0,0 +1,250 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { loadCustomSource, validateSource } from "./load-custom.ts"; +import { __resetCustomSourcesForTesting } from "./registry.ts"; + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-custom-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +afterEach(() => { + __resetCustomSourcesForTesting(); +}); + +/** + * Writes a source file with a unique name. + * + * Names must not repeat: a dynamic `import()` caches by URL, so reusing one + * would silently return the previous test's module. + */ +function writeSource(source: string, ext = "ts"): string { + const name = `custom-${counter++}.${ext}`; + fs.writeFileSync(path.join(workDir, name), source); + return `./${name}`; +} + +const VALID = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, +};`; + +describe("loadCustomSource", () => { + test("loads a user-authored TypeScript source", async () => { + const entry = await loadCustomSource(writeSource(VALID)); + + expect(entry).toMatchObject({ + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId", contact_email: "email" }, + }); + }); + + test("loads plain JavaScript too", async () => { + const entry = await loadCustomSource(writeSource(VALID, "js")); + expect(entry.key).toBe("myplatform"); + }); + + // The file is the user's own code; the CLI must transpile whatever they wrote. + test("transpiles TypeScript syntax the runtime has to strip", async () => { + const entry = await loadCustomSource( + writeSource(` + interface Entry { key: string; label: string; transformer: Record; carries: unknown } + const mapping = { my_id: "userId" } as const; + const custom: Entry = { key: "tsplatform", label: "TS", transformer: { ...mapping }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } }; + export default custom satisfies Entry; + `), + ); + expect(entry.key).toBe("tsplatform"); + }); + + test("carries the optional hooks through", async () => { + const entry = await loadCustomSource( + writeSource(`export default { + key: "hooked", label: "Hooked", + transformer: { id: "userId" }, + carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { user.firstName = "set"; }, + };`), + ); + + expect(entry.defaults).toEqual({ passwordHasher: "bcrypt" }); + const user: Record = {}; + entry.postTransform?.(user, {}); + expect(user.firstName).toBe("set"); + }); + + test("supplies a description when the author omitted one", async () => { + const entry = await loadCustomSource( + writeSource( + `export default { key: "bare", label: "Bare", transformer: { id: "userId" }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } };`, + ), + ); + expect(entry.description).toBe("Custom source"); + }); + + test("reports a path that is not there", async () => { + await expect(loadCustomSource("./nope.ts")).rejects.toThrow(/No source file at/); + }); + + test("reports a directory given instead of a file", async () => { + fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); + await expect(loadCustomSource("./adir")).rejects.toThrow(/is a directory/); + }); + + test("reports a file that does not parse, quoting the syntax error", async () => { + await expect(loadCustomSource(writeSource("export default { key: ,,, }"))).rejects.toThrow( + /Could not load/, + ); + }); + + test("reports a file that throws while loading", async () => { + await expect( + loadCustomSource(writeSource(`throw new Error("boom"); export default {};`)), + ).rejects.toThrow(/Could not load .*boom/s); + }); + + test("points at a named export when the default is missing", async () => { + const file = writeSource( + `export const myPlatform = { key: "x", label: "X", transformer: { a: "userId" } };`, + ); + + await expect(loadCustomSource(file)).rejects.toThrow( + /has no default export.*`myPlatform`.*did you mean `export default`/s, + ); + }); + + test("reports a missing default with no named exports to suggest", async () => { + await expect(loadCustomSource(writeSource("const unused = 1;"))).rejects.toThrow( + /has no default export\.$/m, + ); + }); +}); + +describe("validateSource", () => { + const valid = { + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + }; + + test("accepts a minimal valid entry", () => { + expect(validateSource(valid, "f.ts").key).toBe("myplatform"); + }); + + test.each([ + ["a null default export", null, /is null, not an object/], + ["a number default export", 42, /is number, not an object/], + ["a string default export", "nope", /is string, not an object/], + ])("rejects %s", (_label, value, expected) => { + expect(() => validateSource(value, "f.ts")).toThrow(expected); + }); + + test.each([ + ["key", { ...valid, key: undefined }], + ["key", { ...valid, key: "" }], + ["key", { ...valid, key: " " }], + ["key", { ...valid, key: 7 }], + ["label", { ...valid, label: undefined }], + ["label", { ...valid, label: "" }], + ])("rejects a bad %s naming the field", (field, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test("rejects a non-string description", () => { + expect(() => validateSource({ ...valid, description: 7 }, "f.ts")).toThrow( + /`description` must be a string/, + ); + }); + + test.each([ + ["missing", { ...valid, transformer: undefined }], + ["null", { ...valid, transformer: null }], + ["an array", { ...valid, transformer: [] }], + ["a string", { ...valid, transformer: "id" }], + ])("rejects a transformer mapping that is %s", (_label, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(/`transformer`|`transformer\./); + }); + + test("names the offending entry when a mapping target is not a field name", () => { + expect(() => + validateSource({ ...valid, transformer: { account_ref: "userId", bad: 7 } }, "f.ts"), + ).toThrow(/`transformer.bad` must map to a Clerk field name, got number/); + }); + + // Without it the import runs to completion and creates every user with no + // external_id — which is what makes a migration reversible. + test("rejects a mapping with no userId target", () => { + expect(() => validateSource({ ...valid, transformer: { a: "email" } }, "f.ts")).toThrow( + /no source field maps to `userId`/, + ); + }); + + test.each([ + ["defaults", { ...valid, defaults: "nope" }], + ["preTransform", { ...valid, preTransform: "nope" }], + ["postTransform", { ...valid, postTransform: 7 }], + ])("rejects a %s of the wrong type", (field, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test.each([["clerk"], ["auth0"], ["supabase"]])( + "rejects %s, which would shadow a built-in", + (key) => { + expect(() => + validateSource({ ...valid, key }, "f.ts", ["clerk", "auth0", "supabase"]), + ).toThrow(/already a built-in source/); + }, + ); + + // What a source brings across is the first thing `sources` shows, and only + // its author knows it. + test("requires carries", () => { + expect(() => validateSource({ ...valid, carries: undefined }, "f.ts")).toThrow( + /`carries` must say what the source brings across/, + ); + }); + + test.each([ + ["a level that is not yes, no or partial", { level: "maybe", note: "x" }], + ["no note", { level: "yes" }], + ])("rejects a carries entry with %s", (_label, mfa) => { + expect(() => validateSource({ ...valid, carries: { ...valid.carries, mfa } }, "f.ts")).toThrow( + /`carries.mfa` must be/, + ); + }); + + test("names the file in every message, so the author knows which one", () => { + expect(() => validateSource({}, "./their-file.ts")).toThrow(/\.\/their-file\.ts/); + }); + + test("raises CliError, so the global handler formats it", () => { + expect(() => validateSource({}, "f.ts")).toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts new file mode 100644 index 000000000..03aacebbb --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -0,0 +1,184 @@ +/** + * Loading a user-authored source at runtime. + * + * In the standalone migration-tool, supporting a new platform meant adding a + * file to `src/transformers/` and one line to the registry — the user had the + * source tree. A compiled binary has neither a source tree to edit nor a way + * for an end user to rebuild it, so `--source ` restores that + * extensibility by importing a file from the user's own project instead. + * + * **Verified before this was built on:** a `bun build --compile` executable can + * `import()` an arbitrary external `.ts` file at runtime, including TypeScript + * that needs transpiling. Bun's transpiler is part of the runtime, not only the + * bundler. Confirmed with a throwaway compiled binary on darwin-arm64, + * linux-arm64 (glibc), linux-arm64-musl and linux-x64. + * + * The file is user-supplied code the CLI executes, so its shape is validated + * up front and rejected with a specific message rather than crashing deep in + * the transform pipeline on a missing field. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import type { CarryLevel, SourceEntry } from "../types.ts"; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; + +const LEVELS: readonly CarryLevel[] = ["yes", "no", "partial"]; + +function invalid(problem: string, file: string): never { + throwUsageError(`${file} is not a valid source: ${problem}`, DOCS_URL); +} + +/** + * Checks a loaded value against the registry entry shape. + * + * Every failure names the specific field and what was wrong with it — the + * author is writing this file by hand against a shape they cannot see. + * + * @param file - Path as the user typed it, for the error message. + * @param reservedKeys - The built-in keys, which a custom source may not reuse. + */ +export function validateSource( + value: unknown, + file: string, + reservedKeys: readonly string[] = [], +): SourceEntry { + if (value === null || typeof value !== "object") { + invalid(`the default export is ${value === null ? "null" : typeof value}, not an object`, file); + } + + const entry = value as Record; + + for (const field of ["key", "label"] as const) { + if (typeof entry[field] !== "string" || entry[field].trim().length === 0) { + invalid(`\`${field}\` must be a non-empty string`, file); + } + } + + if (entry.description !== undefined && typeof entry.description !== "string") { + invalid("`description` must be a string when present", file); + } + + // Arrays are objects, and an author who wrote `transformer: []` should hear + // that rather than the downstream "no field maps to userId". + if ( + entry.transformer === null || + typeof entry.transformer !== "object" || + Array.isArray(entry.transformer) + ) { + invalid("`transformer` must be an object mapping source fields to Clerk fields", file); + } + + const mapping = entry.transformer as Record; + for (const [source, target] of Object.entries(mapping)) { + if (typeof target !== "string" || target.length === 0) { + invalid( + `\`transformer.${source}\` must map to a Clerk field name, got ${typeof target}`, + file, + ); + } + } + + // Without this the import runs to completion and creates every user with no + // external_id, which is what makes a migration re-runnable and reversible. + if (!Object.values(mapping).includes("userId")) { + invalid( + "no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id", + file, + ); + } + + if ( + entry.defaults !== undefined && + (entry.defaults === null || typeof entry.defaults !== "object" || Array.isArray(entry.defaults)) + ) { + invalid("`defaults` must be an object when present", file); + } + + // What a source brings across is the first thing `sources` shows, and the + // author is the only one who knows it. + const carries = entry.carries as Record | undefined; + if (!carries || typeof carries !== "object" || Array.isArray(carries)) { + invalid( + "`carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note }", + file, + ); + } + for (const kind of ["passwords", "mfa", "metadata"] as const) { + const carry = carries[kind] as { level?: unknown; note?: unknown } | undefined; + if (!carry || !LEVELS.includes(carry.level as CarryLevel) || typeof carry.note !== "string") { + invalid( + `\`carries.${kind}\` must be { level: "yes" | "no" | "partial", note: string }`, + file, + ); + } + } + + for (const hook of ["preTransform", "postTransform"] as const) { + if (entry[hook] !== undefined && typeof entry[hook] !== "function") { + invalid(`\`${hook}\` must be a function when present`, file); + } + } + + if (reservedKeys.includes(entry.key as string)) { + invalid( + `\`key\` is "${String(entry.key)}", which is already a built-in source. Choose another key`, + file, + ); + } + + return { + ...(entry as unknown as SourceEntry), + description: (entry.description as string | undefined) ?? "Custom source", + }; +} + +/** + * Imports and validates a user-authored source. + * + * @throws CliError when the path is missing, the module fails to load, or the + * exported value does not match the registry entry shape. + */ +export async function loadCustomSource( + file: string, + reservedKeys: readonly string[] = [], +): Promise { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No source file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + if (fs.statSync(resolved).isDirectory()) { + throwUsageError(`${resolved} is a directory, not a source file.`); + } + + let module: Record; + try { + // A file URL rather than a bare path: an absolute POSIX path happens to + // work, but a Windows path (`C:\...`) is not a valid import specifier. + module = (await import(Bun.pathToFileURL(resolved).href)) as Record; + } catch (error) { + throwUsageError( + `Could not load ${file}: ${(error as Error).message}\n` + + "The file must be valid JavaScript or TypeScript that this CLI can import.", + DOCS_URL, + ); + } + + if (module.default === undefined) { + // Point at what they probably meant rather than just restating the rule. + const named = Object.keys(module).filter((key) => key !== "default"); + const hint = + named.length > 0 + ? ` Found named export${named.length === 1 ? "" : "s"} ${named.map((n) => `\`${n}\``).join(", ")} — did you mean \`export default\`?` + : ""; + throwUsageError(`${file} has no default export.${hint}`, DOCS_URL); + } + + return validateSource(module.default, file, reservedKeys); +} diff --git a/packages/cli-core/src/commands/migrate/sources/registry.ts b/packages/cli-core/src/commands/migrate/sources/registry.ts index 25d376e71..226ff7065 100644 --- a/packages/cli-core/src/commands/migrate/sources/registry.ts +++ b/packages/cli-core/src/commands/migrate/sources/registry.ts @@ -1,12 +1,16 @@ /** * Source registry. * - * `migrate import` resolves `--source` here. + * `migrate import` resolves `--source` here, and `migrate sources` lists what + * each one brings across. * * To add a platform: create `sources/.ts` exporting a `SourceEntry`, * then add it to the array below. */ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; import { throwUsageError } from "../../../lib/errors.ts"; import type { SourceEntry } from "../types.ts"; import auth0Source from "./auth0.ts"; @@ -14,6 +18,7 @@ import authjsSource from "./authjs.ts"; import betterAuthSource from "./betterauth.ts"; import clerkSource from "./clerk.ts"; import firebaseSource from "./firebase.ts"; +import { loadCustomSource } from "./load-custom.ts"; import supabaseSource from "./supabase.ts"; import workosSource from "./workos.ts"; @@ -41,36 +46,83 @@ export const ACCOUNT_LINKING_NOTE = "Social sign-ins are not copied. Enable the same providers in Clerk, and a user who signs in " + `with one is linked to their imported account by verified email. See ${ACCOUNT_LINKING_URL}`; +/** + * A source loaded from a user's `--source ` for this invocation. + * + * Kept beside the built-ins rather than pushed into them, so the shipped list + * is never mutated. One invocation loads at most one, so this holding a + * single entry is the normal case; the array shape just avoids a special case + * in the lookups. + */ +const customSources: SourceEntry[] = []; + +export function registerCustomSource(entry: SourceEntry): void { + customSources.push(entry); +} + +/** Test-only: drops anything a previous test registered. */ +export function __resetCustomSourcesForTesting(): void { + customSources.length = 0; +} + +/** Built-ins plus whatever `--source ` loaded. */ +export function allSources(): SourceEntry[] { + return [...sources, ...customSources]; +} + /** The built-in keys, for tab-completion and error messages. */ export function sourceKeys(): string[] { return sources.map((entry) => entry.key); } /** - * Looks up a source by key. + * Looks up a source by key, custom ones included. * * @throws Error when no source is registered under that key. */ export function getSource(key: string): SourceEntry { - const source = sources.find((entry) => entry.key === key); + const source = allSources().find((entry) => entry.key === key); if (!source) { throw new Error(`Source not found for key: ${key}`); } return source; } -/** A resolved `--source`. */ -export type ResolvedSource = { key: string; entry: SourceEntry }; +/** A `--source` value that names a file rather than a built-in key. */ +export function isSourcePath(value: string): boolean { + return /^(\.\.?\/|\/)/.test(value) || /\.(ts|js|mjs)$/.test(value); +} + +/** A resolved `--source`: its key, and for a custom one, the file's content hash. */ +export type ResolvedSource = { key: string; entry: SourceEntry; path?: string; hash?: string }; /** - * Resolves `--source` to a built-in source. + * Resolves `--source`: a built-in key, or a path to a source you wrote. + * + * A custom source is keyed by a hash of its file as well as its key, so an + * edited source is a different source when a re-run decides whether to + * continue an earlier import. * * @throws UsageError for an unknown key, listing the valid ones. */ export async function resolveSource(value: string): Promise { + if (isSourcePath(value)) { + const entry = await loadCustomSource(value, sourceKeys()); + registerCustomSource(entry); + const resolved = path.resolve(process.cwd(), value); + const hash = createHash("sha256").update(fs.readFileSync(resolved)).digest("hex"); + return { key: entry.key, entry, path: resolved, hash }; + } + const entry = sources.find((candidate) => candidate.key === value); if (!entry) { - throwUsageError(`Unknown source "${value}". Valid sources: ${sourceKeys().join(", ")}.`); + throwUsageError( + `Unknown source "${value}". Valid sources: ${sourceKeys().join(", ")}.\n` + + "For a platform with no built-in, pass the path to a source you wrote, e.g. --source ./my-source.ts.", + undefined, + undefined, + [{ command: "clerk migrate sources", description: "List the built-in sources" }], + ); } return { key: entry.key, entry }; } diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index e3ad5c39d..1a4be8998 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -5,8 +5,8 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import { loadUsersFromFile, normalizeUserData, transformUsers } from "../lib/transform.ts"; import type { FirebaseHashConfig, User } from "../types.ts"; -import { getSource, sourceKeys, sources } from "./registry.ts"; import { isAuthJsVerified } from "./authjs.ts"; +import { getSource, isSourcePath, sourceKeys, sources } from "./registry.ts"; import { isVerified, toIsoDate } from "./shared.ts"; const FIREBASE_HASH: FirebaseHashConfig = { @@ -79,6 +79,19 @@ describe("registry", () => { }); }); +describe("isSourcePath", () => { + test.each([["./mine.ts"], ["../up/mine.js"], ["/abs/mine.mjs"], ["mine.ts"], ["dir/mine.js"]])( + "%s is a path", + (value) => { + expect(isSourcePath(value)).toBe(true); + }, + ); + + test.each([["clerk"], ["betterauth"], ["okta"]])("%s is a key", (value) => { + expect(isSourcePath(value)).toBe(false); + }); +}); + describe("isVerified", () => { // A CSV export stringifies everything, so the boolean style must read // "false" as false. Treating it as truthy would mark unconfirmed addresses diff --git a/packages/cli-core/src/test/integration/completion.test.ts b/packages/cli-core/src/test/integration/completion.test.ts index 822704218..886fbce93 100644 --- a/packages/cli-core/src/test/integration/completion.test.ts +++ b/packages/cli-core/src/test/integration/completion.test.ts @@ -334,6 +334,23 @@ describe("generateCompletions", () => { expect(names).toContain("settings"); }); + test("migrate sources: suggests the built-in sources", () => { + const saved = process.env.CLERK_EXPERIMENTAL; + process.env.CLERK_EXPERIMENTAL = "migrate"; + try { + const names = generateCompletions(createProgram(), [ + "migrate", + "sources", + "", + ]).completions.map((c) => c.name); + expect(names).toContain("supabase"); + expect(names).toContain("workos"); + } finally { + if (saved === undefined) delete process.env.CLERK_EXPERIMENTAL; + else process.env.CLERK_EXPERIMENTAL = saved; + } + }); + test("open dashboard: filters subpaths by prefix", () => { const names = completionNames("open", "dashboard", "u"); expect(names).toContain("users"); From 8c781dda47b5a37200c9485520aea6523da96f7a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 13:11:06 -0400 Subject: [PATCH 02/42] docs(changeset): Add clerk migrate sources and custom sources Co-Authored-By: Claude Opus 5.5 --- .changeset/migrate-custom-sources.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/migrate-custom-sources.md diff --git a/.changeset/migrate-custom-sources.md b/.changeset/migrate-custom-sources.md new file mode 100644 index 000000000..fb9542f26 --- /dev/null +++ b/.changeset/migrate-custom-sources.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Add `clerk migrate sources`, and let `clerk migrate import --source` take the path to a source you wrote. `clerk migrate sources` lists the built-in sources with the passwords, MFA and metadata each brings across, and `clerk migrate sources ` shows where each field lands, how to export, and any caveats. A source you wrote is a `.ts` or `.js` file in your project, checked before use, so a platform with no built-in source can be imported without rebuilding the CLI. `clerk migrate` is still experimental and needs `CLERK_EXPERIMENTAL=migrate`. From e3cd071ee926c8a87a65ac9895db500ff2ce63a8 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 18:51:22 -0400 Subject: [PATCH 03/42] docs(migrate): list both hook validation errors for a custom source The validation table quoted only the postTransform message, without the backticks the real one has, though preTransform is checked the same way. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index ef4c66301..e2f701bcd 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1214,15 +1214,15 @@ edited source counts as a different source. The file is code the CLI executes, so its shape is checked before use and rejected with the specific problem rather than crashing mid-pipeline: -| Problem | Message | -| ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| Path does not exist | `No source file at /abs/path.ts.` | -| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | -| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | -| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | -| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | -| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | -| A hook is not a function | `postTransform must be a function when present` | +| Problem | Message | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Path does not exist | `No source file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | +| `preTransform` or `postTransform` is not a function | `` `preTransform` must be a function when present `` (likewise `postTransform`) | The `userId` check is the load-bearing one: without it the import would run to completion and create every user with no `external_id`, which is what makes a From d0096a50e832f997387092c66593a3cb225585f0 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 13:30:43 -0400 Subject: [PATCH 04/42] fix(migrate): refuse a second import while one of the same file is live The live-run check sat inside resume matching, after the `sourceHash` filter, so editing a custom source (or passing --new-run) skipped it: a second run could start beside the live one, and both would send the same external IDs. The check now runs first, for any import of the same file, source key and instance, whatever its source hash or --new-run. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 22 +++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 33 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 0d0577293..e3ce57722 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2035,6 +2035,8 @@ describe("run", () => { fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); expect(await exitCodeOf(run(baseOptions))).toBe(EXIT_CODE.USAGE); + // A new run would send the same external IDs alongside it. + expect(await exitCodeOf(run({ ...baseOptions, newRun: true }))).toBe(EXIT_CODE.USAGE); }); // An edited file is a different job. @@ -2177,6 +2179,26 @@ describe("run", () => { expect("first_name" in (bodies[1] ?? {})).toBe(false); }); + // An edited source misses the live run's `sourceHash`, but both would send + // the same external IDs. + test("refuses while a run of this file with an earlier version of the source is live", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + // PID 1 is always alive, and never this test. + fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); + + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + await expect( + run({ input: "export.json", source: edited, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/importing this file right now in another process/); + expect(created()).toHaveLength(0); + }); + // An edited source is a different source, so the run records which one. test("records the custom source's content hash on the run", async () => { await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index ad7ae1c17..eeb926733 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -429,6 +429,33 @@ function applyEnvelope( // --- Continuing an earlier run --------------------------------------------- +/** + * Refuses while another process is importing this file, with this source key, + * into this instance. Checked before any resume matching, and under + * --new-run too: an edited custom source (another `sourceHash`) or a new run + * would otherwise send the same external IDs alongside it. + */ +function assertNoActiveImport( + runsDir: string, + match: { sha256: string; source: string; instanceId: string; keyInstanceId?: string }, +): void { + const active = listRuns(runsDir).find( + (record) => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + record.source === match.source && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId) && + runState(runsDir, record) === "running", + ); + if (active) { + throwUsageError( + `Run ${active.id} is importing this file right now in another process (PID ${liveLockPid(runsDir, active.id)}). ` + + `Wait for it to finish. If that process is not a migrate run, delete ${lockFile(runsDir, active.id)}.`, + ); + } +} + /** How this import relates to earlier runs of the same file, source and instance. */ type ResumeCase = | { kind: "new" } @@ -890,6 +917,12 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (expectedSha256 !== undefined && sha256 !== expectedSha256) throwChanged(filePath); const runsDir = await resolveRunsDir(options.runsDir); + assertNoActiveImport(runsDir, { + sha256, + source, + instanceId: target.instanceId, + keyInstanceId: keyInstanceId(secretKey), + }); const match = { sha256, source, From fc671e33564f6c56702b6c4858f2039c7a0b6247 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 14:25:51 -0400 Subject: [PATCH 05/42] fix(migrate): say --source takes a key or a path in the sources listing The listing said a file takes `--source `, but a custom source is loaded by its path, and its key reads as an unknown built-in. It now says `--source `, as the flag's own help does. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/sources/list.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts index 6955e37db..7991bb05a 100644 --- a/packages/cli-core/src/commands/migrate/sources/list.ts +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -93,7 +93,7 @@ function toJson(entry: SourceEntry, custom?: string) { function printList(width: number): void { for (const line of wrapText( "A source maps one platform's export onto the fields Clerk imports. A file from " + - "`clerk migrate export` names its own; anything else takes `--source `.", + "`clerk migrate export` names its own; anything else takes `--source `.", width, )) { log.info(line); From 3bb0fd2bb08227cfef687cc2e757a5d714e78c98 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 15:03:35 -0400 Subject: [PATCH 06/42] fix(migrate): hold an exclusive lock for an import of a file, from before its checks The check for a live import of the same file read the runs folder before the checks and prompts, but a run is locked only once it is written: two processes could both pass the check and start runs side by side. An import now takes an exclusive lock keyed by the runs folder, file hash, source key and instance before its checks and holds it to the end. It lives in the temp directory, since nothing may be written to the runs folder before consent; a dry run takes none. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/run-store.ts | 74 ++++++++++++++++--- .../cli-core/src/commands/migrate/run.test.ts | 32 ++++++++ packages/cli-core/src/commands/migrate/run.ts | 22 ++++++ 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 1423be233..4c156fd44 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -20,6 +20,7 @@ import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { resolveProfile } from "../../../lib/config.ts"; import { throwUsageError } from "../../../lib/errors.ts"; @@ -221,9 +222,14 @@ function isPidAlive(pid: number): boolean { * the same PID every run, so a killed run's lock would otherwise read as live. */ export function liveLockPid(runsDir: string, id: string): number | undefined { + return livePidIn(lockFile(runsDir, id)); +} + +/** The live PID a lock file holds, by the same rules as {@link liveLockPid}. */ +function livePidIn(file: string): number | undefined { let raw: string; try { - raw = fs.readFileSync(lockFile(runsDir, id), "utf-8"); + raw = fs.readFileSync(file, "utf-8"); } catch { return undefined; } @@ -246,31 +252,38 @@ const isMissing = (error: unknown) => (error as NodeJS.ErrnoException).code === * both read "free" and both write; a stale lock is removed and taken once. */ function acquireLock(runsDir: string, id: string): void { - const file = lockFile(runsDir, id); - const refuse = (holder: number | undefined): never => + acquireLockFile(lockFile(runsDir, id), (holder, file) => throwUsageError( `Run ${id} is in use by another process${holder ? ` (PID ${holder})` : ""}. Wait for it to finish, then try again. ` + `If that process is not a migrate run, delete ${file}.`, - ); + ), + ); +} + +/** Takes the lock at `file`, or calls `refuse` with its live holder. */ +function acquireLockFile( + file: string, + refuse: (holder: number | undefined, file: string) => never, +): void { const take = () => fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); try { take(); - log.debug(`migrate: locked run ${id} (${file})`); + log.debug(`migrate: took the lock ${file}`); return; } catch (error) { if (!isExists(error)) throw error; } - const holder = liveLockPid(runsDir, id); - if (holder !== undefined) refuse(holder); - log.debug(`migrate: removing a stale lock on run ${id} (${file})`); + const holder = livePidIn(file); + if (holder !== undefined) refuse(holder, file); + log.debug(`migrate: removing a stale lock (${file})`); fs.rmSync(file, { force: true }); try { take(); - log.debug(`migrate: locked run ${id} (${file})`); + log.debug(`migrate: took the lock ${file}`); } catch (error) { // Another process took the stale lock between the remove and the write. - if (isExists(error)) refuse(liveLockPid(runsDir, id)); + if (isExists(error)) refuse(livePidIn(file), file); throw error; } } @@ -280,6 +293,47 @@ function releaseLock(runsDir: string, id: string): void { fs.rmSync(lockFile(runsDir, id), { force: true }); } +/** What makes two imports the same job: the file, the source and the instance. */ +export type ImportIdentity = { sha256: string; source: string; instanceId: string }; + +/** + * The lock one import of a file, by a source, into an instance holds. In the + * temp directory, keyed by the runs folder too: it is taken before consent, + * when nothing may be written to the runs folder yet. + */ +export function importLockFile(runsDir: string, identity: ImportIdentity): string { + const key = createHash("sha256") + .update( + JSON.stringify([ + path.resolve(runsDir), + identity.sha256, + identity.source, + identity.instanceId, + ]), + ) + .digest("hex") + .slice(0, 32); + return path.join(os.tmpdir(), `clerk-migrate-import-${key}.lock`); +} + +/** + * Takes the lock for one import of this file, source and instance, before + * its checks, so no second process can pass them and start a run beside it. + * + * @returns Releases the lock. + * @throws UsageError when a live process holds it. + */ +export function lockImport(runsDir: string, identity: ImportIdentity): () => void { + const file = importLockFile(runsDir, identity); + acquireLockFile(file, (holder) => + throwUsageError( + `Another process${holder ? ` (PID ${holder})` : ""} is importing this file into this instance right now. ` + + `Wait for it to finish. If that process is not a migrate run, delete ${file}.`, + ), + ); + return () => fs.rmSync(file, { force: true }); +} + // --- Reading --------------------------------------------------------------- /** One run's record, or `undefined` when the folder holds no readable run. */ diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index e3ce57722..b1939e856 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -28,6 +28,7 @@ import { readRun, sha256File, startRun, + importLockFile, } from "./lib/run-store.ts"; import { explainErrors, run, validateRunOptions } from "./run.ts"; import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; @@ -2039,6 +2040,37 @@ describe("run", () => { expect(await exitCodeOf(run({ ...baseOptions, newRun: true }))).toBe(EXIT_CODE.USAGE); }); + // The scan of runs alone has a gap: a second process can pass it before + // either has written a run. The import's own lock closes it. + test("refuses while another process holds this import's lock, with no run yet", async () => { + const lock = importLockFile(runsDir(), { + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId: "ins_1", + }); + // PID 1 is always alive, and never this test. + fs.writeFileSync(lock, "1"); + try { + await expect(run(baseOptions)).rejects.toThrow( + /importing this file into this instance right now/, + ); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); + }); + + test("lets go of the import's lock when the run ends", async () => { + await run(baseOptions); + const lock = importLockFile(runsDir(), { + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId: "ins_1", + }); + expect(fs.existsSync(lock)).toBe(false); + }); + // An edited file is a different job. // Rate limiting right after a large import is when this lookup fails, and // that is when someone re-runs after an interruption. diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index eeb926733..b3d79fd3e 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -61,6 +61,9 @@ import { continueRun, latestUserLines, listRuns, + liveLockPid, + lockFile, + lockImport, readRun, readUserLines, resolveRunsDir, @@ -827,6 +830,19 @@ function recordRejects(run: Run, checks: ImportChecks): void { } export async function run(rawOptions: MigrateRunOptions): Promise { + // Released however the import ends: a return, a refusal or a Ctrl-C. + const lock: ImportLock = {}; + try { + await runImport(rawOptions, lock); + } finally { + lock.release?.(); + } +} + +/** The import's identity lock, taken part-way through and released by {@link run}. */ +type ImportLock = { release?: () => void }; + +async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promise { // A mistyped --source fails before any sign-in or link. let options = await applySource(rawOptions); await ensureImportTarget(options); @@ -917,6 +933,12 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (expectedSha256 !== undefined && sha256 !== expectedSha256) throwChanged(filePath); const runsDir = await resolveRunsDir(options.runsDir); + // Held from before the checks to the end of the run: the scan below + // alone leaves a gap a second process can pass through before either + // one writes a run. + if (!options.dryRun) { + lock.release = lockImport(runsDir, { sha256, source, instanceId: target.instanceId }); + } assertNoActiveImport(runsDir, { sha256, source, From 5d60274c2dc92984945b9a1f758ba2d67a54935f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 15:06:41 -0400 Subject: [PATCH 07/42] fix(migrate): load a custom source by the hash a run records for it An import hashed a custom source's file and recorded that, but the module loader caches by specifier, so loading an edited file at the same path again in one process returned the old module, and the registry kept the first entry registered under the key. A run could record the new hash beside the old mappings. The file is hashed first and imported by plain path with that hash as its query, which Bun loads afresh (a `file://` URL keeps its first module whatever the query), and a later load of a key replaces the earlier one. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 22 +++++++++++++++++++ .../commands/migrate/sources/load-custom.ts | 16 +++++++++++++- .../src/commands/migrate/sources/registry.ts | 13 ++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index b1939e856..3c19afd7b 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2231,6 +2231,28 @@ describe("run", () => { expect(created()).toHaveLength(0); }); + // The loader caches by URL: an edit at the same path must still load the + // edited mappings, the ones the run's hash names. + test("an edited source at the same path maps with its edited code", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + fs.writeFileSync( + path.join(workDir, customFile), + CUSTOM.replace('given: "firstName"', 'given: "lastName"'), + ); + requests = []; + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ last_name: "Ada" }); + expect("first_name" in (bodies[0] ?? {})).toBe(false); + }); + // An edited source is a different source, so the run records which one. test("records the custom source's content hash on the run", async () => { await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts index 03aacebbb..cec25c327 100644 --- a/packages/cli-core/src/commands/migrate/sources/load-custom.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -144,6 +144,12 @@ export function validateSource( export async function loadCustomSource( file: string, reservedKeys: readonly string[] = [], + /** + * The file's content hash. The module loader caches by URL, so an edited + * file at the same path would load the old module; the hash in the URL + * loads the version that was hashed. + */ + version?: string, ): Promise { const resolved = path.resolve(process.cwd(), file); @@ -161,7 +167,15 @@ export async function loadCustomSource( try { // A file URL rather than a bare path: an absolute POSIX path happens to // work, but a Windows path (`C:\...`) is not a valid import specifier. - module = (await import(Bun.pathToFileURL(resolved).href)) as Record; + // Bun keys its module cache by specifier and keeps a `file://` URL's + // module whatever its query, but a plain path's query loads afresh. + // ponytail: Windows keeps the URL form, so an edit loaded twice in one + // process reuses the first; each CLI run is a process of its own. + const specifier = + version && process.platform !== "win32" + ? `${resolved}?v=${version}` + : Bun.pathToFileURL(resolved).href; + module = (await import(specifier)) as Record; } catch (error) { throwUsageError( `Could not load ${file}: ${(error as Error).message}\n` + diff --git a/packages/cli-core/src/commands/migrate/sources/registry.ts b/packages/cli-core/src/commands/migrate/sources/registry.ts index 226ff7065..257732d9a 100644 --- a/packages/cli-core/src/commands/migrate/sources/registry.ts +++ b/packages/cli-core/src/commands/migrate/sources/registry.ts @@ -57,6 +57,9 @@ export const ACCOUNT_LINKING_NOTE = const customSources: SourceEntry[] = []; export function registerCustomSource(entry: SourceEntry): void { + // The latest load of a key wins: an edited source replaces its old self. + const earlier = customSources.findIndex((source) => source.key === entry.key); + if (earlier >= 0) customSources.splice(earlier, 1); customSources.push(entry); } @@ -107,10 +110,14 @@ export type ResolvedSource = { key: string; entry: SourceEntry; path?: string; h */ export async function resolveSource(value: string): Promise { if (isSourcePath(value)) { - const entry = await loadCustomSource(value, sourceKeys()); - registerCustomSource(entry); const resolved = path.resolve(process.cwd(), value); - const hash = createHash("sha256").update(fs.readFileSync(resolved)).digest("hex"); + // Hashed first, and loaded by that hash: the record names the very code + // that maps the users, even if the file was edited since a last load. + const hash = fs.existsSync(resolved) + ? createHash("sha256").update(fs.readFileSync(resolved)).digest("hex") + : undefined; + const entry = await loadCustomSource(value, sourceKeys(), hash); + registerCustomSource(entry); return { key: entry.key, entry, path: resolved, hash }; } From 3969ed4c5f8f08694bb1c7975f42117db1dc3109 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 15:06:41 -0400 Subject: [PATCH 08/42] docs(migrate): say a custom source's hash covers its own file only A local helper a custom source imports is not hashed with it, so an edit to one does not mark the source as changed. The README now says to pass --new-run after editing a helper. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index e2f701bcd..a4b3da20a 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1207,7 +1207,9 @@ TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, Plain `.js` works too. An import run records a custom source's key and a hash of the file, so an -edited source counts as a different source. +edited source counts as a different source. The hash covers that file only: +a local helper it imports is not part of it, so after editing one, pass +`--new-run` rather than continue a run made with the old helper. #### Validation From 72b60c51bd5398d6a94df4dea0669248c0f643e8 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 17:19:40 -0400 Subject: [PATCH 09/42] fix(migrate): link a lock into place whole, so no reader sees it without its PID Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/run-store.test.ts | 16 ++++++++++++++ .../src/commands/migrate/lib/run-store.ts | 21 ++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 974b28740..e49675b02 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -19,6 +19,7 @@ import { latestUserLines, listRuns, lockFile, + lockImport, newRunId, patchRun, readRun, @@ -204,6 +205,21 @@ describe("locks and interruptions", () => { expect(() => continueRun(runsDir, run.record)).not.toThrow(); }); + // A lock created empty and written after reads as stale to a second + // process in between, which would remove it and import beside the first. + test("a lock is linked into place with its PID already written", () => { + const identity = { sha256: "a".repeat(64), source: "supabase", instanceId: "ins_lock" }; + const write = spyOn(fs, "writeFileSync"); + try { + const release = lockImport(runsDir, identity); + const lockPaths = write.mock.calls.map(([target]) => String(target)); + expect(lockPaths.every((target) => target.endsWith(".tmp"))).toBe(true); + release(); + } finally { + write.mockRestore(); + } + }); + test("reads as running while another live process holds the lock", () => { const run = startRun(runsDir, init); fs.writeFileSync(path.join(run.dir, "lock"), "1"); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 4c156fd44..a3c63318e 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -265,7 +265,22 @@ function acquireLockFile( file: string, refuse: (holder: number | undefined, file: string) => never, ): void { - const take = () => fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); + // Written whole, then linked into place: the link fails on an existing lock + // as `wx` does, but the lock never exists without its PID, which a second + // process would read as stale and remove. + const take = () => { + const whole = `${file}.${process.pid}.tmp`; + fs.writeFileSync(whole, String(process.pid), { mode: 0o600 }); + try { + fs.linkSync(whole, file); + } catch (error) { + if (isExists(error)) throw error; + // A filesystem without hard links. + fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); + } finally { + fs.rmSync(whole, { force: true }); + } + }; try { take(); @@ -274,6 +289,10 @@ function acquireLockFile( } catch (error) { if (!isExists(error)) throw error; } + // ponytail: two processes that both read the same stale lock can both + // remove it, and the second can remove the first's new one. Clerk still + // refuses the second create of any email, phone or username, so the cost + // is failed lines, not duplicate users. A lock on the reclaim would close it. const holder = livePidIn(file); if (holder !== undefined) refuse(holder, file); log.debug(`migrate: removing a stale lock (${file})`); From c1d8c6d479c83c54c6adac2507e9c3ecfaa800bd Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 17:19:40 -0400 Subject: [PATCH 10/42] docs(migrate): say a custom source can import its own files, but not the CLI Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index a4b3da20a..38c73d222 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1175,8 +1175,9 @@ clerk migrate import users.json --source ./my-platform.ts ``` The file lives in **your** project, not in the CLI, and is imported at runtime. -It exports the same shape the built-ins use — plain data, no imports, since -there is nothing in a compiled binary for your file to import from: +It exports the same shape the built-ins use, as plain data. It can't import +from the CLI, since there is nothing in a compiled binary to import from, but +it can import your project's own files: ```ts export default { From 952c9df5d5a03e45712761b7e1778fd85dda3b94 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 19:37:00 -0400 Subject: [PATCH 11/42] fix(migrate): key the import lock by the secret key, so a failed instance lookup takes the same one Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/lib/run-store.ts | 5 ++++- packages/cli-core/src/commands/migrate/run.test.ts | 7 +++++-- packages/cli-core/src/commands/migrate/run.ts | 9 ++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index a3c63318e..5ecc39fcf 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -312,7 +312,10 @@ function releaseLock(runsDir: string, id: string): void { fs.rmSync(lockFile(runsDir, id), { force: true }); } -/** What makes two imports the same job: the file, the source and the instance. */ +/** + * What makes two imports the same job: the file, the source and the instance, + * named by its key's stand-in ID so it does not depend on `GET /v1/instance`. + */ export type ImportIdentity = { sha256: string; source: string; instanceId: string }; /** diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 3c19afd7b..1f244d24c 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -30,6 +30,7 @@ import { startRun, importLockFile, } from "./lib/run-store.ts"; +import { keyInstanceId } from "./lib/target.ts"; import { explainErrors, run, validateRunOptions } from "./run.ts"; import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; // After run.ts: imported first, signals.ts loads version.ts ahead of its Bun @@ -2043,10 +2044,11 @@ describe("run", () => { // The scan of runs alone has a gap: a second process can pass it before // either has written a run. The import's own lock closes it. test("refuses while another process holds this import's lock, with no run yet", async () => { + // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. const lock = importLockFile(runsDir(), { sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", - instanceId: "ins_1", + instanceId: keyInstanceId(baseOptions.secretKey), }); // PID 1 is always alive, and never this test. fs.writeFileSync(lock, "1"); @@ -2063,10 +2065,11 @@ describe("run", () => { test("lets go of the import's lock when the run ends", async () => { await run(baseOptions); + // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. const lock = importLockFile(runsDir(), { sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", - instanceId: "ins_1", + instanceId: keyInstanceId(baseOptions.secretKey), }); expect(fs.existsSync(lock)).toBe(false); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index b3d79fd3e..aa96f2391 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -936,8 +936,15 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi // Held from before the checks to the end of the run: the scan below // alone leaves a gap a second process can pass through before either // one writes a run. + // Keyed by the secret key's stand-in ID, not `target.instanceId`: that is + // a `key_…` ID too when `GET /v1/instance` fails, so two processes on one + // instance would otherwise take different locks. if (!options.dryRun) { - lock.release = lockImport(runsDir, { sha256, source, instanceId: target.instanceId }); + lock.release = lockImport(runsDir, { + sha256, + source, + instanceId: keyInstanceId(secretKey), + }); } assertNoActiveImport(runsDir, { sha256, From f8cce65ea82d897f1aeca5b548fde86a9648e758 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 20:45:26 -0400 Subject: [PATCH 12/42] fix(migrate): lock an import by the instance too, so a second key for it takes the same lock Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/run-store.ts | 2 +- .../cli-core/src/commands/migrate/run.test.ts | 34 ++++++++++++++++--- packages/cli-core/src/commands/migrate/run.ts | 25 +++++++++----- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 5ecc39fcf..8bc3c3875 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -314,7 +314,7 @@ function releaseLock(runsDir: string, id: string): void { /** * What makes two imports the same job: the file, the source and the instance, - * named by its key's stand-in ID so it does not depend on `GET /v1/instance`. + * by its ID or its key's stand-in ID. */ export type ImportIdentity = { sha256: string; source: string; instanceId: string }; diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 1f244d24c..5ce175c1f 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2063,15 +2063,41 @@ describe("run", () => { expect(listRuns(runsDir())).toHaveLength(0); }); - test("lets go of the import's lock when the run ends", async () => { - await run(baseOptions); - // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. + // A second key for the same instance has its own stand-in ID. + test("refuses while another key's process holds this instance's lock", async () => { const lock = importLockFile(runsDir(), { + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId: "ins_1", + }); + fs.writeFileSync(lock, "1"); + try { + await expect(run(baseOptions)).rejects.toThrow( + /importing this file into this instance right now/, + ); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + // The key's own lock, taken first, is let go when the second is refused. + const own = importLockFile(runsDir(), { sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), }); - expect(fs.existsSync(lock)).toBe(false); + expect(fs.existsSync(own)).toBe(false); + }); + + test("lets go of the import's locks when the run ends", async () => { + await run(baseOptions); + const lockFor = (instanceId: string) => + importLockFile(runsDir(), { + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId, + }); + expect(fs.existsSync(lockFor(keyInstanceId(baseOptions.secretKey)))).toBe(false); + expect(fs.existsSync(lockFor("ins_1"))).toBe(false); }); // An edited file is a different job. diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index aa96f2391..6d2574406 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -936,15 +936,24 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi // Held from before the checks to the end of the run: the scan below // alone leaves a gap a second process can pass through before either // one writes a run. - // Keyed by the secret key's stand-in ID, not `target.instanceId`: that is - // a `key_…` ID too when `GET /v1/instance` fails, so two processes on one - // instance would otherwise take different locks. + // Two locks: the key's stand-in ID, which the same key takes when + // `GET /v1/instance` fails, and the instance's ID when Clerk named it, + // which a second key for the instance takes. + // ponytail: a second key whose lookup failed takes neither; the run scan + // below is all that stops it. A lock on the key's instance needs Clerk. if (!options.dryRun) { - lock.release = lockImport(runsDir, { - sha256, - source, - instanceId: keyInstanceId(secretKey), - }); + const releases: (() => void)[] = []; + try { + for (const instanceId of new Set([keyInstanceId(secretKey), target.instanceId])) { + releases.push(lockImport(runsDir, { sha256, source, instanceId })); + } + } catch (error) { + for (const release of releases) release(); + throw error; + } + lock.release = () => { + for (const release of releases) release(); + }; } assertNoActiveImport(runsDir, { sha256, From 57cd24fbd3295097b902c5a8940c0c486acaa049 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 14 Oct 2026 19:27:19 -0400 Subject: [PATCH 13/42] fix(migrate): refuse a custom source whose caveats are not a list of strings `migrate sources` wraps each caveat as text, so `caveats: [42]` crashed it with a raw TypeError and `caveats: "text"` printed one bullet per character. validateSource now checks it next to the hooks. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/sources/load-custom.test.ts | 11 +++++++++++ .../src/commands/migrate/sources/load-custom.ts | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts index 513457375..73d518d82 100644 --- a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts @@ -214,6 +214,17 @@ describe("validateSource", () => { expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); }); + // `migrate sources` wraps each caveat as text: a number crashed it, and a + // string printed one bullet per character. + test.each([ + ["a non-string caveat", [42]], + ["a string instead of a list", "text"], + ])("rejects caveats holding %s", (_label, caveats) => { + expect(() => validateSource({ ...valid, caveats }, "f.ts")).toThrow( + /`caveats` must be an array of strings/, + ); + }); + test.each([["clerk"], ["auth0"], ["supabase"]])( "rejects %s, which would shadow a built-in", (key) => { diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts index cec25c327..9c1774e6c 100644 --- a/packages/cli-core/src/commands/migrate/sources/load-custom.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -122,6 +122,13 @@ export function validateSource( } } + if ( + entry.caveats !== undefined && + (!Array.isArray(entry.caveats) || entry.caveats.some((caveat) => typeof caveat !== "string")) + ) { + invalid("`caveats` must be an array of strings when present", file); + } + if (reservedKeys.includes(entry.key as string)) { invalid( `\`key\` is "${String(entry.key)}", which is already a built-in source. Choose another key`, From 370f9f611b0a483466830f7cedd4059d8cc763d7 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 14 Oct 2026 19:27:19 -0400 Subject: [PATCH 14/42] fix(migrate): refuse an edited custom source beside its earlier run, and lock an import whatever its runs folder An edited custom source misses the earlier run's sourceHash, so a re-run started a new run without saying so, and the users the earlier run created came back as conflicts. findResume now refuses with exit 2 naming that run, unless it was undone; --new-run starts a new one anyway. Tests cover the resume decision: an unchanged source continues, an edited one is refused, and --new-run starts a new run. The import lock was keyed by the runs folder too, so two imports of the same file into the same instance with different --runs-dir values did not block each other. It is keyed by the file, source and instance alone. The README documents the lock file, and says `migrate sources` runs a source file when given its path. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 26 +++++- .../commands/migrate/lib/run-store.test.ts | 2 +- .../src/commands/migrate/lib/run-store.ts | 20 ++-- .../cli-core/src/commands/migrate/run.test.ts | 91 ++++++++++++++++++- packages/cli-core/src/commands/migrate/run.ts | 56 +++++++++--- 5 files changed, 162 insertions(+), 33 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 38c73d222..d4d9f716f 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -179,6 +179,19 @@ every run. Run folders are created owner-only (`0700`), and export files `0600`: they hold password hashes and user data. +#### The import lock + +Every import but a dry run also takes a lock outside the run store, before its +checks and prompts: `clerk-migrate-import-.lock` in the temp directory +(`os.tmpdir()`, usually `$TMPDIR` or `/tmp`). The hash covers the file's sha256, +the source and the instance, so a second import of the same file into the same +instance is refused with exit 2 even before either has written a run, and +whichever `--runs-dir` each uses. It lives outside the runs folder because +nothing may be written there before consent. It holds the importing process's +PID and is removed however the import ends. One left by a process that died is +reclaimed by the next import. The refusal names the file, which is safe to +delete when no `clerk migrate import` is running. + `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. A line that cannot be written stops that user's create from going out. An export's file @@ -1122,7 +1135,9 @@ clerk migrate sources --json full: its export command, what it carries with a note for each, where each field lands (`encrypted_password → password`), its fixed defaults, and any caveats. An unknown key exits 2 and lists the valid ones. There is no -intro/outro gutter: this reads a static registry rather than running anything. +intro/outro gutter. `sources` alone, or with a built-in key, reads a static +registry and runs nothing. With a path, it loads and runs that file, as +`--source` does, to check it and show what it maps. ### `clerk migrate help` @@ -1208,8 +1223,12 @@ TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, Plain `.js` works too. An import run records a custom source's key and a hash of the file, so an -edited source counts as a different source. The hash covers that file only: -a local helper it imports is not part of it, so after editing one, pass +edited source counts as a different source. A re-run with an edited source +won't continue the earlier version's run, and won't start a new one beside it +unasked either: it refuses with exit 2 and names the earlier run, since the +users that run created would come back as conflicts. Pass `--new-run` to start +a new run anyway, or undo the earlier one first. The hash covers that file +only: a local helper it imports is not part of it, so after editing one, pass `--new-run` rather than continue a run made with the old helper. #### Validation @@ -1226,6 +1245,7 @@ rejected with the specific problem rather than crashing mid-pipeline: | No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | | `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | | `preTransform` or `postTransform` is not a function | `` `preTransform` must be a function when present `` (likewise `postTransform`) | +| `caveats` is not a list of strings | `` `caveats` must be an array of strings when present `` | The `userId` check is the load-bearing one: without it the import would run to completion and create every user with no `external_id`, which is what makes a diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index e49675b02..30a2ad818 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -211,7 +211,7 @@ describe("locks and interruptions", () => { const identity = { sha256: "a".repeat(64), source: "supabase", instanceId: "ins_lock" }; const write = spyOn(fs, "writeFileSync"); try { - const release = lockImport(runsDir, identity); + const release = lockImport(identity); const lockPaths = write.mock.calls.map(([target]) => String(target)); expect(lockPaths.every((target) => target.endsWith(".tmp"))).toBe(true); release(); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 8bc3c3875..b47822029 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -320,19 +320,13 @@ export type ImportIdentity = { sha256: string; source: string; instanceId: strin /** * The lock one import of a file, by a source, into an instance holds. In the - * temp directory, keyed by the runs folder too: it is taken before consent, - * when nothing may be written to the runs folder yet. + * temp directory, as it is taken before consent, when nothing may be written + * to the runs folder yet. Not keyed by the runs folder: two imports with + * different `--runs-dir` values would still send the same users. */ -export function importLockFile(runsDir: string, identity: ImportIdentity): string { +export function importLockFile(identity: ImportIdentity): string { const key = createHash("sha256") - .update( - JSON.stringify([ - path.resolve(runsDir), - identity.sha256, - identity.source, - identity.instanceId, - ]), - ) + .update(JSON.stringify([identity.sha256, identity.source, identity.instanceId])) .digest("hex") .slice(0, 32); return path.join(os.tmpdir(), `clerk-migrate-import-${key}.lock`); @@ -345,8 +339,8 @@ export function importLockFile(runsDir: string, identity: ImportIdentity): strin * @returns Releases the lock. * @throws UsageError when a live process holds it. */ -export function lockImport(runsDir: string, identity: ImportIdentity): () => void { - const file = importLockFile(runsDir, identity); +export function lockImport(identity: ImportIdentity): () => void { + const file = importLockFile(identity); acquireLockFile(file, (holder) => throwUsageError( `Another process${holder ? ` (PID ${holder})` : ""} is importing this file into this instance right now. ` + diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 5ce175c1f..ef56813a8 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2045,7 +2045,7 @@ describe("run", () => { // either has written a run. The import's own lock closes it. test("refuses while another process holds this import's lock, with no run yet", async () => { // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. - const lock = importLockFile(runsDir(), { + const lock = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), @@ -2063,9 +2063,27 @@ describe("run", () => { expect(listRuns(runsDir())).toHaveLength(0); }); + // A different --runs-dir is a different record of the same job. + test("refuses while another process holds this import's lock under another runs folder", async () => { + const lock = importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId: keyInstanceId(baseOptions.secretKey), + }); + fs.writeFileSync(lock, "1"); + try { + await expect( + run({ ...baseOptions, runsDir: path.join(workDir, "other-runs") }), + ).rejects.toThrow(/importing this file into this instance right now/); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + }); + // A second key for the same instance has its own stand-in ID. test("refuses while another key's process holds this instance's lock", async () => { - const lock = importLockFile(runsDir(), { + const lock = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", instanceId: "ins_1", @@ -2080,7 +2098,7 @@ describe("run", () => { } expect(created()).toHaveLength(0); // The key's own lock, taken first, is let go when the second is refused. - const own = importLockFile(runsDir(), { + const own = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), @@ -2091,7 +2109,7 @@ describe("run", () => { test("lets go of the import's locks when the run ends", async () => { await run(baseOptions); const lockFor = (instanceId: string) => - importLockFile(runsDir(), { + importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), source: "clerk", instanceId, @@ -2260,6 +2278,71 @@ describe("run", () => { expect(created()).toHaveLength(0); }); + /** Rewrites the only run as one a crash stopped, and returns its ID. */ + const interruptOnly = (): string => { + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + return first!.id; + }; + + test("continues an interrupted run when the source is unchanged", async () => { + stubClerk({ failing: new Set(["mp_2"]) }); + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const id = interruptOnly(); + requests = []; + process.exitCode = 0; + stubClerk(); + + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_2", + ]); + expect(captured.err).toContain(`Continuing run ${id}`); + expect(listRuns(runsDir())).toHaveLength(1); + }); + + // Starting over would bring the users the earlier run created back as conflicts. + test("refuses an edited source while an earlier version's run is interrupted, naming it", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const id = interruptOnly(); + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + + const error = (await run({ + input: "export.json", + source: edited, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain(`Run ${id} imported this file with an earlier version`); + expect(error.message).toContain("--new-run"); + expect(created()).toHaveLength(0); + }); + + test("an edited source with --new-run starts a new run beside the earlier one", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + interruptOnly(); + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + + await run({ + input: "export.json", + source: edited, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + + expect(listRuns(runsDir())).toHaveLength(2); + }); + // The loader caches by URL: an edit at the same path must still load the // edited mappings, the ones the run's hash names. test("an edited source at the same path maps with its edited code", async () => { diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 6d2574406..a1f7c34e0 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -459,6 +459,17 @@ function assertNoActiveImport( } } +/** True when a run's users are gone: it is marked undone, or an undo of it finished. */ +function undoneInFull(runsDir: string, record: RunRecord): boolean { + return ( + runState(runsDir, record) === "undone" || + listRuns(runsDir).some( + (undo) => + undo.kind === "undo" && undo.undoes === record.id && runState(runsDir, undo) === "complete", + ) + ); +} + /** How this import relates to earlier runs of the same file, source and instance. */ type ResumeCase = | { kind: "new" } @@ -477,7 +488,9 @@ type ResumeCase = * @throws UsageError when that run is still running in another process, or * has an undo that never finished: some of its users are gone and some are * not, so neither continuing it nor starting over is safe. Also when Clerk - * did not confirm the instance and a run of this file names a real one. + * did not confirm the instance and a run of this file names a real one, or + * when the latest run of this file was made with an earlier version of a + * custom source and was not undone. */ function findResume( runsDir: string, @@ -492,20 +505,37 @@ function findResume( /** The import's runs-folder and target flags, for the command a refusal suggests. */ carry: string[], ): ResumeCase { - const runs = listRuns(runsDir); - const ofFile = importsOfFile(runs, match); - const latest = ofFile.find( - (record) => - record.sourceHash === match.sourceHash && - (record.target.instanceId === match.instanceId || - record.target.instanceId === match.keyInstanceId), + const sameJob = (record: RunRecord): boolean => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + record.source === match.source && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId); + const latest = listRuns(runsDir).find( + (record) => sameJob(record) && record.sourceHash === match.sourceHash, ); if (!latest) { + // An edited custom source is a different source, but its earlier run sent + // the same users: starting over beside it would reject every user it + // created and leave it unfinished, so that takes --new-run. + const earlier = listRuns(runsDir).find(sameJob); + if (earlier && !undoneInFull(runsDir, earlier)) { + throwUsageError( + `Run ${earlier.id} imported this file with an earlier version of this source, so the users it created would come back as conflicts. Nothing was imported.\n` + + `Pass --new-run to start a new run beside it, or undo it first with \`clerk migrate undo ${earlier.id}\`.`, + ); + } // Clerk did not name the instance (GET /v1/instance failed), so a run of // this file recorded under a real ID can be neither matched nor ruled out. // Starting over would reject every user it created and leave it unfinished. if (isStandInInstanceId(match.instanceId)) { - const unconfirmed = ofFile.find((record) => isClerkInstanceId(record.target.instanceId)); + const unconfirmed = listRuns(runsDir).find( + (record) => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + record.source === match.source && + isClerkInstanceId(record.target.instanceId), + ); if (unconfirmed) { throwUsageError( `Clerk did not confirm which instance this key addresses, so this import cannot tell whether run ${unconfirmed.id} is for it. ` + @@ -524,10 +554,12 @@ function findResume( } if (state === "undone") return { kind: "new" }; - const openUndo = openUndoOf(runsDir, latest.id, runs); + const openUndo = openUndoOf(runsDir, latest.id); if (openUndo) throwUsageError(unfinishedUndoMessage(latest.id, openUndo.id, carry)); // Undone in full, though the import run was never marked so. - if (undosOf(runs, latest.id).length > 0) return { kind: "new" }; + if (listRuns(runsDir).some((record) => record.kind === "undo" && record.undoes === latest.id)) { + return { kind: "new" }; + } if (state === "complete") return { kind: "complete", record: latest }; return { @@ -945,7 +977,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi const releases: (() => void)[] = []; try { for (const instanceId of new Set([keyInstanceId(secretKey), target.instanceId])) { - releases.push(lockImport(runsDir, { sha256, source, instanceId })); + releases.push(lockImport({ sha256, source, instanceId })); } } catch (error) { for (const release of releases) release(); From fc9c896e58ad594ac0369fd19d81b03d41c94f2f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 14 Oct 2026 19:51:21 -0400 Subject: [PATCH 15/42] test(migrate): let the lock-linking test pass where hard links are refused On a filesystem without hard links, acquireLockFile writes the lock itself with `wx`, so "every write is a .tmp" failed there although the code was right. The check now holds only when the link went through, and a new test forces the link to fail and checks the fallback writes the lock exclusively. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/run-store.test.ts | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 30a2ad818..e40c58ad6 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -19,6 +19,7 @@ import { latestUserLines, listRuns, lockFile, + importLockFile, lockImport, newRunId, patchRun, @@ -210,13 +211,36 @@ describe("locks and interruptions", () => { test("a lock is linked into place with its PID already written", () => { const identity = { sha256: "a".repeat(64), source: "supabase", instanceId: "ins_lock" }; const write = spyOn(fs, "writeFileSync"); + const link = spyOn(fs, "linkSync"); try { const release = lockImport(identity); const lockPaths = write.mock.calls.map(([target]) => String(target)); - expect(lockPaths.every((target) => target.endsWith(".tmp"))).toBe(true); + // Where hard links are refused, the fallback below writes the lock itself. + if (link.mock.results.every((result) => result.type === "return")) { + expect(lockPaths.every((target) => target.endsWith(".tmp"))).toBe(true); + } release(); } finally { write.mockRestore(); + link.mockRestore(); + } + }); + + test("a filesystem without hard links gets the lock written exclusively", () => { + const identity = { sha256: "b".repeat(64), source: "supabase", instanceId: "ins_nolink" }; + const link = spyOn(fs, "linkSync").mockImplementation(() => { + throw Object.assign(new Error("not supported"), { code: "EPERM" }); + }); + const write = spyOn(fs, "writeFileSync"); + try { + const release = lockImport(identity); + const lock = write.mock.calls.find(([target]) => String(target).endsWith(".lock")); + expect(lock?.[2]).toMatchObject({ flag: "wx" }); + expect(fs.readFileSync(importLockFile(identity), "utf-8")).toBe(String(process.pid)); + release(); + } finally { + write.mockRestore(); + link.mockRestore(); } }); From 2f1a0d20d2beb0bb8991a414fe126f6c30aed99a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 14 Oct 2026 23:41:25 -0400 Subject: [PATCH 16/42] feat(migrate): let a source you wrote replace an export file's own `clerk migrate import --source ./custom.ts` was refused whenever the envelope named a built-in source, so bringing across a field the built-in mapping leaves out (a WorkOS metadata.username, an Auth.js name that is a username) meant stripping the envelope into a new file and losing the link to the export run. A source given by path now replaces the envelope's and reads its users in the platform's own shape. The import keeps fromExport, records the source the envelope named as envelopeSource, and its printed commands keep --source. One built-in name against another is still refused, and that message now points at a source you write. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 8 +++ .../cli-core/src/commands/migrate/index.ts | 2 +- .../src/commands/migrate/lib/run-store.ts | 2 + .../cli-core/src/commands/migrate/run.test.ts | 68 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 27 ++++++-- 5 files changed, 101 insertions(+), 6 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index d4d9f716f..bfb0994df 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -289,6 +289,14 @@ The file is an envelope around the users: parameters, when it could read them: the import then needs no `--firebase-*` flags. When it couldn't, pass them to the import, or enter them at its prompt. +`--source` with a different built-in name is refused, since it would map the +users with the wrong fields. `--source` with the path to a +[source you wrote](#custom-sources) replaces the envelope's: it gets the same +users, in the platform's own shape, so it can bring across what the built-in +mapping leaves out, such as a WorkOS `metadata.username`. The import keeps its +link to the export run, and `run.json` records the source the envelope named +as `envelopeSource`. + `--json` prints the result on stdout instead — `{ target, run, output, users, skipped, coverage, next }` — and never prompts, so a missing credential exits 2 naming the flag to pass. `next`, and the commands printed after a human run, diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index a4bb4afed..0e27c76d2 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -84,7 +84,7 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") .option( "--source ", - "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`", + "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`, though a source you wrote replaces its own", ) .option("--dry-run", "Check the file against the instance, report, and write nothing") .option("--allow-partial", "Import the users that pass the checks, and skip the rest") diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index b47822029..e88859f43 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -90,6 +90,8 @@ export type RunRecord = { * continue for those. */ flags?: ImportFlags; + /** The source the export file named, when a custom source replaced it. */ + envelopeSource?: string; /** * A Firebase import's hash parameters, as a scrypt hash of the four (never the * parameters themselves): every digest it builds carries them, so a continue diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index ef56813a8..0d6430890 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -630,6 +630,74 @@ describe("run", () => { expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); }); + test("says how to map the users differently when a built-in source contradicts it", async () => { + const { record } = exportRun("clerk", export2); + + await expect(run({ ...noSource, source: "auth0", input: record.id })).rejects.toThrow( + /pass the path to a source you wrote/, + ); + }); + + /** A source of your own, for an Auth0 export, that also brings `nickname` across. */ + const ownAuth0Source = () => { + const file = `./own-auth0-${Date.now()}-${Math.random().toString(36).slice(2)}.ts`; + fs.writeFileSync( + path.join(workDir, file), + `export default { + key: "myauth0", + label: "My Auth0", + transformer: { user_id: "userId", email: "email", nickname: "firstName" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + };`, + ); + return file; + }; + const auth0Rows = [{ user_id: "auth0|a1", email: "a@x.dev", nickname: "Ada" }]; + + // The built-in mapping leaves `nickname` out; the customer's source brings it. + test("a source you wrote replaces the envelope's, reading its users", async () => { + const { record } = exportRun("auth0", auth0Rows); + const own = ownAuth0Source(); + + await run({ ...noSource, source: own, input: record.id }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created).toHaveLength(1); + expect(created[0]?.body).toMatchObject({ external_id: "auth0|a1", first_name: "Ada" }); + expect(captured.err).toContain("in place of the built-in"); + }); + + test("an import through a source you wrote keeps its export run, and records what it replaced", async () => { + const { record } = exportRun("auth0", auth0Rows); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id }); + + const imported = listRuns(runsDir()).find((r) => r.kind === "import"); + expect(imported).toMatchObject({ + source: "myauth0", + fromExport: record.id, + envelopeSource: "auth0", + }); + }); + + // A re-run without it would map the rest with the built-in source. + test("the printed command keeps a source you wrote in place of the envelope's", async () => { + const { record } = exportRun("auth0", auth0Rows); + const own = ownAuth0Source(); + + const error = (await run({ ...noSource, source: own, input: record.id, yes: false }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.examples?.[0]?.command).toContain( + `clerk migrate import ${record.id} --source ${own}`, + ); + }); + test("refuses a run ID that is not an export", async () => { await run(baseOptions); const [imported] = listRuns(runsDir()); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index a1f7c34e0..29b337527 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -120,6 +120,8 @@ export type MigrateRunOptions = { sourceHash?: string; /** `--source` as typed, for printed commands: a custom source's path. Not a flag. */ sourceArg?: string; + /** The source an export file named, when a custom `--source` replaced it. Not a flag. */ + envelopeSource?: string; /** The file to read, once `input` is resolved. Not a flag. */ file?: string; requirePassword?: boolean; @@ -403,20 +405,33 @@ async function applySource(options: MigrateRunOptions): Promise"; const parts = ["clerk migrate import", quoteArg(input)]; const source = options.sourceArg ?? options.source; - if (!fromExport && source) parts.push("--source", quoteArg(source)); + // An export names its own source, unless a source you wrote replaced it. + if ((!fromExport || options.envelopeSource) && source) parts.push("--source", quoteArg(source)); if (options.newRun) parts.push("--new-run"); parts.push(...importFlags(options)); // Names, not `<…>`: pasted as is, a shell reads `` as a redirect. @@ -1371,6 +1387,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), file: { path: filePath, sha256 }, ...(input.fromExport ? { fromExport: input.fromExport } : {}), + ...(options.envelopeSource ? { envelopeSource: options.envelopeSource } : {}), ...(firebaseHash ? { firebaseHash } : {}), flags, }); From cb61df96003e0dc5e8debebc9480b7b7c6217452 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 15 Oct 2026 02:02:56 -0400 Subject: [PATCH 17/42] fix(migrate): refuse an edited custom source while any earlier version's run is unfinished, not just the newest The check took the newest run of the file, source and instance; when that one was undone, an older run that still held its users was never looked at, and the import started a new run beside it. Any run not undone in full now refuses, and the suggested undo keeps the runs folder and target flags. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 34 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 10 ++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 0d6430890..675d06fb9 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2393,6 +2393,40 @@ describe("run", () => { expect(created()).toHaveLength(0); }); + // The newest run being undone doesn't clear an older one that still holds users. + test("refuses an edited source while any earlier version's run is unfinished, not just the newest", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const older = interruptOnly(); + const second = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, second), `${CUSTOM}\n// second`); + await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + const newer = listRuns(runsDir()).find((record) => record.id !== older)!; + fs.writeFileSync( + path.join(runsDir(), newer.id, "run.json"), + JSON.stringify({ ...readRun(runsDir(), newer.id), status: "undone" }), + ); + const third = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, third), `${CUSTOM}\n// third`); + requests = []; + + const error = (await run({ + input: "export.json", + source: third, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.message).toContain(`Run ${older} imported this file with an earlier version`); + expect(error.message).toContain(`clerk migrate undo ${older} --secret-key `); + expect(created()).toHaveLength(0); + }); + test("an edited source with --new-run starts a new run beside the earlier one", async () => { await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); interruptOnly(); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 29b337527..e791df36c 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -533,11 +533,15 @@ function findResume( // An edited custom source is a different source, but its earlier run sent // the same users: starting over beside it would reject every user it // created and leave it unfinished, so that takes --new-run. - const earlier = listRuns(runsDir).find(sameJob); - if (earlier && !undoneInFull(runsDir, earlier)) { + // Any run of it not undone, not just the newest: an older interrupted run + // behind a newer undone one still holds its users. + const earlier = listRuns(runsDir).find( + (record) => sameJob(record) && !undoneInFull(runsDir, record), + ); + if (earlier) { throwUsageError( `Run ${earlier.id} imported this file with an earlier version of this source, so the users it created would come back as conflicts. Nothing was imported.\n` + - `Pass --new-run to start a new run beside it, or undo it first with \`clerk migrate undo ${earlier.id}\`.`, + `Pass --new-run to start a new run beside it, or undo it first with \`${withFlags(`clerk migrate undo ${earlier.id}`, carry)}\`.`, ); } // Clerk did not name the instance (GET /v1/instance failed), so a run of From 21eb5fe716aa9e6b470341728a7fdd3cd292af5a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 15 Oct 2026 14:58:43 -0400 Subject: [PATCH 18/42] fix(migrate): match a file's earlier imports by file and instance, and pass Firebase's parameters to a source you wrote From Wyatt's review of #530: - A run is recorded under its source's key, and the import lock, the live-run scan and the earlier-run refusal all matched on it. So importing an export with the built-in, then re-running it through a source you wrote, started a second run beside the first, and two imports of one export through different sources took different locks. The three now match on the file and the instance only; only the run to continue needs the same source and version. The refusal names the other source. - A Firebase export through a source you wrote lost its hash parameters: every Firebase check keyed on `source === "firebase"`, and the source was the custom key. They now key on the envelope's source too, so the parameters reach `context.firebaseHashConfig` and the run records its firebaseHash. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 26 +++-- .../commands/migrate/lib/run-store.test.ts | 4 +- .../src/commands/migrate/lib/run-store.ts | 13 +-- .../cli-core/src/commands/migrate/run.test.ts | 100 ++++++++++++++++-- packages/cli-core/src/commands/migrate/run.ts | 88 ++++++++------- 5 files changed, 167 insertions(+), 64 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index bfb0994df..81c694c6d 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -183,10 +183,10 @@ password hashes and user data. Every import but a dry run also takes a lock outside the run store, before its checks and prompts: `clerk-migrate-import-.lock` in the temp directory -(`os.tmpdir()`, usually `$TMPDIR` or `/tmp`). The hash covers the file's sha256, -the source and the instance, so a second import of the same file into the same -instance is refused with exit 2 even before either has written a run, and -whichever `--runs-dir` each uses. It lives outside the runs folder because +(`os.tmpdir()`, usually `$TMPDIR` or `/tmp`). The hash covers the file's sha256 +and the instance, not the source, so a second import of the same file into the +same instance is refused with exit 2 even before either has written a run, +whichever source and `--runs-dir` each uses. It lives outside the runs folder because nothing may be written there before consent. It holds the importing process's PID and is removed however the import ends. One left by a process that died is reclaimed by the next import. The refusal names the file, which is safe to @@ -295,7 +295,14 @@ users with the wrong fields. `--source` with the path to a users, in the platform's own shape, so it can bring across what the built-in mapping leaves out, such as a WorkOS `metadata.username`. The import keeps its link to the export run, and `run.json` records the source the envelope named -as `envelopeSource`. +as `envelopeSource`. A Firebase export keeps its hash parameters through it: +the envelope's, the `--firebase-*` flags or the variables reach the source's +`postTransform` as `context.firebaseHashConfig`, as they do the built-in's. + +An earlier import of the same export into the same instance, through the +built-in or another source, holds the same users. While it isn't undone, a +re-run through your source refuses with exit 2 and names it, rather than +starting a new run beside it: pass `--new-run`, or undo it first. `--json` prints the result on stdout instead — `{ target, run, output, users, skipped, coverage, next }` — and never prompts, so a missing credential exits 2 @@ -1231,10 +1238,11 @@ TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, Plain `.js` works too. An import run records a custom source's key and a hash of the file, so an -edited source counts as a different source. A re-run with an edited source -won't continue the earlier version's run, and won't start a new one beside it -unasked either: it refuses with exit 2 and names the earlier run, since the -users that run created would come back as conflicts. Pass `--new-run` to start +edited source counts as a different source. A re-run with an edited source, +or with another source altogether, won't continue the earlier run of the file, +and won't start a new one beside it unasked either: while that run isn't +undone, it refuses with exit 2 and names it, since the users it created would +come back as conflicts. Pass `--new-run` to start a new run anyway, or undo the earlier one first. The hash covers that file only: a local helper it imports is not part of it, so after editing one, pass `--new-run` rather than continue a run made with the old helper. diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index e40c58ad6..44b1f9997 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -209,7 +209,7 @@ describe("locks and interruptions", () => { // A lock created empty and written after reads as stale to a second // process in between, which would remove it and import beside the first. test("a lock is linked into place with its PID already written", () => { - const identity = { sha256: "a".repeat(64), source: "supabase", instanceId: "ins_lock" }; + const identity = { sha256: "a".repeat(64), instanceId: "ins_lock" }; const write = spyOn(fs, "writeFileSync"); const link = spyOn(fs, "linkSync"); try { @@ -227,7 +227,7 @@ describe("locks and interruptions", () => { }); test("a filesystem without hard links gets the lock written exclusively", () => { - const identity = { sha256: "b".repeat(64), source: "supabase", instanceId: "ins_nolink" }; + const identity = { sha256: "b".repeat(64), instanceId: "ins_nolink" }; const link = spyOn(fs, "linkSync").mockImplementation(() => { throw Object.assign(new Error("not supported"), { code: "EPERM" }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index e88859f43..86d9f0c48 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -315,27 +315,28 @@ function releaseLock(runsDir: string, id: string): void { } /** - * What makes two imports the same job: the file, the source and the instance, - * by its ID or its key's stand-in ID. + * What makes two imports the same job: the file and the instance, by its ID + * or its key's stand-in ID. Not the source: whichever source maps the file, + * its users are the same people. */ -export type ImportIdentity = { sha256: string; source: string; instanceId: string }; +export type ImportIdentity = { sha256: string; instanceId: string }; /** - * The lock one import of a file, by a source, into an instance holds. In the + * The lock one import of a file into an instance holds. In the * temp directory, as it is taken before consent, when nothing may be written * to the runs folder yet. Not keyed by the runs folder: two imports with * different `--runs-dir` values would still send the same users. */ export function importLockFile(identity: ImportIdentity): string { const key = createHash("sha256") - .update(JSON.stringify([identity.sha256, identity.source, identity.instanceId])) + .update(JSON.stringify([identity.sha256, identity.instanceId])) .digest("hex") .slice(0, 32); return path.join(os.tmpdir(), `clerk-migrate-import-${key}.lock`); } /** - * Takes the lock for one import of this file, source and instance, before + * Takes the lock for one import of this file into this instance, before * its checks, so no second process can pass them and start a run beside it. * * @returns Releases the lock. diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 675d06fb9..90713ab28 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -685,6 +685,93 @@ describe("run", () => { }); // A re-run without it would map the rest with the built-in source. + // Import with the built-in, notice a field is missing, write a source, re-run: + // the first run's users are the same people, whichever source maps them. + test("refuses a source you wrote while the built-in's run of the same export holds its users", async () => { + const { record } = exportRun("auth0", auth0Rows); + await run({ ...noSource, input: record.id }); + const [first] = listRuns(runsDir()).filter((r) => r.kind === "import"); + requests = []; + + const error = (await run({ ...noSource, source: ownAuth0Source(), input: record.id }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain( + `Run ${first!.id} imported this file into this instance with the \`auth0\` source`, + ); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id, newRun: true }); + expect(listRuns(runsDir()).filter((r) => r.kind === "import")).toHaveLength(2); + }); + + // One lock for the file and instance, whichever source each import uses. + test("a source you wrote takes the same import lock as the built-in", async () => { + const { record, file } = exportRun("auth0", auth0Rows); + const lock = importLockFile({ + sha256: sha256File(file), + instanceId: keyInstanceId(baseOptions.secretKey), + }); + fs.writeFileSync(lock, "1"); + try { + await expect( + run({ ...noSource, source: ownAuth0Source(), input: record.id }), + ).rejects.toThrow(/importing this file into this instance right now/); + } finally { + fs.rmSync(lock, { force: true }); + } + }); + + // A Firebase export's digests need the project's parameters, whatever maps it. + test("a source you wrote over a Firebase export gets its hash parameters", async () => { + const { record } = exportRun( + "firebase", + [{ localId: "f1", email: "f@x.dev", passwordHash: "HASH", passwordSalt: "SALT" }], + { + firebase: { + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }, + }, + ); + const own = `./own-firebase-${Date.now()}.ts`; + fs.writeFileSync( + path.join(workDir, own), + `export default { + key: "myfirebase", + label: "My Firebase", + transformer: { localId: "userId", email: "email", passwordHash: "passwordHash", passwordSalt: "salt" }, + carries: { + passwords: { level: "yes", note: "scrypt." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + defaults: { passwordHasher: "scrypt_firebase" }, + postTransform: (user, context) => { + const c = context.firebaseHashConfig; + if (!c) throw new Error("no hash parameters"); + user.password = [user.passwordHash, user.salt, c.base64_signer_key, c.base64_salt_separator, c.rounds, c.mem_cost].join("$"); + delete user.passwordHash; + delete user.salt; + }, + };`, + ); + + await run({ ...noSource, source: own, input: record.id }); + + const created = requests.find((r) => r.url.endsWith("/v1/users")); + expect(created?.body).toMatchObject({ + password_hasher: "scrypt_firebase", + password_digest: "HASH$SALT$SIGNER$Bw==$8$14", + }); + const imported = listRuns(runsDir()).find((r) => r.kind === "import"); + expect(imported?.firebaseHash).toMatch(/^[0-9a-f]{64}$/); + }); + test("the printed command keeps a source you wrote in place of the envelope's", async () => { const { record } = exportRun("auth0", auth0Rows); const own = ownAuth0Source(); @@ -2115,7 +2202,6 @@ describe("run", () => { // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. const lock = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), - source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), }); // PID 1 is always alive, and never this test. @@ -2135,7 +2221,6 @@ describe("run", () => { test("refuses while another process holds this import's lock under another runs folder", async () => { const lock = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), - source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), }); fs.writeFileSync(lock, "1"); @@ -2153,7 +2238,6 @@ describe("run", () => { test("refuses while another key's process holds this instance's lock", async () => { const lock = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), - source: "clerk", instanceId: "ins_1", }); fs.writeFileSync(lock, "1"); @@ -2168,7 +2252,6 @@ describe("run", () => { // The key's own lock, taken first, is let go when the second is refused. const own = importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), - source: "clerk", instanceId: keyInstanceId(baseOptions.secretKey), }); expect(fs.existsSync(own)).toBe(false); @@ -2179,7 +2262,6 @@ describe("run", () => { const lockFor = (instanceId: string) => importLockFile({ sha256: sha256File(path.join(workDir, "export.json")), - source: "clerk", instanceId, }); expect(fs.existsSync(lockFor(keyInstanceId(baseOptions.secretKey)))).toBe(false); @@ -2388,7 +2470,9 @@ describe("run", () => { }).catch((caught: unknown) => caught)) as CliError; expect(error.exitCode).toBe(EXIT_CODE.USAGE); - expect(error.message).toContain(`Run ${id} imported this file with an earlier version`); + expect(error.message).toContain( + `Run ${id} imported this file into this instance with an earlier version`, + ); expect(error.message).toContain("--new-run"); expect(created()).toHaveLength(0); }); @@ -2422,7 +2506,9 @@ describe("run", () => { secretKey: "sk_test_x", }).catch((caught: unknown) => caught)) as CliError; - expect(error.message).toContain(`Run ${older} imported this file with an earlier version`); + expect(error.message).toContain( + `Run ${older} imported this file into this instance with an earlier version`, + ); expect(error.message).toContain(`clerk migrate undo ${older} --secret-key `); expect(created()).toHaveLength(0); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index e791df36c..ea4f22d3f 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -61,8 +61,6 @@ import { continueRun, latestUserLines, listRuns, - liveLockPid, - lockFile, lockImport, readRun, readUserLines, @@ -455,25 +453,35 @@ function applyEnvelope( */ function assertNoActiveImport( runsDir: string, - match: { sha256: string; source: string; instanceId: string; keyInstanceId?: string }, + match: { sha256: string; instanceId: string; keyInstanceId?: string }, ): void { const active = listRuns(runsDir).find( - (record) => - record.kind === "import" && - record.file?.sha256 === match.sha256 && - record.source === match.source && - (record.target.instanceId === match.instanceId || - record.target.instanceId === match.keyInstanceId) && - runState(runsDir, record) === "running", + (record) => sameFileAndInstance(record, match) && runState(runsDir, record) === "running", ); if (active) { throwUsageError( - `Run ${active.id} is importing this file right now in another process (PID ${liveLockPid(runsDir, active.id)}). ` + - `Wait for it to finish. If that process is not a migrate run, delete ${lockFile(runsDir, active.id)}.`, + inUseMessage(runsDir, active.id, "importing this file", "Wait for it to finish."), ); } } +/** + * An import of this file into this instance, by its ID or its key's stand-in + * ID, under any source. Whichever source maps them, its users are the same + * people, so another source's run, or another version of one, holds them too. + */ +function sameFileAndInstance( + record: RunRecord, + match: { sha256: string; instanceId: string; keyInstanceId?: string }, +): boolean { + return ( + record.kind === "import" && + record.file?.sha256 === match.sha256 && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId) + ); +} + /** True when a run's users are gone: it is marked undone, or an undo of it finished. */ function undoneInFull(runsDir: string, record: RunRecord): boolean { return ( @@ -520,39 +528,39 @@ function findResume( /** The import's runs-folder and target flags, for the command a refusal suggests. */ carry: string[], ): ResumeCase { - const sameJob = (record: RunRecord): boolean => - record.kind === "import" && - record.file?.sha256 === match.sha256 && - record.source === match.source && - (record.target.instanceId === match.instanceId || - record.target.instanceId === match.keyInstanceId); - const latest = listRuns(runsDir).find( - (record) => sameJob(record) && record.sourceHash === match.sourceHash, + const runs = listRuns(runsDir); + // Only a run of this same source, and the same version of a custom one, + // is continued. + const latest = importsOfFile(runs, match).find( + (record) => sameFileAndInstance(record, match) && record.sourceHash === match.sourceHash, ); if (!latest) { - // An edited custom source is a different source, but its earlier run sent - // the same users: starting over beside it would reject every user it - // created and leave it unfinished, so that takes --new-run. - // Any run of it not undone, not just the newest: an older interrupted run - // behind a newer undone one still holds its users. - const earlier = listRuns(runsDir).find( - (record) => sameJob(record) && !undoneInFull(runsDir, record), + // Another source's run of this file, or an earlier version of this one, + // sent the same users: starting over beside it would reject every user it + // created and leave it unfinished, so that takes --new-run. Any such run + // not undone, not just the newest: an older interrupted run behind a + // newer undone one still holds its users. + const earlier = runs.find( + (record) => sameFileAndInstance(record, match) && !undoneInFull(runsDir, record), ); if (earlier) { + const how = + earlier.source === match.source + ? "with an earlier version of this source" + : `with the \`${earlier.source}\` source`; throwUsageError( - `Run ${earlier.id} imported this file with an earlier version of this source, so the users it created would come back as conflicts. Nothing was imported.\n` + - `Pass --new-run to start a new run beside it, or undo it first with \`${withFlags(`clerk migrate undo ${earlier.id}`, carry)}\`.`, + `Run ${earlier.id} imported this file into this instance ${how}, so the users it created would come back as conflicts. Nothing was imported.\n` + + `Pass --new-run to start a new run beside it, or undo it first with \`${undoCommand(earlier.id, carry)}\`.`, ); } // Clerk did not name the instance (GET /v1/instance failed), so a run of // this file recorded under a real ID can be neither matched nor ruled out. // Starting over would reject every user it created and leave it unfinished. if (isStandInInstanceId(match.instanceId)) { - const unconfirmed = listRuns(runsDir).find( + const unconfirmed = runs.find( (record) => record.kind === "import" && record.file?.sha256 === match.sha256 && - record.source === match.source && isClerkInstanceId(record.target.instanceId), ); if (unconfirmed) { @@ -573,12 +581,10 @@ function findResume( } if (state === "undone") return { kind: "new" }; - const openUndo = openUndoOf(runsDir, latest.id); + const openUndo = openUndoOf(runsDir, latest.id, runs); if (openUndo) throwUsageError(unfinishedUndoMessage(latest.id, openUndo.id, carry)); // Undone in full, though the import run was never marked so. - if (listRuns(runsDir).some((record) => record.kind === "undo" && record.undoes === latest.id)) { - return { kind: "new" }; - } + if (undosOf(runs, latest.id).length > 0) return { kind: "new" }; if (state === "complete") return { kind: "complete", record: latest }; return { @@ -925,15 +931,18 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi // The flags every suggested command needs to reach this runs folder and instance. const carry = carryFlags(options); + // A Firebase export through a source you wrote is still Firebase's: its + // digests need the project's parameters, from the envelope or the flags. + const hashSource = options.envelopeSource ?? source; // A flag is typed for this run, so it wins over the export file. A variable // is ambient, and may be left over from another project, so it only fills in // for an export file without the parameters: with one, a partial flag set is // refused rather than completed from the variables. - const fromEnvelope = source === "firebase" ? envelope?.firebase : undefined; + const fromEnvelope = hashSource === "firebase" ? envelope?.firebase : undefined; const fromFlagsAtAll = FIREBASE_FLAGS.some(([key]) => options[key] !== undefined); const fromFlags = fromFlagsAtAll || !fromEnvelope - ? resolveFirebaseHashConfig(options, source, fromEnvelope ? {} : process.env) + ? resolveFirebaseHashConfig(options, hashSource, fromEnvelope ? {} : process.env) : undefined; let firebaseHashConfig = fromFlags ?? fromEnvelope; let configFrom = fromFlags @@ -942,7 +951,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi : "the CLERK_FIREBASE_* variables" : "the export file"; // `-y` is "import without prompting": it gets the usage error naming the flags. - if (source === "firebase" && !firebaseHashConfig && !options.yes && canPrompt(options)) { + if (hashSource === "firebase" && !firebaseHashConfig && !options.yes && canPrompt(options)) { firebaseHashConfig = await promptForFirebaseHashConfig(); configFrom = "the parameters entered"; // A suggested command runs with --yes, which never asks: it names the flags. @@ -997,7 +1006,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi const releases: (() => void)[] = []; try { for (const instanceId of new Set([keyInstanceId(secretKey), target.instanceId])) { - releases.push(lockImport({ sha256, source, instanceId })); + releases.push(lockImport({ sha256, instanceId })); } } catch (error) { for (const release of releases) release(); @@ -1009,7 +1018,6 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi } assertNoActiveImport(runsDir, { sha256, - source, instanceId: target.instanceId, keyInstanceId: keyInstanceId(secretKey), }); From b474a9b975ec32d22527213b1ca06d45849a4ee3 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 16 Oct 2026 16:20:52 -0400 Subject: [PATCH 19/42] fix(migrate): check other unfinished runs behind an undone exact match, and create the lock's temp file exclusively - findResume started a new run when the exact-version run was undone, without checking the other runs of the file. An older version's interrupted run behind it then had its users come back as conflicts. It now checks them, as it does when there's no exact match. - The lock's temporary file gets a random name and `wx`: no planted link at a predictable name in a shared /tmp, and a crashed run's leftover under the same PID never reads as a held lock. - runs says SOURCE_FILE is to be replaced with the source's path. - The README says the export command shows only where one exists, and that a copy of a source beside different helpers also needs --new-run. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 20 ++++++----- .../commands/migrate/lib/run-store.test.ts | 16 +++++++++ .../src/commands/migrate/lib/run-store.ts | 6 ++-- .../cli-core/src/commands/migrate/run.test.ts | 35 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 11 +++--- .../src/commands/migrate/runs.test.ts | 1 + .../cli-core/src/commands/migrate/runs.ts | 6 +++- 7 files changed, 79 insertions(+), 16 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 81c694c6d..4228871c7 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1146,13 +1146,14 @@ clerk migrate sources --json | `[source]` | A built-in key, or the path to a source you wrote, to show fully | | `--json` | The same data, on stdout | -`sources` alone prints the table under [Sources](#sources). `sources ` shows one source in -full: its export command, what it carries with a note for each, where each -field lands (`encrypted_password → password`), its fixed defaults, and any -caveats. An unknown key exits 2 and lists the valid ones. There is no -intro/outro gutter. `sources` alone, or with a built-in key, reads a static -registry and runs nothing. With a path, it loads and runs that file, as -`--source` does, to check it and show what it maps. +`sources` alone prints the table under [Sources](#sources). `sources ` +shows one source in full: its export command (when `clerk migrate export` has +one for it), what it carries with a note for each, where each field lands +(`encrypted_password → password`), its fixed defaults, and any caveats. An +unknown key exits 2 and lists the valid ones. There is no intro/outro gutter. +`sources` alone, or with a built-in key, reads a static registry and runs +nothing. With a path, it loads and runs that file, as `--source` does, to check +it and show what it maps. ### `clerk migrate help` @@ -1244,8 +1245,9 @@ and won't start a new one beside it unasked either: while that run isn't undone, it refuses with exit 2 and names it, since the users it created would come back as conflicts. Pass `--new-run` to start a new run anyway, or undo the earlier one first. The hash covers that file -only: a local helper it imports is not part of it, so after editing one, pass -`--new-run` rather than continue a run made with the old helper. +only: a local helper it imports is not part of it, so after editing one, or +when running a copy of the file beside different helpers, pass `--new-run` +rather than continue a run made with the old ones. #### Validation diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 44b1f9997..e6636720b 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -206,6 +206,22 @@ describe("locks and interruptions", () => { expect(() => continueRun(runsDir, run.record)).not.toThrow(); }); + // A predictable name in a shared /tmp could be a planted symlink, and a + // crashed run's leftover under the same PID would read as a held lock. + test("the lock's temporary file has a random name and is created exclusively", () => { + const identity = { sha256: "b".repeat(64), instanceId: "ins_tmp" }; + const write = spyOn(fs, "writeFileSync"); + try { + const release = lockImport(identity); + const tmp = write.mock.calls.find(([target]) => String(target).endsWith(".tmp")); + expect(tmp?.[2]).toMatchObject({ flag: "wx" }); + expect(String(tmp?.[0])).toMatch(new RegExp(`\\.${process.pid}\\.[0-9a-f]{16}\\.tmp$`)); + release(); + } finally { + write.mockRestore(); + } + }); + // A lock created empty and written after reads as stale to a second // process in between, which would remove it and import beside the first. test("a lock is linked into place with its PID already written", () => { diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 86d9f0c48..e327817ad 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -271,8 +271,10 @@ function acquireLockFile( // as `wx` does, but the lock never exists without its PID, which a second // process would read as stale and remove. const take = () => { - const whole = `${file}.${process.pid}.tmp`; - fs.writeFileSync(whole, String(process.pid), { mode: 0o600 }); + // Random and exclusive: a name no one can plant a link at, and a crashed + // run's leftover (same PID, in a container) never reads as a held lock. + const whole = `${file}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`; + fs.writeFileSync(whole, String(process.pid), { flag: "wx", mode: 0o600 }); try { fs.linkSync(whole, file); } catch (error) { diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 90713ab28..314bcbaec 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2513,6 +2513,41 @@ describe("run", () => { expect(created()).toHaveLength(0); }); + // The newer, undone run is of this same version: undone, it starts a new + // run, so the older interrupted one has to be checked too. + test("refuses while an earlier version's run is unfinished behind an undone run of this version", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const older = interruptOnly(); + const second = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, second), `${CUSTOM}\n// second`); + await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + const newer = listRuns(runsDir()).find((record) => record.id !== older)!; + fs.writeFileSync( + path.join(runsDir(), newer.id, "run.json"), + JSON.stringify({ ...readRun(runsDir(), newer.id), status: "undone" }), + ); + requests = []; + + const error = (await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain( + `Run ${older} imported this file into this instance with an earlier version`, + ); + expect(created()).toHaveLength(0); + }); + test("an edited source with --new-run starts a new run beside the earlier one", async () => { await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); interruptOnly(); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index ea4f22d3f..f0f0a7b2d 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -534,7 +534,8 @@ function findResume( const latest = importsOfFile(runs, match).find( (record) => sameFileAndInstance(record, match) && record.sourceHash === match.sourceHash, ); - if (!latest) { + // An undone exact match starts a new run, so it checks the others too. + if (!latest || undoneInFull(runsDir, latest)) { // Another source's run of this file, or an earlier version of this one, // sent the same users: starting over beside it would reject every user it // created and leave it unfinished, so that takes --new-run. Any such run @@ -545,9 +546,11 @@ function findResume( ); if (earlier) { const how = - earlier.source === match.source - ? "with an earlier version of this source" - : `with the \`${earlier.source}\` source`; + earlier.source !== match.source + ? `with the \`${earlier.source}\` source` + : earlier.sourceHash !== match.sourceHash + ? "with an earlier version of this source" + : "too, and was not undone"; throwUsageError( `Run ${earlier.id} imported this file into this instance ${how}, so the users it created would come back as conflicts. Nothing was imported.\n` + `Pass --new-run to start a new run beside it, or undo it first with \`${undoCommand(earlier.id, carry)}\`.`, diff --git a/packages/cli-core/src/commands/migrate/runs.test.ts b/packages/cli-core/src/commands/migrate/runs.test.ts index 655537637..d9b541333 100644 --- a/packages/cli-core/src/commands/migrate/runs.test.ts +++ b/packages/cli-core/src/commands/migrate/runs.test.ts @@ -414,6 +414,7 @@ describe("runs ", () => { const { next } = JSON.parse(captured.out) as { next: string[] }; expect(next[0]).toContain(`clerk migrate import ${quoteArg(usersFile())} --source SOURCE_FILE`); + expect(next[0]).toContain("with SOURCE_FILE replaced by the path to the source you wrote"); }); test("an unknown ID is a usage error", async () => { diff --git a/packages/cli-core/src/commands/migrate/runs.ts b/packages/cli-core/src/commands/migrate/runs.ts index 3e14c0934..427c7fe93 100644 --- a/packages/cli-core/src/commands/migrate/runs.ts +++ b/packages/cli-core/src/commands/migrate/runs.ts @@ -182,7 +182,11 @@ function nextCommands( // Continuing is what a stopped run is for: the same file, source and // target pick it up where it left off. const why = state === "interrupted" ? "continue it" : "retry the users it didn't finish"; - return [`Run \`${again}\` to ${why}`, undo]; + // The record knows a source you wrote only by its hash. + const placeholder = record.sourceHash + ? ", with SOURCE_FILE replaced by the path to the source you wrote" + : ""; + return [`Run \`${again}\` to ${why}${placeholder}`, undo]; } if (record.kind === "undo" && state !== "complete" && record.undoes) { return [`Run \`${undoCommand(record.undoes, carry)}\` to retry the users that failed`]; From 06cbbc0358fe655c3581dc470957bd0c94cd0a89 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 16 Oct 2026 19:34:24 -0400 Subject: [PATCH 20/42] fix(migrate): name an earlier run of any source when a refusal meets its users earlierRunHolding still skipped runs of another source, though findResume and the import lock match on file and instance only. A custom-source --new-run beside a built-in run of the same file got the generic "Fix them" text instead of the run holding its users. It now matches any source. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 30 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 8 ++--- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 314bcbaec..9017011a0 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -2378,6 +2378,36 @@ describe("run", () => { ); }); + // The checks reject the users a run under another source created, so the + // refusal names that run, as it does for one of the same source. + test("a refusal names the earlier run of another source that holds the users", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const [first] = listRuns(runsDir()); + const other = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, other), + CUSTOM.replace('key: "myplatform"', 'key: "otherplatform"'), + ); + stubClerk({ + existing: [ + { id: "user_1", external_id: "mp_1" }, + { id: "user_2", external_id: "mp_2" }, + ], + }); + + const error = (await run({ + input: "export.json", + source: other, + newRun: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.message).toContain(`2 of them were created by run ${first!.id}`); + expect(error.examples?.map((example) => example.description)).toContain( + `Undo run ${first!.id} first`, + ); + }); + test("imports through a user-authored source", async () => { await run({ input: "export.json", diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index f0f0a7b2d..4de223909 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -804,14 +804,13 @@ function rejectSummary(checks: ImportChecks, json = false): string { } /** - * The earlier import run, of this source into this instance, that created the + * The earlier import run into this instance, of any source, that created the * users the checks reject as already there: how many of them, and whether it * ran this same file. Fixing a skipped row changes the file, and the changed * file's run meets them; so does the same file run with `--new-run`. */ function earlierRunHolding( runsDir: string, - source: string, instanceId: string, checks: ImportChecks, sha256: string, @@ -821,7 +820,8 @@ function earlierRunHolding( ); if (already.size === 0) return undefined; for (const record of listRuns(runsDir)) { - if (record.kind !== "import" || record.source !== source) continue; + // Any source: findResume and the import lock match on file and instance. + if (record.kind !== "import") continue; if (record.target.instanceId !== instanceId || runState(runsDir, record) === "undone") continue; let count = 0; for (const line of latestUserLines(runsDir, record.id).values()) { @@ -1258,7 +1258,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi if (refused) { if (options.json) preview({ refused: true }); - const earlier = earlierRunHolding(runsDir, source, target.instanceId, checks, sha256); + const earlier = earlierRunHolding(runsDir, target.instanceId, checks, sha256); const undoEarlier = earlier ? undoCommand(earlier.id, carry) : undefined; const earlierRun = options.newRun && earlier From a8455e1ee19fd3161a81b32ea8d86550c7425d20 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 21 Oct 2026 16:15:03 -0400 Subject: [PATCH 21/42] fix(migrate): include a source's fixed defaults in `sources --json` The README says --json carries the same detail as the human view, fixed defaults included, but the serializer left them out. A source with none gets an empty object. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/sources/list.test.ts | 2 ++ packages/cli-core/src/commands/migrate/sources/list.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/sources/list.test.ts b/packages/cli-core/src/commands/migrate/sources/list.test.ts index a28a89cd1..e0214fe3e 100644 --- a/packages/cli-core/src/commands/migrate/sources/list.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/list.test.ts @@ -108,6 +108,8 @@ describe("sources ", () => { key: "auth0", export_command: "clerk migrate export auth0", fields: { user_id: "userId" }, + // The fixed defaults the detail view prints, as the README says. + defaults: { passwordHasher: "bcrypt" }, carries: { passwords: { level: "partial" } }, }); }); diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts index 7991bb05a..27782e743 100644 --- a/packages/cli-core/src/commands/migrate/sources/list.ts +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -86,6 +86,7 @@ function toJson(entry: SourceEntry, custom?: string) { carries: entry.carries, export_command: exportCommand(entry) ?? null, fields: entry.transformer, + defaults: entry.defaults ?? {}, caveats: entry.caveats ?? [], }; } From 085c9d540311e7bb885c89be458c223c4f67bb80 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 22 Oct 2026 15:26:17 -0400 Subject: [PATCH 22/42] docs(migrate): say a Ctrl-C can leave the import lock behind The SIGINT handler kills the process after a short telemetry flush, so run()'s finally doesn't always release the lock. The next import reclaims it. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 2 +- packages/cli-core/src/commands/migrate/run.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 4228871c7..77504ccd3 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -188,7 +188,7 @@ and the instance, not the source, so a second import of the same file into the same instance is refused with exit 2 even before either has written a run, whichever source and `--runs-dir` each uses. It lives outside the runs folder because nothing may be written there before consent. It holds the importing process's -PID and is removed however the import ends. One left by a process that died is +PID and is removed when the import finishes. One left by a Ctrl-C or a process that died is reclaimed by the next import. The refusal names the file, which is safe to delete when no `clerk migrate import` is running. diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 4de223909..4e64605ae 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -891,7 +891,8 @@ function recordRejects(run: Run, checks: ImportChecks): void { } export async function run(rawOptions: MigrateRunOptions): Promise { - // Released however the import ends: a return, a refusal or a Ctrl-C. + // Released on a return or a refusal. A Ctrl-C can kill the process first; + // the next import reclaims a lock whose PID is gone. const lock: ImportLock = {}; try { await runImport(rawOptions, lock); From f9e3c1ecd5a36df9c61c8190e8493d06a0dcbb09 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 22 Oct 2026 15:27:08 -0400 Subject: [PATCH 23/42] fix(migrate): point source authors at the Custom sources docs An agent with only the CLI had no way to learn a source file's shape: the `sources` footer and `--source` help didn't describe it, and every load error linked to the old migration-tool's overview. All three now link the README's Custom sources section. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/index.ts | 3 ++- .../src/commands/migrate/sources/list.test.ts | 1 + .../cli-core/src/commands/migrate/sources/list.ts | 2 ++ .../src/commands/migrate/sources/load-custom.test.ts | 11 +++++++++++ .../src/commands/migrate/sources/load-custom.ts | 12 +++++++----- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index 0e27c76d2..ffc826d7f 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -8,6 +8,7 @@ import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; import { run } from "./run.ts"; import { runs } from "./runs.ts"; import { undo } from "./undo.ts"; +import { CUSTOM_SOURCES_URL } from "./sources/load-custom.ts"; import { list as sources } from "./sources/list.ts"; export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = process.env): void { @@ -84,7 +85,7 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") .option( "--source ", - "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`, though a source you wrote replaces its own", + `Where the file came from: a built-in source, or a source you wrote (${CUSTOM_SOURCES_URL}). Not needed for a file from \`clerk migrate export\`, though a source you wrote replaces its own`, ) .option("--dry-run", "Check the file against the instance, report, and write nothing") .option("--allow-partial", "Import the users that pass the checks, and skip the rest") diff --git a/packages/cli-core/src/commands/migrate/sources/list.test.ts b/packages/cli-core/src/commands/migrate/sources/list.test.ts index e0214fe3e..0529ded1e 100644 --- a/packages/cli-core/src/commands/migrate/sources/list.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/list.test.ts @@ -62,6 +62,7 @@ describe("sources", () => { test("says how to add one", async () => { await list(undefined); expect(captured.err).toContain("--source ./my-source.ts"); + expect(captured.err).toContain("README.md#custom-sources"); }); test("--json lists every built-in with its carries on stdout", async () => { diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts index 27782e743..850af7578 100644 --- a/packages/cli-core/src/commands/migrate/sources/list.ts +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -12,6 +12,7 @@ import { bold, cyan, dim, green, red, yellow } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { exportPlatforms } from "../export/registry.ts"; import type { Carry, CarryLevel, SourceEntry } from "../types.ts"; +import { CUSTOM_SOURCES_URL } from "./load-custom.ts"; import { ACCOUNT_LINKING_NOTE, resolveSource, sources as builtIns } from "./registry.ts"; export type SourcesOptions = { @@ -121,6 +122,7 @@ function printList(width: number): void { log.blank(); log.info("Run `clerk migrate sources ` for what each field becomes."); log.info("Migrating from something else? Write a source and pass --source ./my-source.ts."); + log.info(`How to write one: ${CUSTOM_SOURCES_URL}`); } function printDetail(entry: SourceEntry, width: number, custom?: string): void { diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts index 73d518d82..6271376c2 100644 --- a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts @@ -158,6 +158,17 @@ describe("validateSource", () => { expect(validateSource(valid, "f.ts").key).toBe("myplatform"); }); + // The old migration-tool's docs say nothing about this file, and an agent + // with only the CLI learns the format from wherever the error points. + test("points a rejection at how to write a source", () => { + try { + validateSource(null, "f.ts"); + throw new Error("expected a rejection"); + } catch (error) { + expect((error as CliError).docsUrl).toContain("README.md#custom-sources"); + } + }); + test.each([ ["a null default export", null, /is null, not an object/], ["a number default export", 42, /is number, not an object/], diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts index 9c1774e6c..b9c6e3959 100644 --- a/packages/cli-core/src/commands/migrate/sources/load-custom.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -23,12 +23,14 @@ import path from "node:path"; import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; import type { CarryLevel, SourceEntry } from "../types.ts"; -const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; +/** How to write a source. The README ships nowhere, so link the published copy. */ +export const CUSTOM_SOURCES_URL = + "https://github.com/clerk/cli/blob/main/packages/cli-core/src/commands/migrate/README.md#custom-sources"; const LEVELS: readonly CarryLevel[] = ["yes", "no", "partial"]; function invalid(problem: string, file: string): never { - throwUsageError(`${file} is not a valid source: ${problem}`, DOCS_URL); + throwUsageError(`${file} is not a valid source: ${problem}`, CUSTOM_SOURCES_URL); } /** @@ -163,7 +165,7 @@ export async function loadCustomSource( if (!fs.existsSync(resolved)) { throw new CliError(`No source file at ${resolved}.`, { code: ERROR_CODE.FILE_NOT_FOUND, - docsUrl: DOCS_URL, + docsUrl: CUSTOM_SOURCES_URL, }); } if (fs.statSync(resolved).isDirectory()) { @@ -187,7 +189,7 @@ export async function loadCustomSource( throwUsageError( `Could not load ${file}: ${(error as Error).message}\n` + "The file must be valid JavaScript or TypeScript that this CLI can import.", - DOCS_URL, + CUSTOM_SOURCES_URL, ); } @@ -198,7 +200,7 @@ export async function loadCustomSource( named.length > 0 ? ` Found named export${named.length === 1 ? "" : "s"} ${named.map((n) => `\`${n}\``).join(", ")} — did you mean \`export default\`?` : ""; - throwUsageError(`${file} has no default export.${hint}`, DOCS_URL); + throwUsageError(`${file} has no default export.${hint}`, CUSTOM_SOURCES_URL); } return validateSource(module.default, file, reservedKeys); From 72ac1d08b0c98e56192a1a60e3ad75de0be68da5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 22 Oct 2026 15:28:08 -0400 Subject: [PATCH 24/42] fix(migrate): reject an async postTransform instead of losing its changes transformUsers calls postTransform synchronously, so an async hook's changes after its first await landed once the user was already normalized and validated. The only trace was a "Clerk won't store" line for the column it meant to consume. A hook that returns a promise now stops the import and names the source. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/transform.test.ts | 19 +++++++++++++++++++ .../src/commands/migrate/lib/transform.ts | 14 ++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 766ae00e7..bc8d479e7 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -204,6 +204,25 @@ describe("transformUsers", () => { }); }); + test("rejects an async postTransform rather than lose its changes", () => { + registerCustomSource({ + ...clerkSource, + key: "async-post", + // A source file is untyped at runtime, so nothing stops it being async. + postTransform: (async (user: Record) => { + await Promise.resolve(); + user.firstName = "late"; + }) as unknown as SourceEntry["postTransform"], + }); + try { + expect(() => transformUsers([{ id: "u1" }], "async-post")).toThrow( + /"async-post" source's postTransform returned a promise/, + ); + } finally { + __resetCustomSourcesForTesting(); + } + }); + test("skips validation when asked, so analysis passes see every row", () => { const { transformedData, validationFailed } = transformUsers([{ id: "u1" }], "clerk", { validate: false, diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 5daae2e23..079bd7c6d 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -10,7 +10,8 @@ import fs from "node:fs"; import path from "node:path"; import csvParser from "csv-parser"; -import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { CUSTOM_SOURCES_URL } from "../sources/load-custom.ts"; import { getSource } from "../sources/registry.ts"; import { normalizeBooleanField, parseDate } from "../sources/shared.ts"; import { type TransformContext, type SourceEntry, type User } from "../types.ts"; @@ -380,7 +381,16 @@ export function transformUsers( // The source's own cleanup first: the Clerk Dashboard's formula-safety TAB // has to come off before identifiers are compared, or "\t+1555…" and // "+1555…" read as two phones. - transformer.postTransform?.(mapped, context); + const result: unknown = transformer.postTransform?.(mapped, context); + // An async hook's changes after its first await land once this user is + // already normalized and validated, so they would be lost without a word. + if (result instanceof Promise) { + result.catch(() => undefined); + throwUsageError( + `The "${key}" source's postTransform returned a promise. It must be synchronous: changes made after an await never reach the import.`, + CUSTOM_SOURCES_URL, + ); + } if (key === "clerk") { consolidateClerkIdentifiers(mapped); } From ef61c1337fcd3f9201bc11ebdd8e1bb5a67cbb1b Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 22 Oct 2026 15:28:27 -0400 Subject: [PATCH 25/42] fix(migrate): name the source and row when postTransform throws MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A throwing hook reached the global handler as `unexpected_error: boom`, with nothing pointing at the source file or the user that tripped it. It now reads `The "" source's postTransform failed on row : …` and links the Custom sources docs. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/transform.test.ts | 17 +++++++++++++++++ .../src/commands/migrate/lib/transform.ts | 13 +++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index bc8d479e7..d70801bea 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -223,6 +223,23 @@ describe("transformUsers", () => { } }); + test("names the source and the row when postTransform throws", () => { + registerCustomSource({ + ...clerkSource, + key: "throwing-post", + postTransform: (user) => { + if (user.userId === "u2") throw new Error("boom"); + }, + }); + try { + expect(() => transformUsers([{ id: "u1" }, { id: "u2" }], "throwing-post")).toThrow( + 'The "throwing-post" source\'s postTransform failed on row 2: boom', + ); + } finally { + __resetCustomSourcesForTesting(); + } + }); + test("skips validation when asked, so analysis passes see every row", () => { const { transformedData, validationFailed } = transformUsers([{ id: "u1" }], "clerk", { validate: false, diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 079bd7c6d..74ed33201 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -375,13 +375,22 @@ export function transformUsers( const context = options.context ?? {}; const transformed: Record[] = []; - for (const user of users) { + for (const [i, user] of users.entries()) { const mapped = transformKeys(user, transformer); // The source's own cleanup first: the Clerk Dashboard's formula-safety TAB // has to come off before identifiers are compared, or "\t+1555…" and // "+1555…" read as two phones. - const result: unknown = transformer.postTransform?.(mapped, context); + let result: unknown; + try { + result = transformer.postTransform?.(mapped, context); + } catch (error) { + // 1-based, like the validation failures: the second user is row 2. + throw new CliError( + `The "${key}" source's postTransform failed on row ${i + 1}: ${error instanceof Error ? error.message : String(error)}`, + { docsUrl: CUSTOM_SOURCES_URL }, + ); + } // An async hook's changes after its first await land once this user is // already normalized and validated, so they would be lost without a word. if (result instanceof Promise) { From e5cc22bea8095d13425ab49f1c6a8f0e4e04df39 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 22 Oct 2026 15:29:06 -0400 Subject: [PATCH 26/42] docs(migrate): write out the custom source hooks' contract The README named postTransform and preTransform but not what they get, what they return, or that postTransform must be synchronous. Spell out both signatures, what happens when a hook is async or throws, and that a transformer target outside Clerk's fields is only a rename. The example now splits a full_name, the common case for a homegrown table. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 77504ccd3..c497a5f08 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1218,9 +1218,8 @@ export default { transformer: { account_ref: "userId", // required: becomes the Clerk user's external_id contact_email: "email", - given: "firstName", - family: "lastName", pw_bcrypt: "password", + // full_name is left unmapped: postTransform splits it below. }, carries: { passwords: { level: "yes", note: "bcrypt hashes from the pw_bcrypt column." }, @@ -1229,11 +1228,38 @@ export default { }, defaults: { passwordHasher: "bcrypt" }, postTransform: (user) => { - if (!user.firstName) delete user.firstName; + // Unmapped columns arrive under their own names. + const [first, ...rest] = String(user.full_name ?? "") + .trim() + .split(/\s+/); + if (first) user.firstName = first; + if (rest.length) user.lastName = rest.join(" "); + delete user.full_name; }, }; ``` +Two optional hooks run around the `transformer` mapping: + +- `postTransform(user, context)` runs once per user, after the mapping. `user` + holds the mapped fields, plus every column the `transformer` doesn't name, + still under its own name. Change `user` in place; the return value is + ignored. It must be synchronous: an `async` hook, or one that returns a + promise, stops the import, since its changes would arrive after the user + was checked. One that throws stops the import with the source's key, the + row number and the error. `context` holds per-run values a built-in needs, + such as Firebase's hash parameters; a custom source can ignore it. +- `preTransform(filePath, fileType)` runs once, before the file is read, and + may be `async`. `fileType` is `"application/json"`, `"text/csv"` or `""`. + Return `{ filePath, data?, csvHeaders? }`: `filePath` is the file to read + (the same one, or one you wrote), `data` is the users when you parsed them + yourself and skips reading the file, and `csvHeaders` names the columns of + a CSV with no header row. It isn't called for a file from + `clerk migrate export`, which holds its users already. + +A `transformer` target that isn't a Clerk field, like `full_name: "name"`, is +only a rename: Clerk won't store it unless `postTransform` moves it. + TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, `satisfies` and `as const` all work in a file the compiled binary imports. Plain `.js` works too. From 26a14edaa35919c420e52ebc4f3503c2096d5962 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 22 Oct 2026 15:29:49 -0400 Subject: [PATCH 27/42] fix(migrate): mark field targets that aren't Clerk fields in `sources` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Where each field lands" listed keys a built-in's postTransform consumes (`name`, `salt`, `workosExternalId`) as if Clerk stored them. Copied into a source you wrote, `full_name → name` showed with no warning and was dropped. A built-in's such target is now marked as moved by the source; a custom source's gets a warning that Clerk won't store it unless its postTransform moves it. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/transform.ts | 2 +- .../src/commands/migrate/sources/list.test.ts | 16 +++++++++++++++- .../src/commands/migrate/sources/list.ts | 12 +++++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 74ed33201..8407ba29c 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -295,7 +295,7 @@ export function consolidateClerkIdentifiers(user: Record): void // --- Validation ------------------------------------------------------------ /** Every field the import schema declares; anything else is stripped. */ -const SCHEMA_FIELDS: ReadonlySet = new Set(Object.keys(userSchema.shape)); +export const SCHEMA_FIELDS: ReadonlySet = new Set(Object.keys(userSchema.shape)); /** * Validates prepared users, dropping each failure from the run and returning diff --git a/packages/cli-core/src/commands/migrate/sources/list.test.ts b/packages/cli-core/src/commands/migrate/sources/list.test.ts index 0529ded1e..3acb96cc2 100644 --- a/packages/cli-core/src/commands/migrate/sources/list.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/list.test.ts @@ -25,7 +25,7 @@ beforeAll(() => { key: "myplatform", label: "My Platform", description: "Exports from My Platform.", - transformer: { account_ref: "userId", mail: "email" }, + transformer: { account_ref: "userId", mail: "email", full_name: "name" }, carries: { passwords: { level: "partial", note: "Only for accounts made after 2020." }, mfa: { level: "no", note: "None." }, @@ -102,6 +102,20 @@ describe("sources ", () => { expect(plain()).toContain("Only for accounts made after 2020."); }); + // `name` reads like a Clerk field and isn't one: copied from a built-in into + // a source you wrote, it was dropped with no word until the import's checks. + test("marks a target that isn't a Clerk field", async () => { + await list("authjs"); + expect(plain()).toContain("name → name not a Clerk field; the source moves it"); + expect(plain()).not.toContain("email → email not a Clerk field"); + + captured.clear(); + await list("./custom.ts"); + expect(plain()).toContain( + "full_name → name not a Clerk field: Clerk won't store it unless postTransform moves it", + ); + }); + test("--json returns the detail on stdout", async () => { await list("auth0", { json: true }); diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts index 850af7578..756ec4292 100644 --- a/packages/cli-core/src/commands/migrate/sources/list.ts +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -11,6 +11,7 @@ import { bold, cyan, dim, green, red, yellow } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { exportPlatforms } from "../export/registry.ts"; +import { SCHEMA_FIELDS } from "../lib/transform.ts"; import type { Carry, CarryLevel, SourceEntry } from "../types.ts"; import { CUSTOM_SOURCES_URL } from "./load-custom.ts"; import { ACCOUNT_LINKING_NOTE, resolveSource, sources as builtIns } from "./registry.ts"; @@ -147,7 +148,16 @@ function printDetail(entry: SourceEntry, width: number, custom?: string): void { const fromWidth = Math.max(...fields.map(([from]) => from.length)) + 2; log.blank(); log.info(bold("Where each field lands")); - for (const [from, to] of fields) log.info(` ${from.padEnd(fromWidth)}→ ${to}`); + for (const [from, to] of fields) { + // A built-in maps some columns to keys its postTransform consumes. In a + // source you wrote, the same is a column Clerk drops unless yours moves it. + const note = SCHEMA_FIELDS.has(to) + ? "" + : custom + ? yellow(" not a Clerk field: Clerk won't store it unless postTransform moves it") + : dim(" not a Clerk field; the source moves it"); + log.info(` ${from.padEnd(fromWidth)}→ ${to}${note}`); + } if (entry.defaults) { for (const [field, value] of Object.entries(entry.defaults)) { log.info(dim(` ${field} is always ${JSON.stringify(value)}`)); From 28aa6c7311d8a4094bb714dc796e2cd2572d83bd Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 22 Oct 2026 17:12:24 -0400 Subject: [PATCH 28/42] fix(migrate): report a directory passed as --source as a usage error resolveSource hashed the path before loadCustomSource checked it, so a directory threw a raw EISDIR (exit 1, unexpected_error in agent mode) ahead of the "is a directory, not a source file" usage error. Only a file is hashed now. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/sources/load-custom.test.ts | 9 ++++++++- .../cli-core/src/commands/migrate/sources/registry.ts | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts index 6271376c2..af604ab25 100644 --- a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import { loadCustomSource, validateSource } from "./load-custom.ts"; -import { __resetCustomSourcesForTesting } from "./registry.ts"; +import { __resetCustomSourcesForTesting, resolveSource } from "./registry.ts"; let workDir: string; let originalCwd: string; @@ -113,6 +113,13 @@ describe("loadCustomSource", () => { await expect(loadCustomSource("./adir")).rejects.toThrow(/is a directory/); }); + // `--source` hashes the file before loading it, and reading a directory + // threw a raw EISDIR ahead of the usage error. + test("--source reports a directory as a usage error, not EISDIR", async () => { + fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); + await expect(resolveSource("./adir")).rejects.toThrow(/is a directory/); + }); + test("reports a file that does not parse, quoting the syntax error", async () => { await expect(loadCustomSource(writeSource("export default { key: ,,, }"))).rejects.toThrow( /Could not load/, diff --git a/packages/cli-core/src/commands/migrate/sources/registry.ts b/packages/cli-core/src/commands/migrate/sources/registry.ts index 257732d9a..6612eb612 100644 --- a/packages/cli-core/src/commands/migrate/sources/registry.ts +++ b/packages/cli-core/src/commands/migrate/sources/registry.ts @@ -113,7 +113,8 @@ export async function resolveSource(value: string): Promise { const resolved = path.resolve(process.cwd(), value); // Hashed first, and loaded by that hash: the record names the very code // that maps the users, even if the file was edited since a last load. - const hash = fs.existsSync(resolved) + // Only a file: loadCustomSource reports a missing path or a directory. + const hash = fs.statSync(resolved, { throwIfNoEntry: false })?.isFile() ? createHash("sha256").update(fs.readFileSync(resolved)).digest("hex") : undefined; const entry = await loadCustomSource(value, sourceKeys(), hash); From bf9111735eaaf86ef91b7f4860a0a065af39479f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 22 Oct 2026 17:12:41 -0400 Subject: [PATCH 29/42] fix(migrate): reject any thenable a postTransform returns The async-hook check used `instanceof Promise`, so a promise from another library, or another realm, still slipped through and its changes were lost. Any result with a callable `then` now stops the import. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/transform.test.ts | 15 +++++++++++++++ .../src/commands/migrate/lib/transform.ts | 5 +++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index d70801bea..be5ce91cd 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -223,6 +223,21 @@ describe("transformUsers", () => { } }); + test("rejects a postTransform that returns a non-native thenable", () => { + registerCustomSource({ + ...clerkSource, + key: "thenable-post", + postTransform: (() => ({ then: () => undefined })) as unknown as SourceEntry["postTransform"], + }); + try { + expect(() => transformUsers([{ id: "u1" }], "thenable-post")).toThrow( + /"thenable-post" source's postTransform returned a promise/, + ); + } finally { + __resetCustomSourcesForTesting(); + } + }); + test("names the source and the row when postTransform throws", () => { registerCustomSource({ ...clerkSource, diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 8407ba29c..4098df839 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -393,8 +393,9 @@ export function transformUsers( } // An async hook's changes after its first await land once this user is // already normalized and validated, so they would be lost without a word. - if (result instanceof Promise) { - result.catch(() => undefined); + // Any thenable, not only a native Promise: a hook may use another library's. + if (typeof (result as PromiseLike | undefined)?.then === "function") { + Promise.resolve(result).catch(() => undefined); throwUsageError( `The "${key}" source's postTransform returned a promise. It must be synchronous: changes made after an await never reach the import.`, CUSTOM_SOURCES_URL, From 956763a7d658c0488805a7cbb71bf8fcc79ffece Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 23 Oct 2026 14:08:58 -0400 Subject: [PATCH 30/42] fix(migrate): keep a postTransform's own error and code Wrapping every postTransform throw in an uncoded CliError dropped the code, so agent mode printed plain text instead of the JSON error. It also swallowed Firebase's own refusal for hashes with no parameters, turning its usage error (exit 2, Firebase guide) into exit 1 with the custom sources docs. A CliError now passes through unchanged, and any other throw is a usage error naming the source and row. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/transform.test.ts | 25 +++++++++++++++++-- .../src/commands/migrate/lib/transform.ts | 7 ++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index be5ce91cd..e31775154 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -2,7 +2,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { CliError } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, EXIT_CODE } from "../../../lib/errors.ts"; import clerkSource from "../sources/clerk.ts"; import type { SourceEntry } from "../types.ts"; import { __resetCustomSourcesForTesting, registerCustomSource } from "../sources/registry.ts"; @@ -248,13 +248,34 @@ describe("transformUsers", () => { }); try { expect(() => transformUsers([{ id: "u1" }, { id: "u2" }], "throwing-post")).toThrow( - 'The "throwing-post" source\'s postTransform failed on row 2: boom', + expect.objectContaining({ + message: 'The "throwing-post" source\'s postTransform failed on row 2: boom', + code: ERROR_CODE.USAGE_ERROR, + exitCode: EXIT_CODE.USAGE, + }), ); } finally { __resetCustomSourcesForTesting(); } }); + test("passes a postTransform's own CliError through with its code", () => { + const hashed = { + localId: "f1", + email: "a@example.com", + passwordHash: "SGFzaA==", + salt: "U2FsdA==", + }; + expect(() => transformUsers([hashed], "firebase")).toThrow( + expect.objectContaining({ + message: expect.stringMatching(/^This export contains Firebase password hashes/), + code: ERROR_CODE.USAGE_ERROR, + exitCode: EXIT_CODE.USAGE, + docsUrl: expect.stringContaining("/migrating/firebase"), + }), + ); + }); + test("skips validation when asked, so analysis passes see every row", () => { const { transformedData, validationFailed } = transformUsers([{ id: "u1" }], "clerk", { validate: false, diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 4098df839..cea8a8221 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -385,10 +385,13 @@ export function transformUsers( try { result = transformer.postTransform?.(mapped, context); } catch (error) { + // A refusal the source meant, like Firebase's missing hash parameters, + // already says what to do and carries its own code. + if (error instanceof CliError) throw error; // 1-based, like the validation failures: the second user is row 2. - throw new CliError( + throwUsageError( `The "${key}" source's postTransform failed on row ${i + 1}: ${error instanceof Error ? error.message : String(error)}`, - { docsUrl: CUSTOM_SOURCES_URL }, + CUSTOM_SOURCES_URL, ); } // An async hook's changes after its first await land once this user is From ea768f2fefc022afe34b756913ec0082e815d794 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 23 Oct 2026 14:10:05 -0400 Subject: [PATCH 31/42] fix(migrate): say a built-in source handles a field it doesn't store `clerk migrate sources ` said "the source moves it" for every target that isn't a Clerk field, but Supabase's `deletedAt` and `isAnonymous` only decide whether to skip the user and are then dropped. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/sources/list.test.ts | 2 +- packages/cli-core/src/commands/migrate/sources/list.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/sources/list.test.ts b/packages/cli-core/src/commands/migrate/sources/list.test.ts index 3acb96cc2..e452580f3 100644 --- a/packages/cli-core/src/commands/migrate/sources/list.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/list.test.ts @@ -106,7 +106,7 @@ describe("sources ", () => { // a source you wrote, it was dropped with no word until the import's checks. test("marks a target that isn't a Clerk field", async () => { await list("authjs"); - expect(plain()).toContain("name → name not a Clerk field; the source moves it"); + expect(plain()).toContain("name → name not a Clerk field; the source handles it"); expect(plain()).not.toContain("email → email not a Clerk field"); captured.clear(); diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts index 756ec4292..af69263d5 100644 --- a/packages/cli-core/src/commands/migrate/sources/list.ts +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -155,7 +155,7 @@ function printDetail(entry: SourceEntry, width: number, custom?: string): void { ? "" : custom ? yellow(" not a Clerk field: Clerk won't store it unless postTransform moves it") - : dim(" not a Clerk field; the source moves it"); + : dim(" not a Clerk field; the source handles it"); log.info(` ${from.padEnd(fromWidth)}→ ${to}${note}`); } if (entry.defaults) { From 22d464e907d6a4663f148f26c7f41657e0d50bc3 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 27 Oct 2026 15:24:40 -0400 Subject: [PATCH 32/42] fix(migrate): keep file completion for a custom source path `clerk migrate sources ./` and `--source ./` completed nothing: both suggest the built-in keys with NO_FILE_COMP, which turns off the shell's file completion, though each also takes a path to a source you wrote. The source keys now come back with the default directive, so the shell still completes files. A positional now keeps its argument's directive instead of always NO_FILE_COMP; every other argument still returns NO_FILE_COMP, so nothing else changes. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/completion/__complete.ts | 22 ++++++++++++++----- .../src/test/integration/completion.test.ts | 18 ++++++--------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/packages/cli-core/src/commands/completion/__complete.ts b/packages/cli-core/src/commands/completion/__complete.ts index ad53d1ba7..383177335 100644 --- a/packages/cli-core/src/commands/completion/__complete.ts +++ b/packages/cli-core/src/commands/completion/__complete.ts @@ -28,6 +28,16 @@ function noFileComp(completions: Completion[]): CompletionResult { return { completions, directive: DIRECTIVE.NO_FILE_COMP }; } +/** The known values that start with `partial`, from a registry entry. */ +function matching(known: Completion[], partial: string): CompletionResult { + const completions = known.filter((c) => c.name.startsWith(partial)); + // A migrate source can be a path to one you wrote, so the shell still + // completes files beside the built-in keys. + return known === SOURCE_COMPLETIONS + ? { completions, directive: DIRECTIVE.DEFAULT } + : noFileComp(completions); +} + const HTTP_METHOD_COMPLETIONS: Completion[] = [ { name: "GET", description: "Read resource" }, { name: "POST", description: "Create resource" }, @@ -134,13 +144,15 @@ export function generateCompletions(root: CommandUnknownOpts, args: string[]): C } // Combine subcommands + argument choices + options + const argument = completeArguments(command, partial, positionalCount); const completions = [ ...completeSubcommands(command, partial).completions, - ...completeArguments(command, partial, positionalCount).completions, + ...argument.completions, ...completeOptions(command, "", usedOptions).completions, ]; - return noFileComp(completions); + // The argument decides file completion: on only for one that can be a path. + return { completions, directive: argument.directive }; } // ── Tree walking ───────────────────────────────────────────────────────────── @@ -258,10 +270,10 @@ function completeArguments( const path = commandPath(cmd); const registered = KNOWN_POSITIONAL_COMPLETIONS[path]; if (registered) { - return noFileComp(registered.filter((c) => c.name.startsWith(partial))); + return matching(registered, partial); } - return EMPTY_DEFAULT; + return EMPTY_NO_FILE; } function commandPath(cmd: CommandUnknownOpts): string { @@ -294,7 +306,7 @@ function completeOptionValue(opt: Option, flag: string, partial: string): Comple const candidates = resolveOptionValues(opt, flag); if (!candidates) return EMPTY_DEFAULT; - return noFileComp(candidates.filter((c) => c.name.startsWith(partial))); + return matching(candidates, partial); } function resolveOptionValues(opt: Option, flag: string): Completion[] | undefined { diff --git a/packages/cli-core/src/test/integration/completion.test.ts b/packages/cli-core/src/test/integration/completion.test.ts index 886fbce93..d27b152d9 100644 --- a/packages/cli-core/src/test/integration/completion.test.ts +++ b/packages/cli-core/src/test/integration/completion.test.ts @@ -239,12 +239,10 @@ describe("generateCompletions", () => { const saved = process.env.CLERK_EXPERIMENTAL; process.env.CLERK_EXPERIMENTAL = "migrate"; try { - const names = generateCompletions(createProgram(), [ - "migrate", - "import", - "--source", - "", - ]).completions.map((c) => c.name); + const result = generateCompletions(createProgram(), ["migrate", "import", "--source", ""]); + // A source you wrote is a path, so the shell's file completion stays on. + expect(result.directive).toBe(0); + const names = result.completions.map((c) => c.name); expect(names).toEqual([ "clerk", "auth0", @@ -338,11 +336,9 @@ describe("generateCompletions", () => { const saved = process.env.CLERK_EXPERIMENTAL; process.env.CLERK_EXPERIMENTAL = "migrate"; try { - const names = generateCompletions(createProgram(), [ - "migrate", - "sources", - "", - ]).completions.map((c) => c.name); + const result = generateCompletions(createProgram(), ["migrate", "sources", ""]); + expect(result.directive).toBe(0); + const names = result.completions.map((c) => c.name); expect(names).toContain("supabase"); expect(names).toContain("workos"); } finally { From 2d20eb379a26ed58393c9d5026b94cb6d750ffbf Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 27 Oct 2026 16:46:02 -0400 Subject: [PATCH 33/42] fix(completion): let fish complete files when the directive allows it The fish script registered `complete -c clerk -f` for every command and dropped the directive line, so fish never completed a file path, even where `__complete` returns DEFAULT: `clerk migrate sources ./` and `--source ./`. The candidates now go through a condition, as Cobra's fish script does: when none starts with what was typed and NO_FILE_COMP is clear, the condition fails and fish falls back to files, the same as bash. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/completion/index.test.ts | 11 +++++++ .../src/commands/completion/shells/fish.ts | 29 ++++++++++--------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/packages/cli-core/src/commands/completion/index.test.ts b/packages/cli-core/src/commands/completion/index.test.ts index 760799008..b8f2d3683 100644 --- a/packages/cli-core/src/commands/completion/index.test.ts +++ b/packages/cli-core/src/commands/completion/index.test.ts @@ -75,6 +75,17 @@ describe("shell script generators", () => { expect(generateFish(BINARY)).toContain(`complete -c ${BINARY}`); }); + test("fish reads the directive, and turns off files only when the candidates apply", () => { + const script = generateFish(BINARY); + // An unconditional -f would turn off file completion for every command. + expect(script).not.toContain(`complete -c ${BINARY} -f -a`); + expect(script).toContain( + `complete -c ${BINARY} -n '__${BINARY}_prepare_completions' -f -a '$__${BINARY}_comp_results'`, + ); + // Bit 4 is NO_FILE_COMP. + expect(script).toContain("math --scale 0 $directive / 4 % 2"); + }); + test("fish banner instructs users to mkdir ~/.config/fish/completions before writing", () => { const script = generateFish(BINARY); const mkdirIdx = script.indexOf("mkdir -p ~/.config/fish/completions"); diff --git a/packages/cli-core/src/commands/completion/shells/fish.ts b/packages/cli-core/src/commands/completion/shells/fish.ts index 55354fa87..2e0f83c55 100644 --- a/packages/cli-core/src/commands/completion/shells/fish.ts +++ b/packages/cli-core/src/commands/completion/shells/fish.ts @@ -11,7 +11,8 @@ export function generate(binaryName: string): string { # mkdir -p ~/.config/fish/completions # ${binaryName} completion fish > ~/.config/fish/completions/${binaryName}.fish -function __${binaryName}_complete +function __${binaryName}_prepare_completions + set -g __${binaryName}_comp_results set -l tokens (commandline -opc) set -l current (commandline -ct) @@ -19,23 +20,25 @@ function __${binaryName}_complete set -l args $tokens[2..] set -l output (${binaryName} __complete $args $current 2>/dev/null) - if test $status -ne 0 - return + if test $status -ne 0; or test (count $output) -eq 0 + return 0 end - # Last line is the directive — skip it - set -l count (count $output) - if test $count -le 1 - return - end + # Last line is the directive; the rest are the candidates + set -l directive (string replace ':' '' -- $output[-1]) + set -e output[-1] + set -g __${binaryName}_comp_results $output - # Output all lines except the last (directive) - for i in (seq 1 (math $count - 1)) - echo $output[$i] + # Nothing starts with what was typed, and the directive allows files (bit 4, + # NO_FILE_COMP, clear): false skips the entry below, so fish completes files. + set -l names (string replace -r '\\t.*' '' -- $output) + if not string match -q -- "$current*" $names; and test (math --scale 0 $directive / 4 % 2) -eq 0 + return 1 end + return 0 end -# Disable file completions by default, let __complete control it -complete -c ${binaryName} -f -a '(__${binaryName}_complete)' +# The candidates, with files off, whenever __complete has something to offer +complete -c ${binaryName} -n '__${binaryName}_prepare_completions' -f -a '$__${binaryName}_comp_results' `; } From 4cc8312c4226b1f25ed7579730694d195c6913b6 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 27 Oct 2026 17:16:42 -0400 Subject: [PATCH 34/42] fix(migrate): check Supabase providers under a source you wrote With `--source ./mine.ts` over a Supabase export, the raw rows were read only when the source key was `supabase`, so the disabled-provider rejects never ran: users whose only sign-in is a provider that's off in Clerk were created with no way in, and the offer to turn it on was skipped. The read now keys on the platform the file came from, as the Firebase hash parameters do; `hashSource` is renamed `platform` to match. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 1 + .../cli-core/src/commands/migrate/run.test.ts | 37 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 15 ++++---- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index c497a5f08..68d4dcbb4 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -298,6 +298,7 @@ link to the export run, and `run.json` records the source the envelope named as `envelopeSource`. A Firebase export keeps its hash parameters through it: the envelope's, the `--firebase-*` flags or the variables reach the source's `postTransform` as `context.firebaseHashConfig`, as they do the built-in's. +The checks still read a Supabase export's sign-in providers from its raw rows. An earlier import of the same export into the same instance, through the built-in or another source, holds the same users. While it isn't undone, a diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 9017011a0..ba6f88f97 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -772,6 +772,43 @@ describe("run", () => { expect(imported?.firebaseHash).toMatch(/^[0-9a-f]{64}$/); }); + // The provider rows come from the raw file, whatever maps it. + test("a source you wrote over a Supabase export still checks its providers", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + enterprise_sso: { enabled: true }, + }, + }); + const { record } = exportRun("supabase", [ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["discord"]}', + }, + ]); + const own = `./own-supabase-${Date.now()}.ts`; + fs.writeFileSync( + path.join(workDir, own), + `export default { + key: "mysupabase", + label: "My Supabase", + transformer: { id: "userId", email: "email" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + };`, + ); + + await run({ ...noSource, source: own, input: record.id, dryRun: true }); + + expect(captured.err).toContain("only signs in with Discord, which is not enabled in Clerk"); + }); + test("the printed command keeps a source you wrote in place of the envelope's", async () => { const { record } = exportRun("auth0", auth0Rows); const own = ownAuth0Source(); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 4e64605ae..aa2503cb0 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -935,18 +935,19 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi // The flags every suggested command needs to reach this runs folder and instance. const carry = carryFlags(options); - // A Firebase export through a source you wrote is still Firebase's: its - // digests need the project's parameters, from the envelope or the flags. - const hashSource = options.envelopeSource ?? source; + // The platform the file came from. An export through a source you wrote is + // still that platform's: Firebase digests need the project's parameters, and + // Supabase's raw rows hold its users' providers. + const platform = options.envelopeSource ?? source; // A flag is typed for this run, so it wins over the export file. A variable // is ambient, and may be left over from another project, so it only fills in // for an export file without the parameters: with one, a partial flag set is // refused rather than completed from the variables. - const fromEnvelope = hashSource === "firebase" ? envelope?.firebase : undefined; + const fromEnvelope = platform === "firebase" ? envelope?.firebase : undefined; const fromFlagsAtAll = FIREBASE_FLAGS.some(([key]) => options[key] !== undefined); const fromFlags = fromFlagsAtAll || !fromEnvelope - ? resolveFirebaseHashConfig(options, hashSource, fromEnvelope ? {} : process.env) + ? resolveFirebaseHashConfig(options, platform, fromEnvelope ? {} : process.env) : undefined; let firebaseHashConfig = fromFlags ?? fromEnvelope; let configFrom = fromFlags @@ -955,7 +956,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi : "the CLERK_FIREBASE_* variables" : "the export file"; // `-y` is "import without prompting": it gets the usage error naming the flags. - if (hashSource === "firebase" && !firebaseHashConfig && !options.yes && canPrompt(options)) { + if (platform === "firebase" && !firebaseHashConfig && !options.yes && canPrompt(options)) { firebaseHashConfig = await promptForFirebaseHashConfig(); configFrom = "the parameters entered"; // A suggested command runs with --yes, which never asks: it names the flags. @@ -1144,7 +1145,7 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi } let supabaseRows: Record[] | undefined; - if (source === "supabase") { + if (platform === "supabase") { try { supabaseRows = await readSupabaseRows(file); } catch (error) { From f7328921d2695a0ece0c51302a6dcfa21a44dec9 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 27 Oct 2026 17:18:30 -0400 Subject: [PATCH 35/42] fix(migrate): count an export's social providers under a source you wrote With a source you wrote over an Auth0 or Auth.js export, the checks took `socialProviders` from the custom entry, which rarely has one, so they counted no providers and never offered to turn them on. A source without its own now falls back to the export platform's. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 3 ++- .../cli-core/src/commands/migrate/run.test.ts | 17 +++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 3 ++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 68d4dcbb4..36c1fd20b 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -298,7 +298,8 @@ link to the export run, and `run.json` records the source the envelope named as `envelopeSource`. A Firebase export keeps its hash parameters through it: the envelope's, the `--firebase-*` flags or the variables reach the source's `postTransform` as `context.firebaseHashConfig`, as they do the built-in's. -The checks still read a Supabase export's sign-in providers from its raw rows. +The checks still read a Supabase export's sign-in providers from its raw rows, +and count an Auth0 or Auth.js export's the way the built-in does. An earlier import of the same export into the same instance, through the built-in or another source, holds the same users. While it isn't undone, a diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index ba6f88f97..ac77d2234 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -809,6 +809,23 @@ describe("run", () => { expect(captured.err).toContain("only signs in with Discord, which is not enabled in Clerk"); }); + // Without its own `socialProviders`, a source you wrote counts the export's. + test("a source you wrote over an Auth0 export still counts its social providers", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true } }, + enterprise_sso: { enabled: true }, + }, + }); + const { record } = exportRun("auth0", [ + { user_id: "google-oauth2|g1", email: "g@x.dev", email_verified: true }, + ]); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id, dryRun: true }); + + expect(captured.err).toContain("1 user signed in with Google, which is not enabled in Clerk"); + }); + test("the printed command keeps a source you wrote in place of the envelope's", async () => { const { record } = exportRun("auth0", auth0Rows); const own = ownAuth0Source(); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index aa2503cb0..5d188b2b4 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -1199,7 +1199,8 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi failures, unknownFields: loaded.unknownFields, ...(supabaseRows ? { supabaseRows } : {}), - socialProviders: getSource(source).socialProviders, + // A source you wrote without its own counts the export platform's. + socialProviders: getSource(source).socialProviders ?? getSource(platform).socialProviders, settings, existingUsers, maxAllowedUsers, From 3e7df678b4988f39e0fee0f7ffe26cb84a8d0fb3 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 28 Oct 2026 13:31:02 -0400 Subject: [PATCH 36/42] fix(migrate): skip the provider fallback for a platform this CLI doesn't know A source you wrote may map an export whose envelope names a platform this CLI doesn't know, such as a file from a newer version. Without its own `socialProviders`, the fallback to the platform's looked it up anyway, and the import stopped on a raw "Source not found for key". The fallback now applies only to a platform in the registry. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/run.test.ts | 10 ++++++++++ packages/cli-core/src/commands/migrate/run.ts | 7 +++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index ac77d2234..d7c4b49c2 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -826,6 +826,16 @@ describe("run", () => { expect(captured.err).toContain("1 user signed in with Google, which is not enabled in Clerk"); }); + // A file from a newer CLI names a platform this one doesn't know; yours maps it. + test("a source you wrote maps an export from a platform this CLI doesn't know", async () => { + const { record } = exportRun("okta", auth0Rows); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created).toHaveLength(1); + }); + test("the printed command keeps a source you wrote in place of the envelope's", async () => { const { record } = exportRun("auth0", auth0Rows); const own = ownAuth0Source(); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 5d188b2b4..ee6afbb75 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -1199,8 +1199,11 @@ async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promi failures, unknownFields: loaded.unknownFields, ...(supabaseRows ? { supabaseRows } : {}), - // A source you wrote without its own counts the export platform's. - socialProviders: getSource(source).socialProviders ?? getSource(platform).socialProviders, + // A source you wrote without its own counts the export platform's, + // when this CLI knows the platform: yours may map one it doesn't. + socialProviders: + getSource(source).socialProviders ?? + (sourceKeys().includes(platform) ? getSource(platform).socialProviders : undefined), settings, existingUsers, maxAllowedUsers, From eb28fa6ef81191970ffd7e1903307f578d57ca51 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 28 Oct 2026 13:31:08 -0400 Subject: [PATCH 37/42] docs(migrate): say the import lock follows the key when the instance is unconfirmed When `GET /v1/instance` fails, the lock's hash uses the key's stand-in for the instance, so two different keys for one instance take different locks. The README said a second import of the file into the same instance is refused whatever else differs. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 36c1fd20b..52c93aedf 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -186,7 +186,9 @@ checks and prompts: `clerk-migrate-import-.lock` in the temp directory (`os.tmpdir()`, usually `$TMPDIR` or `/tmp`). The hash covers the file's sha256 and the instance, not the source, so a second import of the same file into the same instance is refused with exit 2 even before either has written a run, -whichever source and `--runs-dir` each uses. It lives outside the runs folder because +whichever source and `--runs-dir` each uses. When Clerk doesn't confirm the +instance, the hash uses a stand-in for the key instead, so two different keys +for that instance don't share it. It lives outside the runs folder because nothing may be written there before consent. It holds the importing process's PID and is removed when the import finishes. One left by a Ctrl-C or a process that died is reclaimed by the next import. The refusal names the file, which is safe to From 220a924b9cf28aeae456af160f8f3b7ac0604a18 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 28 Oct 2026 14:07:07 -0400 Subject: [PATCH 38/42] docs(migrate): say two imports can both reclaim a stale import lock The README said a second import of the same file into the same instance is refused. Two imports that read the same stale lock can both remove it and both proceed, as the comment in `acquireLockFile` already says. Clerk still refuses the second create of any email, phone or username, so the cost is failed lines, not duplicate users. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 52c93aedf..eb6785cae 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -191,7 +191,9 @@ instance, the hash uses a stand-in for the key instead, so two different keys for that instance don't share it. It lives outside the runs folder because nothing may be written there before consent. It holds the importing process's PID and is removed when the import finishes. One left by a Ctrl-C or a process that died is -reclaimed by the next import. The refusal names the file, which is safe to +reclaimed by the next import. Two imports that start together over a stale lock +can both take it; Clerk still refuses the second create of any email, phone or +username, so the cost is failed lines, not duplicate users. The refusal names the file, which is safe to delete when no `clerk migrate import` is running. `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C From 56729839bb0d45f704423c1555d255a8d551435c Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 28 Oct 2026 14:07:16 -0400 Subject: [PATCH 39/42] docs(migrate): say when an import holds only its key's lock The README said that when Clerk doesn't confirm the instance, the lock's hash uses a stand-in for the key instead. An import takes a lock under its key as well as one under the instance whenever Clerk, `--app` or the linked profile names it; only one whose instance nothing names holds the key's lock alone, which a second key for that instance doesn't meet. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index eb6785cae..de3b267e1 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -186,9 +186,10 @@ checks and prompts: `clerk-migrate-import-.lock` in the temp directory (`os.tmpdir()`, usually `$TMPDIR` or `/tmp`). The hash covers the file's sha256 and the instance, not the source, so a second import of the same file into the same instance is refused with exit 2 even before either has written a run, -whichever source and `--runs-dir` each uses. When Clerk doesn't confirm the -instance, the hash uses a stand-in for the key instead, so two different keys -for that instance don't share it. It lives outside the runs folder because +whichever source and `--runs-dir` each uses. It also takes a lock under its +key, for when Clerk can't name the instance. When nothing names it (Clerk, +`--app` or the linked profile), that's the only lock it holds, so an import +with a second key for that instance doesn't meet it. It lives outside the runs folder because nothing may be written there before consent. It holds the importing process's PID and is removed when the import finishes. One left by a Ctrl-C or a process that died is reclaimed by the next import. Two imports that start together over a stale lock From f4b5fff5294160aa13bfd62d12e31b1f55a3f9fd Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 28 Oct 2026 18:51:03 -0400 Subject: [PATCH 40/42] docs(migrate): say the custom-source example's email is created verified The README's custom-source example maps `contact_email` to `email` without saying that `email` is created as a verified primary address. A reader copying it for a source with unconfirmed addresses would import them verified. The line now says so, and where an unconfirmed one goes. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index de3b267e1..33abd15da 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1224,6 +1224,7 @@ export default { description: "Exports from My Platform's admin console.", transformer: { account_ref: "userId", // required: becomes the Clerk user's external_id + // Created verified: map an address the source never confirmed to "unverifiedEmailAddresses". contact_email: "email", pw_bcrypt: "password", // full_name is left unmapped: postTransform splits it below. From d2b969c7424860722c185f52d284a105f4c2ba5e Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 28 Oct 2026 18:51:56 -0400 Subject: [PATCH 41/42] fix(migrate): let a preTransform parse a JSON file the CLI can't The README says `preTransform` runs before the file is read. For JSON input the loader parsed the file first, to spot an export envelope, so a file the CLI's parser rejects (comments, trailing commas) stopped with "is not valid JSON" before a source's own hook could parse it. When that parse fails and the source has a `preTransform`, the hook now gets the file; without one, the error stands. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/transform.test.ts | 25 +++++++++++++++++++ .../src/commands/migrate/lib/transform.ts | 8 +++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index e31775154..5c6af6394 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -392,6 +392,31 @@ describe("loadUsersFromFile", () => { } }); + // The hook runs before the file is read: JSON this CLI can't parse is its to parse. + test("a custom preTransform parses a JSON file the CLI can't", async () => { + registerCustomSource({ + ...clerkSource, + key: "json-with-comments", + preTransform: (filePath) => ({ + filePath, + data: JSON.parse(fs.readFileSync(filePath, "utf-8").replace(/^\s*\/\/.*$/gm, "")) as Record< + string, + unknown + >[], + }), + }); + try { + fs.writeFileSync( + path.join(workDir, "commented.json"), + '// exported 2026-10-28\n[{ "id": "c1", "primary_email_address": "c@x.dev" }]\n', + ); + const { users } = await loadUsersFromFile("commented.json", "json-with-comments"); + expect(users.map((user) => user.userId)).toEqual(["c1"]); + } finally { + __resetCustomSourcesForTesting(); + } + }); + test("rejects a JSON file that is not an array of users", async () => { fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index cea8a8221..07b8891b2 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -467,7 +467,13 @@ async function readUsersFromFile( // An export's envelope already holds the users in the source's own shape, // so there is nothing left for a pre-transform to unwrap. if (type === "application/json") { - const parsed = readJsonFile(filePath); + let parsed: unknown; + try { + parsed = readJsonFile(filePath); + } catch (error) { + // Not JSON this CLI reads, and so no envelope: a pre-transform may parse it. + if (!transformer.preTransform) throw error; + } if (isEnvelope(parsed)) return assertUserRows(parsed.users, file); if (!transformer.preTransform) return assertUserRows(parsed, file); } From dcf656f87211a0887aca57cadf031cd5d18b19f5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 28 Oct 2026 18:52:04 -0400 Subject: [PATCH 42/42] docs(migrate): say a preTransform's file keeps the import's format The loader picks its reader from the import file's type, not from the path a `preTransform` returns, so a file in another format, returned without `data`, misparses. The README now says the file must be in the same format as `fileType`. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 33abd15da..94f3b7def 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -1260,7 +1260,7 @@ Two optional hooks run around the `transformer` mapping: - `preTransform(filePath, fileType)` runs once, before the file is read, and may be `async`. `fileType` is `"application/json"`, `"text/csv"` or `""`. Return `{ filePath, data?, csvHeaders? }`: `filePath` is the file to read - (the same one, or one you wrote), `data` is the users when you parsed them + (the same one, or one you wrote in the same format as `fileType`), `data` is the users when you parsed them yourself and skips reading the file, and `csvHeaders` names the columns of a CSV with no header row. It isn't called for a file from `clerk migrate export`, which holds its users already.