diff --git a/.changeset/migrate-custom-sources.md b/.changeset/migrate-custom-sources.md new file mode 100644 index 000000000..fb9542f26 --- /dev/null +++ b/.changeset/migrate-custom-sources.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Add `clerk migrate sources`, and let `clerk migrate import --source` take the path to a source you wrote. `clerk migrate sources` lists the built-in sources with the passwords, MFA and metadata each brings across, and `clerk migrate sources ` shows where each field lands, how to export, and any caveats. A source you wrote is a `.ts` or `.js` file in your project, checked before use, so a platform with no built-in source can be imported without rebuilding the CLI. `clerk migrate` is still experimental and needs `CLERK_EXPERIMENTAL=migrate`. diff --git a/packages/cli-core/src/commands/completion/__complete.ts b/packages/cli-core/src/commands/completion/__complete.ts index d709f04f6..383177335 100644 --- a/packages/cli-core/src/commands/completion/__complete.ts +++ b/packages/cli-core/src/commands/completion/__complete.ts @@ -28,6 +28,16 @@ function noFileComp(completions: Completion[]): CompletionResult { return { completions, directive: DIRECTIVE.NO_FILE_COMP }; } +/** The known values that start with `partial`, from a registry entry. */ +function matching(known: Completion[], partial: string): CompletionResult { + const completions = known.filter((c) => c.name.startsWith(partial)); + // A migrate source can be a path to one you wrote, so the shell still + // completes files beside the built-in keys. + return known === SOURCE_COMPLETIONS + ? { completions, directive: DIRECTIVE.DEFAULT } + : noFileComp(completions); +} + const HTTP_METHOD_COMPLETIONS: Completion[] = [ { name: "GET", description: "Read resource" }, { name: "POST", description: "Create resource" }, @@ -36,7 +46,10 @@ const HTTP_METHOD_COMPLETIONS: Completion[] = [ { name: "DELETE", description: "Delete resource" }, ]; -/** The built-in migrate sources. Kept off `.choices()` so an unknown key gets the source error. */ +/** + * The built-in migrate sources. `--source` also takes a path to a source you + * wrote, so it cannot use `.choices()`. + */ const SOURCE_COMPLETIONS: Completion[] = sources.map((entry) => ({ name: entry.key, description: entry.label, @@ -86,6 +99,7 @@ const KNOWN_OPTION_VALUES: Record = { * to root, e.g. "open dashboard". */ const KNOWN_POSITIONAL_COMPLETIONS: Record = { + "migrate sources": SOURCE_COMPLETIONS, "open dashboard": KNOWN_DASHBOARD_PATHS.map((path) => ({ name: path, description: "Dashboard subpath", @@ -130,13 +144,15 @@ export function generateCompletions(root: CommandUnknownOpts, args: string[]): C } // Combine subcommands + argument choices + options + const argument = completeArguments(command, partial, positionalCount); const completions = [ ...completeSubcommands(command, partial).completions, - ...completeArguments(command, partial, positionalCount).completions, + ...argument.completions, ...completeOptions(command, "", usedOptions).completions, ]; - return noFileComp(completions); + // The argument decides file completion: on only for one that can be a path. + return { completions, directive: argument.directive }; } // ── Tree walking ───────────────────────────────────────────────────────────── @@ -254,10 +270,10 @@ function completeArguments( const path = commandPath(cmd); const registered = KNOWN_POSITIONAL_COMPLETIONS[path]; if (registered) { - return noFileComp(registered.filter((c) => c.name.startsWith(partial))); + return matching(registered, partial); } - return EMPTY_DEFAULT; + return EMPTY_NO_FILE; } function commandPath(cmd: CommandUnknownOpts): string { @@ -290,7 +306,7 @@ function completeOptionValue(opt: Option, flag: string, partial: string): Comple const candidates = resolveOptionValues(opt, flag); if (!candidates) return EMPTY_DEFAULT; - return noFileComp(candidates.filter((c) => c.name.startsWith(partial))); + return matching(candidates, partial); } function resolveOptionValues(opt: Option, flag: string): Completion[] | undefined { diff --git a/packages/cli-core/src/commands/completion/index.test.ts b/packages/cli-core/src/commands/completion/index.test.ts index 760799008..b8f2d3683 100644 --- a/packages/cli-core/src/commands/completion/index.test.ts +++ b/packages/cli-core/src/commands/completion/index.test.ts @@ -75,6 +75,17 @@ describe("shell script generators", () => { expect(generateFish(BINARY)).toContain(`complete -c ${BINARY}`); }); + test("fish reads the directive, and turns off files only when the candidates apply", () => { + const script = generateFish(BINARY); + // An unconditional -f would turn off file completion for every command. + expect(script).not.toContain(`complete -c ${BINARY} -f -a`); + expect(script).toContain( + `complete -c ${BINARY} -n '__${BINARY}_prepare_completions' -f -a '$__${BINARY}_comp_results'`, + ); + // Bit 4 is NO_FILE_COMP. + expect(script).toContain("math --scale 0 $directive / 4 % 2"); + }); + test("fish banner instructs users to mkdir ~/.config/fish/completions before writing", () => { const script = generateFish(BINARY); const mkdirIdx = script.indexOf("mkdir -p ~/.config/fish/completions"); diff --git a/packages/cli-core/src/commands/completion/shells/fish.ts b/packages/cli-core/src/commands/completion/shells/fish.ts index 55354fa87..2e0f83c55 100644 --- a/packages/cli-core/src/commands/completion/shells/fish.ts +++ b/packages/cli-core/src/commands/completion/shells/fish.ts @@ -11,7 +11,8 @@ export function generate(binaryName: string): string { # mkdir -p ~/.config/fish/completions # ${binaryName} completion fish > ~/.config/fish/completions/${binaryName}.fish -function __${binaryName}_complete +function __${binaryName}_prepare_completions + set -g __${binaryName}_comp_results set -l tokens (commandline -opc) set -l current (commandline -ct) @@ -19,23 +20,25 @@ function __${binaryName}_complete set -l args $tokens[2..] set -l output (${binaryName} __complete $args $current 2>/dev/null) - if test $status -ne 0 - return + if test $status -ne 0; or test (count $output) -eq 0 + return 0 end - # Last line is the directive — skip it - set -l count (count $output) - if test $count -le 1 - return - end + # Last line is the directive; the rest are the candidates + set -l directive (string replace ':' '' -- $output[-1]) + set -e output[-1] + set -g __${binaryName}_comp_results $output - # Output all lines except the last (directive) - for i in (seq 1 (math $count - 1)) - echo $output[$i] + # Nothing starts with what was typed, and the directive allows files (bit 4, + # NO_FILE_COMP, clear): false skips the entry below, so fish completes files. + set -l names (string replace -r '\\t.*' '' -- $output) + if not string match -q -- "$current*" $names; and test (math --scale 0 $directive / 4 % 2) -eq 0 + return 1 end + return 0 end -# Disable file completions by default, let __complete control it -complete -c ${binaryName} -f -a '(__${binaryName}_complete)' +# The candidates, with files off, whenever __complete has something to offer +complete -c ${binaryName} -n '__${binaryName}_prepare_completions' -f -a '$__${binaryName}_comp_results' `; } diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 5ce1d74c2..94f3b7def 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -19,6 +19,7 @@ clerk migrate export [source] [-o ] [--json] clerk migrate import [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] clerk migrate runs [run-id] [--json] clerk migrate undo [--dry-run] [--yes] [--include-signed-in] [--json] +clerk migrate sources [source] [--json] clerk migrate help ``` @@ -178,6 +179,24 @@ every run. Run folders are created owner-only (`0700`), and export files `0600`: they hold password hashes and user data. +#### The import lock + +Every import but a dry run also takes a lock outside the run store, before its +checks and prompts: `clerk-migrate-import-.lock` in the temp directory +(`os.tmpdir()`, usually `$TMPDIR` or `/tmp`). The hash covers the file's sha256 +and the instance, not the source, so a second import of the same file into the +same instance is refused with exit 2 even before either has written a run, +whichever source and `--runs-dir` each uses. It also takes a lock under its +key, for when Clerk can't name the instance. When nothing names it (Clerk, +`--app` or the linked profile), that's the only lock it holds, so an import +with a second key for that instance doesn't meet it. It lives outside the runs folder because +nothing may be written there before consent. It holds the importing process's +PID and is removed when the import finishes. One left by a Ctrl-C or a process that died is +reclaimed by the next import. Two imports that start together over a stale lock +can both take it; Clerk still refuses the second create of any email, phone or +username, so the cost is failed lines, not duplicate users. The refusal names the file, which is safe to +delete when no `clerk migrate import` is running. + `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. A line that cannot be written stops that user's create from going out. An export's file @@ -275,6 +294,23 @@ The file is an envelope around the users: parameters, when it could read them: the import then needs no `--firebase-*` flags. When it couldn't, pass them to the import, or enter them at its prompt. +`--source` with a different built-in name is refused, since it would map the +users with the wrong fields. `--source` with the path to a +[source you wrote](#custom-sources) replaces the envelope's: it gets the same +users, in the platform's own shape, so it can bring across what the built-in +mapping leaves out, such as a WorkOS `metadata.username`. The import keeps its +link to the export run, and `run.json` records the source the envelope named +as `envelopeSource`. A Firebase export keeps its hash parameters through it: +the envelope's, the `--firebase-*` flags or the variables reach the source's +`postTransform` as `context.firebaseHashConfig`, as they do the built-in's. +The checks still read a Supabase export's sign-in providers from its raw rows, +and count an Auth0 or Auth.js export's the way the built-in does. + +An earlier import of the same export into the same instance, through the +built-in or another source, holds the same users. While it isn't undone, a +re-run through your source refuses with exit 2 and names it, rather than +starting a new run beside it: pass `--new-run`, or undo it first. + `--json` prints the result on stdout instead — `{ target, run, output, users, skipped, coverage, next }` — and never prompts, so a missing credential exits 2 naming the flag to pass. `next`, and the commands printed after a human run, @@ -682,7 +718,7 @@ clerk migrate import # a human is asked | Flag | Description | | --------------------------------------- | ------------------------------------------------------------------------------------------------- | | `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | -| `--source ` | Where the file came from: one of the [sources](#sources) | +| `--source ` | Where the file came from: a [source](#sources), or one you wrote | | `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | | `--allow-partial` | Import the users that pass, and record the rest as skipped | | `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | @@ -753,8 +789,8 @@ which skip the checks too. #### Re-running Running the same import again continues where it left off. The match is the -file's sha256, the source, and the instance ID; the latest matching import run -decides what happens: +file's sha256, the source (and a custom source's content hash), and the +instance ID; the latest matching import run decides what happens: | Latest match | Re-running does | | ----------------------------------------- | ---------------------------------------------------------------------- | @@ -1103,6 +1139,29 @@ With `writeFailed` or `stopped`, `run` is the undo run as it stands on disk, once one was written: its counts come from its lines, and `state` says how it stopped, as in `clerk migrate runs`. +### `clerk migrate sources` + +```sh +clerk migrate sources # every source, with what it carries +clerk migrate sources betterauth # one source in full +clerk migrate sources ./my-source.ts # a source you wrote +clerk migrate sources --json +``` + +| Flag | Description | +| ---------- | ---------------------------------------------------------------- | +| `[source]` | A built-in key, or the path to a source you wrote, to show fully | +| `--json` | The same data, on stdout | + +`sources` alone prints the table under [Sources](#sources). `sources ` +shows one source in full: its export command (when `clerk migrate export` has +one for it), what it carries with a note for each, where each field lands +(`encrypted_password → password`), its fixed defaults, and any caveats. An +unknown key exits 2 and lists the valid ones. There is no intro/outro gutter. +`sources` alone, or with a built-in key, reads a static registry and runs +nothing. With a path, it loads and runs that file, as `--source` does, to check +it and show what it maps. + ### `clerk migrate help` `clerk migrate help` and `clerk migrate --help` print the help for the @@ -1125,16 +1184,125 @@ brings across. Adding a platform is one file in `sources/` plus one line in | `supabase` | Supabase `auth.users` export | yes | no | partial | | `workos` | WorkOS User Management API | no | no | yes | -An unknown `--source` exits 2 with the list of valid keys. - There is no column for social sign-ins, because no source copies them and none -needs to. Enable the same providers in Clerk, and a user who signs in with one -is linked to their imported account by verified email. See +needs to. Every source shows the same note instead: enable the same providers in +Clerk, and a user who signs in with one is linked to their imported account by +verified email. See [account linking](https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking). The import's checks count the providers in a Supabase, Auth0 or Auth.js file and offer to enable any that are off. Auth0's come from each `user_id`'s prefix (`github|…`), so a user who linked a second provider counts under the first. +### `--source` + +`clerk migrate import` takes `--source `: + +- A value starting with `./`, `../` or `/`, or ending in `.ts`, `.js` or + `.mjs`, is loaded as a [custom source](#custom-sources). +- Anything else must be a built-in key. An unknown key exits 2 with the list of + valid keys. + +A file `clerk migrate export` wrote names its own source, so it needs none. + +### Custom sources + +Migrating from a platform with no built-in, without recompiling the CLI: + +```sh +clerk migrate import users.json --source ./my-platform.ts +``` + +The file lives in **your** project, not in the CLI, and is imported at runtime. +It exports the same shape the built-ins use, as plain data. It can't import +from the CLI, since there is nothing in a compiled binary to import from, but +it can import your project's own files: + +```ts +export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform's admin console.", + transformer: { + account_ref: "userId", // required: becomes the Clerk user's external_id + // Created verified: map an address the source never confirmed to "unverifiedEmailAddresses". + contact_email: "email", + pw_bcrypt: "password", + // full_name is left unmapped: postTransform splits it below. + }, + carries: { + passwords: { level: "yes", note: "bcrypt hashes from the pw_bcrypt column." }, + mfa: { level: "no", note: "Not exported." }, + metadata: { level: "no", note: "Not exported." }, + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { + // Unmapped columns arrive under their own names. + const [first, ...rest] = String(user.full_name ?? "") + .trim() + .split(/\s+/); + if (first) user.firstName = first; + if (rest.length) user.lastName = rest.join(" "); + delete user.full_name; + }, +}; +``` + +Two optional hooks run around the `transformer` mapping: + +- `postTransform(user, context)` runs once per user, after the mapping. `user` + holds the mapped fields, plus every column the `transformer` doesn't name, + still under its own name. Change `user` in place; the return value is + ignored. It must be synchronous: an `async` hook, or one that returns a + promise, stops the import, since its changes would arrive after the user + was checked. One that throws stops the import with the source's key, the + row number and the error. `context` holds per-run values a built-in needs, + such as Firebase's hash parameters; a custom source can ignore it. +- `preTransform(filePath, fileType)` runs once, before the file is read, and + may be `async`. `fileType` is `"application/json"`, `"text/csv"` or `""`. + Return `{ filePath, data?, csvHeaders? }`: `filePath` is the file to read + (the same one, or one you wrote in the same format as `fileType`), `data` is the users when you parsed them + yourself and skips reading the file, and `csvHeaders` names the columns of + a CSV with no header row. It isn't called for a file from + `clerk migrate export`, which holds its users already. + +A `transformer` target that isn't a Clerk field, like `full_name: "name"`, is +only a rename: Clerk won't store it unless `postTransform` moves it. + +TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, +`satisfies` and `as const` all work in a file the compiled binary imports. +Plain `.js` works too. + +An import run records a custom source's key and a hash of the file, so an +edited source counts as a different source. A re-run with an edited source, +or with another source altogether, won't continue the earlier run of the file, +and won't start a new one beside it unasked either: while that run isn't +undone, it refuses with exit 2 and names it, since the users it created would +come back as conflicts. Pass `--new-run` to start +a new run anyway, or undo the earlier one first. The hash covers that file +only: a local helper it imports is not part of it, so after editing one, or +when running a copy of the file beside different helpers, pass `--new-run` +rather than continue a run made with the old ones. + +#### Validation + +The file is code the CLI executes, so its shape is checked before use and +rejected with the specific problem rather than crashing mid-pipeline: + +| Problem | Message | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Path does not exist | `No source file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | +| `preTransform` or `postTransform` is not a function | `` `preTransform` must be a function when present `` (likewise `postTransform`) | +| `caveats` is not a list of strings | `` `caveats` must be an array of strings when present `` | + +The `userId` check is the load-bearing one: without it the import would run to +completion and create every user with no `external_id`, which is what makes a +migration re-runnable. + ### Metadata Metadata a user can edit on the source platform — Auth0's `user_metadata`, @@ -1167,6 +1335,7 @@ Better Auth's scrypt uses the hex salt string as the salt and a 64-byte key, which is what `scrypt_werkzeug` verifies once N, r and p are written inline. It also normalizes a password to NFKC before hashing, and Clerk does not, so a password whose NFKC form differs will not verify and that user resets it. +`clerk migrate sources betterauth` lists that caveat. A password Clerk cannot verify is **dropped, not rejected**: dropping it doesn't reject the user, who imports without it and can sign in another way or reset @@ -1233,7 +1402,8 @@ An export with no password hashes needs no parameters at all. What a source maps _onto_. Every user is validated against this schema before any request is made, so a field a source produces that is not listed here is dropped — Zod strips unknown keys — and never reaches Clerk. The checks -warn about each one (`Clerk won't store: …`). +warn about each one (`Clerk won't store: …`). Writing a custom source means +targeting these names exactly. The schema lives in `validator.ts`; adding a platform means adding a source, not editing it. diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index f798e640d..d3e5857c8 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -32,12 +32,13 @@ describe("registerMigrate", () => { expect(migrate?.description()).toContain("Migrate users"); }); - test("registers import, export, undo and runs", () => { + test("registers import, export, undo, runs and sources", () => { expect(findCommand(["migrate"])?.commands.map((cmd) => cmd.name())).toEqual([ "import", "export", "undo", "runs", + "sources", ]); }); @@ -158,6 +159,12 @@ describe("registerMigrate", () => { expect(runsCommand?.options.map((option) => option.long)).toEqual(["--json", "--runs-dir"]); }); + test("registers sources with an optional source and --json", () => { + const sourcesCommand = findCommand(["migrate", "sources"]); + expect(sourcesCommand?.registeredArguments[0]?.required).toBe(false); + expect(sourcesCommand?.options.map((option) => option.long)).toEqual(["--json"]); + }); + test("registers undo with a required run ID and its flags", () => { const undoCommand = findCommand(["migrate", "undo"]); expect(undoCommand?.registeredArguments[0]?.required).toBe(true); @@ -173,9 +180,9 @@ describe("registerMigrate", () => { ]); }); - // No `.choices()`: an unknown key reaches resolveSource, whose error lists - // the valid ones. Completion offers them through `KNOWN_OPTION_VALUES`. - test("--source accepts any value", () => { + // It also takes a path, so it cannot use `.choices()`: completion offers the + // built-in keys through `KNOWN_OPTION_VALUES` instead. + test("--source accepts any value, so a path to a source you wrote gets through", () => { const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === "--source"); expect(option?.argChoices).toBeUndefined(); }); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index e0c1728c1..ffc826d7f 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -8,6 +8,8 @@ import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; import { run } from "./run.ts"; import { runs } from "./runs.ts"; import { undo } from "./undo.ts"; +import { CUSTOM_SOURCES_URL } from "./sources/load-custom.ts"; +import { list as sources } from "./sources/list.ts"; export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = process.env): void { if (!isExperimentEnabled("migrate", env)) { @@ -51,6 +53,7 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce command: "clerk migrate undo 20260929-141502-a1b2", description: "Delete the users an import created", }, + { command: "clerk migrate sources", description: "What each source brings across" }, ]); // `-y` is read several layers down — by the credential-retry loop and the @@ -81,8 +84,8 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .description("Import users from an exported JSON or CSV file") .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") .option( - "--source ", - "Where the file came from. Not needed for a file from `clerk migrate export`", + "--source ", + `Where the file came from: a built-in source, or a source you wrote (${CUSTOM_SOURCES_URL}). Not needed for a file from \`clerk migrate export\`, though a source you wrote replaces its own`, ) .option("--dry-run", "Check the file against the instance, report, and write nothing") .option("--allow-partial", "Import the users that pass the checks, and skip the rest") @@ -129,6 +132,10 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce command: "clerk migrate import users.json --source clerk --allow-partial --yes", description: "Import a Clerk Dashboard export, skipping users that would be rejected", }, + { + command: "clerk migrate import users.json --source ./my-source.ts --yes", + description: "Import with a source you wrote", + }, ]) .action(async (input, _opts, cmd) => run({ @@ -184,4 +191,23 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .action(async (runId, _opts, cmd) => runs(runId, cmd.optsWithGlobals() as Parameters[1]), ); + + // A compiled binary has no source tree to grep, so the available mappings + // need a command rather than only appearing in the interactive picker. + migrateCommand + .command("sources") + .description("List the sources an import can read, or show one in full") + .argument("[source]", "A built-in source, or the path to a source you wrote") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate sources", description: "What each source brings across" }, + { + command: "clerk migrate sources betterauth", + description: "Where each field lands, how to export, and caveats", + }, + { command: "clerk migrate sources ./my-source.ts", description: "Check a source you wrote" }, + ]) + .action(async (source, _opts, cmd) => + sources(source, cmd.optsWithGlobals() as Parameters[1]), + ); } diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts index 974b28740..e6636720b 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -19,6 +19,8 @@ import { latestUserLines, listRuns, lockFile, + importLockFile, + lockImport, newRunId, patchRun, readRun, @@ -204,6 +206,60 @@ describe("locks and interruptions", () => { expect(() => continueRun(runsDir, run.record)).not.toThrow(); }); + // A predictable name in a shared /tmp could be a planted symlink, and a + // crashed run's leftover under the same PID would read as a held lock. + test("the lock's temporary file has a random name and is created exclusively", () => { + const identity = { sha256: "b".repeat(64), instanceId: "ins_tmp" }; + const write = spyOn(fs, "writeFileSync"); + try { + const release = lockImport(identity); + const tmp = write.mock.calls.find(([target]) => String(target).endsWith(".tmp")); + expect(tmp?.[2]).toMatchObject({ flag: "wx" }); + expect(String(tmp?.[0])).toMatch(new RegExp(`\\.${process.pid}\\.[0-9a-f]{16}\\.tmp$`)); + release(); + } finally { + write.mockRestore(); + } + }); + + // A lock created empty and written after reads as stale to a second + // process in between, which would remove it and import beside the first. + test("a lock is linked into place with its PID already written", () => { + const identity = { sha256: "a".repeat(64), instanceId: "ins_lock" }; + const write = spyOn(fs, "writeFileSync"); + const link = spyOn(fs, "linkSync"); + try { + const release = lockImport(identity); + const lockPaths = write.mock.calls.map(([target]) => String(target)); + // Where hard links are refused, the fallback below writes the lock itself. + if (link.mock.results.every((result) => result.type === "return")) { + expect(lockPaths.every((target) => target.endsWith(".tmp"))).toBe(true); + } + release(); + } finally { + write.mockRestore(); + link.mockRestore(); + } + }); + + test("a filesystem without hard links gets the lock written exclusively", () => { + const identity = { sha256: "b".repeat(64), instanceId: "ins_nolink" }; + const link = spyOn(fs, "linkSync").mockImplementation(() => { + throw Object.assign(new Error("not supported"), { code: "EPERM" }); + }); + const write = spyOn(fs, "writeFileSync"); + try { + const release = lockImport(identity); + const lock = write.mock.calls.find(([target]) => String(target).endsWith(".lock")); + expect(lock?.[2]).toMatchObject({ flag: "wx" }); + expect(fs.readFileSync(importLockFile(identity), "utf-8")).toBe(String(process.pid)); + release(); + } finally { + write.mockRestore(); + link.mockRestore(); + } + }); + test("reads as running while another live process holds the lock", () => { const run = startRun(runsDir, init); fs.writeFileSync(path.join(run.dir, "lock"), "1"); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts index 1423be233..e327817ad 100644 --- a/packages/cli-core/src/commands/migrate/lib/run-store.ts +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -20,6 +20,7 @@ import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { resolveProfile } from "../../../lib/config.ts"; import { throwUsageError } from "../../../lib/errors.ts"; @@ -89,6 +90,8 @@ export type RunRecord = { * continue for those. */ flags?: ImportFlags; + /** The source the export file named, when a custom source replaced it. */ + envelopeSource?: string; /** * A Firebase import's hash parameters, as a scrypt hash of the four (never the * parameters themselves): every digest it builds carries them, so a continue @@ -221,9 +224,14 @@ function isPidAlive(pid: number): boolean { * the same PID every run, so a killed run's lock would otherwise read as live. */ export function liveLockPid(runsDir: string, id: string): number | undefined { + return livePidIn(lockFile(runsDir, id)); +} + +/** The live PID a lock file holds, by the same rules as {@link liveLockPid}. */ +function livePidIn(file: string): number | undefined { let raw: string; try { - raw = fs.readFileSync(lockFile(runsDir, id), "utf-8"); + raw = fs.readFileSync(file, "utf-8"); } catch { return undefined; } @@ -246,31 +254,59 @@ const isMissing = (error: unknown) => (error as NodeJS.ErrnoException).code === * both read "free" and both write; a stale lock is removed and taken once. */ function acquireLock(runsDir: string, id: string): void { - const file = lockFile(runsDir, id); - const refuse = (holder: number | undefined): never => + acquireLockFile(lockFile(runsDir, id), (holder, file) => throwUsageError( `Run ${id} is in use by another process${holder ? ` (PID ${holder})` : ""}. Wait for it to finish, then try again. ` + `If that process is not a migrate run, delete ${file}.`, - ); - const take = () => fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); + ), + ); +} + +/** Takes the lock at `file`, or calls `refuse` with its live holder. */ +function acquireLockFile( + file: string, + refuse: (holder: number | undefined, file: string) => never, +): void { + // Written whole, then linked into place: the link fails on an existing lock + // as `wx` does, but the lock never exists without its PID, which a second + // process would read as stale and remove. + const take = () => { + // Random and exclusive: a name no one can plant a link at, and a crashed + // run's leftover (same PID, in a container) never reads as a held lock. + const whole = `${file}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`; + fs.writeFileSync(whole, String(process.pid), { flag: "wx", mode: 0o600 }); + try { + fs.linkSync(whole, file); + } catch (error) { + if (isExists(error)) throw error; + // A filesystem without hard links. + fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); + } finally { + fs.rmSync(whole, { force: true }); + } + }; try { take(); - log.debug(`migrate: locked run ${id} (${file})`); + log.debug(`migrate: took the lock ${file}`); return; } catch (error) { if (!isExists(error)) throw error; } - const holder = liveLockPid(runsDir, id); - if (holder !== undefined) refuse(holder); - log.debug(`migrate: removing a stale lock on run ${id} (${file})`); + // ponytail: two processes that both read the same stale lock can both + // remove it, and the second can remove the first's new one. Clerk still + // refuses the second create of any email, phone or username, so the cost + // is failed lines, not duplicate users. A lock on the reclaim would close it. + const holder = livePidIn(file); + if (holder !== undefined) refuse(holder, file); + log.debug(`migrate: removing a stale lock (${file})`); fs.rmSync(file, { force: true }); try { take(); - log.debug(`migrate: locked run ${id} (${file})`); + log.debug(`migrate: took the lock ${file}`); } catch (error) { // Another process took the stale lock between the remove and the write. - if (isExists(error)) refuse(liveLockPid(runsDir, id)); + if (isExists(error)) refuse(livePidIn(file), file); throw error; } } @@ -280,6 +316,45 @@ function releaseLock(runsDir: string, id: string): void { fs.rmSync(lockFile(runsDir, id), { force: true }); } +/** + * What makes two imports the same job: the file and the instance, by its ID + * or its key's stand-in ID. Not the source: whichever source maps the file, + * its users are the same people. + */ +export type ImportIdentity = { sha256: string; instanceId: string }; + +/** + * The lock one import of a file into an instance holds. In the + * temp directory, as it is taken before consent, when nothing may be written + * to the runs folder yet. Not keyed by the runs folder: two imports with + * different `--runs-dir` values would still send the same users. + */ +export function importLockFile(identity: ImportIdentity): string { + const key = createHash("sha256") + .update(JSON.stringify([identity.sha256, identity.instanceId])) + .digest("hex") + .slice(0, 32); + return path.join(os.tmpdir(), `clerk-migrate-import-${key}.lock`); +} + +/** + * Takes the lock for one import of this file into this instance, before + * its checks, so no second process can pass them and start a run beside it. + * + * @returns Releases the lock. + * @throws UsageError when a live process holds it. + */ +export function lockImport(identity: ImportIdentity): () => void { + const file = importLockFile(identity); + acquireLockFile(file, (holder) => + throwUsageError( + `Another process${holder ? ` (PID ${holder})` : ""} is importing this file into this instance right now. ` + + `Wait for it to finish. If that process is not a migrate run, delete ${file}.`, + ), + ); + return () => fs.rmSync(file, { force: true }); +} + // --- Reading --------------------------------------------------------------- /** One run's record, or `undefined` when the folder holds no readable run. */ diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 561ad655e..5c6af6394 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -2,9 +2,10 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { CliError } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, EXIT_CODE } from "../../../lib/errors.ts"; import clerkSource from "../sources/clerk.ts"; import type { SourceEntry } from "../types.ts"; +import { __resetCustomSourcesForTesting, registerCustomSource } from "../sources/registry.ts"; import { consolidateClerkIdentifiers, flattenObjectSelectively, @@ -203,6 +204,78 @@ describe("transformUsers", () => { }); }); + test("rejects an async postTransform rather than lose its changes", () => { + registerCustomSource({ + ...clerkSource, + key: "async-post", + // A source file is untyped at runtime, so nothing stops it being async. + postTransform: (async (user: Record) => { + await Promise.resolve(); + user.firstName = "late"; + }) as unknown as SourceEntry["postTransform"], + }); + try { + expect(() => transformUsers([{ id: "u1" }], "async-post")).toThrow( + /"async-post" source's postTransform returned a promise/, + ); + } finally { + __resetCustomSourcesForTesting(); + } + }); + + test("rejects a postTransform that returns a non-native thenable", () => { + registerCustomSource({ + ...clerkSource, + key: "thenable-post", + postTransform: (() => ({ then: () => undefined })) as unknown as SourceEntry["postTransform"], + }); + try { + expect(() => transformUsers([{ id: "u1" }], "thenable-post")).toThrow( + /"thenable-post" source's postTransform returned a promise/, + ); + } finally { + __resetCustomSourcesForTesting(); + } + }); + + test("names the source and the row when postTransform throws", () => { + registerCustomSource({ + ...clerkSource, + key: "throwing-post", + postTransform: (user) => { + if (user.userId === "u2") throw new Error("boom"); + }, + }); + try { + expect(() => transformUsers([{ id: "u1" }, { id: "u2" }], "throwing-post")).toThrow( + expect.objectContaining({ + message: 'The "throwing-post" source\'s postTransform failed on row 2: boom', + code: ERROR_CODE.USAGE_ERROR, + exitCode: EXIT_CODE.USAGE, + }), + ); + } finally { + __resetCustomSourcesForTesting(); + } + }); + + test("passes a postTransform's own CliError through with its code", () => { + const hashed = { + localId: "f1", + email: "a@example.com", + passwordHash: "SGFzaA==", + salt: "U2FsdA==", + }; + expect(() => transformUsers([hashed], "firebase")).toThrow( + expect.objectContaining({ + message: expect.stringMatching(/^This export contains Firebase password hashes/), + code: ERROR_CODE.USAGE_ERROR, + exitCode: EXIT_CODE.USAGE, + docsUrl: expect.stringContaining("/migrating/firebase"), + }), + ); + }); + test("skips validation when asked, so analysis passes see every row", () => { const { transformedData, validationFailed } = transformUsers([{ id: "u1" }], "clerk", { validate: false, @@ -298,6 +371,52 @@ describe("loadUsersFromFile", () => { } }); + test("a custom preTransform's rows win over a CSV file", async () => { + registerCustomSource({ + ...clerkSource, + key: "rows-from-pretransform", + preTransform: (filePath) => ({ + filePath, + data: [{ id: "from-pretransform", primary_email_address: "p@x.dev" }], + }), + }); + try { + fs.writeFileSync( + path.join(workDir, "ignored.csv"), + "id,primary_email_address\nfile,f@x.dev\n", + ); + const { users } = await loadUsersFromFile("ignored.csv", "rows-from-pretransform"); + expect(users.map((user) => user.userId)).toEqual(["from-pretransform"]); + } finally { + __resetCustomSourcesForTesting(); + } + }); + + // The hook runs before the file is read: JSON this CLI can't parse is its to parse. + test("a custom preTransform parses a JSON file the CLI can't", async () => { + registerCustomSource({ + ...clerkSource, + key: "json-with-comments", + preTransform: (filePath) => ({ + filePath, + data: JSON.parse(fs.readFileSync(filePath, "utf-8").replace(/^\s*\/\/.*$/gm, "")) as Record< + string, + unknown + >[], + }), + }); + try { + fs.writeFileSync( + path.join(workDir, "commented.json"), + '// exported 2026-10-28\n[{ "id": "c1", "primary_email_address": "c@x.dev" }]\n', + ); + const { users } = await loadUsersFromFile("commented.json", "json-with-comments"); + expect(users.map((user) => user.userId)).toEqual(["c1"]); + } finally { + __resetCustomSourcesForTesting(); + } + }); + test("rejects a JSON file that is not an array of users", async () => { fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 5daae2e23..07b8891b2 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -10,7 +10,8 @@ import fs from "node:fs"; import path from "node:path"; import csvParser from "csv-parser"; -import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { CUSTOM_SOURCES_URL } from "../sources/load-custom.ts"; import { getSource } from "../sources/registry.ts"; import { normalizeBooleanField, parseDate } from "../sources/shared.ts"; import { type TransformContext, type SourceEntry, type User } from "../types.ts"; @@ -294,7 +295,7 @@ export function consolidateClerkIdentifiers(user: Record): void // --- Validation ------------------------------------------------------------ /** Every field the import schema declares; anything else is stripped. */ -const SCHEMA_FIELDS: ReadonlySet = new Set(Object.keys(userSchema.shape)); +export const SCHEMA_FIELDS: ReadonlySet = new Set(Object.keys(userSchema.shape)); /** * Validates prepared users, dropping each failure from the run and returning @@ -374,13 +375,35 @@ export function transformUsers( const context = options.context ?? {}; const transformed: Record[] = []; - for (const user of users) { + for (const [i, user] of users.entries()) { const mapped = transformKeys(user, transformer); // The source's own cleanup first: the Clerk Dashboard's formula-safety TAB // has to come off before identifiers are compared, or "\t+1555…" and // "+1555…" read as two phones. - transformer.postTransform?.(mapped, context); + let result: unknown; + try { + result = transformer.postTransform?.(mapped, context); + } catch (error) { + // A refusal the source meant, like Firebase's missing hash parameters, + // already says what to do and carries its own code. + if (error instanceof CliError) throw error; + // 1-based, like the validation failures: the second user is row 2. + throwUsageError( + `The "${key}" source's postTransform failed on row ${i + 1}: ${error instanceof Error ? error.message : String(error)}`, + CUSTOM_SOURCES_URL, + ); + } + // An async hook's changes after its first await land once this user is + // already normalized and validated, so they would be lost without a word. + // Any thenable, not only a native Promise: a hook may use another library's. + if (typeof (result as PromiseLike | undefined)?.then === "function") { + Promise.resolve(result).catch(() => undefined); + throwUsageError( + `The "${key}" source's postTransform returned a promise. It must be synchronous: changes made after an await never reach the import.`, + CUSTOM_SOURCES_URL, + ); + } if (key === "clerk") { consolidateClerkIdentifiers(mapped); } @@ -444,7 +467,13 @@ async function readUsersFromFile( // An export's envelope already holds the users in the source's own shape, // so there is nothing left for a pre-transform to unwrap. if (type === "application/json") { - const parsed = readJsonFile(filePath); + let parsed: unknown; + try { + parsed = readJsonFile(filePath); + } catch (error) { + // Not JSON this CLI reads, and so no envelope: a pre-transform may parse it. + if (!transformer.preTransform) throw error; + } if (isEnvelope(parsed)) return assertUserRows(parsed.users, file); if (!transformer.preTransform) return assertUserRows(parsed, file); } diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 395360b4e..d7c4b49c2 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -28,8 +28,11 @@ import { readRun, sha256File, startRun, + importLockFile, } from "./lib/run-store.ts"; +import { keyInstanceId } from "./lib/target.ts"; import { explainErrors, run, validateRunOptions } from "./run.ts"; +import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; // After run.ts: imported first, signals.ts loads version.ts ahead of its Bun // macro here, and the file fails to load. import { _resetInterruptState, abortInFlight, beginInterrupt } from "../../lib/signals.ts"; @@ -627,6 +630,225 @@ describe("run", () => { expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); }); + test("says how to map the users differently when a built-in source contradicts it", async () => { + const { record } = exportRun("clerk", export2); + + await expect(run({ ...noSource, source: "auth0", input: record.id })).rejects.toThrow( + /pass the path to a source you wrote/, + ); + }); + + /** A source of your own, for an Auth0 export, that also brings `nickname` across. */ + const ownAuth0Source = () => { + const file = `./own-auth0-${Date.now()}-${Math.random().toString(36).slice(2)}.ts`; + fs.writeFileSync( + path.join(workDir, file), + `export default { + key: "myauth0", + label: "My Auth0", + transformer: { user_id: "userId", email: "email", nickname: "firstName" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + };`, + ); + return file; + }; + const auth0Rows = [{ user_id: "auth0|a1", email: "a@x.dev", nickname: "Ada" }]; + + // The built-in mapping leaves `nickname` out; the customer's source brings it. + test("a source you wrote replaces the envelope's, reading its users", async () => { + const { record } = exportRun("auth0", auth0Rows); + const own = ownAuth0Source(); + + await run({ ...noSource, source: own, input: record.id }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created).toHaveLength(1); + expect(created[0]?.body).toMatchObject({ external_id: "auth0|a1", first_name: "Ada" }); + expect(captured.err).toContain("in place of the built-in"); + }); + + test("an import through a source you wrote keeps its export run, and records what it replaced", async () => { + const { record } = exportRun("auth0", auth0Rows); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id }); + + const imported = listRuns(runsDir()).find((r) => r.kind === "import"); + expect(imported).toMatchObject({ + source: "myauth0", + fromExport: record.id, + envelopeSource: "auth0", + }); + }); + + // A re-run without it would map the rest with the built-in source. + // Import with the built-in, notice a field is missing, write a source, re-run: + // the first run's users are the same people, whichever source maps them. + test("refuses a source you wrote while the built-in's run of the same export holds its users", async () => { + const { record } = exportRun("auth0", auth0Rows); + await run({ ...noSource, input: record.id }); + const [first] = listRuns(runsDir()).filter((r) => r.kind === "import"); + requests = []; + + const error = (await run({ ...noSource, source: ownAuth0Source(), input: record.id }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain( + `Run ${first!.id} imported this file into this instance with the \`auth0\` source`, + ); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id, newRun: true }); + expect(listRuns(runsDir()).filter((r) => r.kind === "import")).toHaveLength(2); + }); + + // One lock for the file and instance, whichever source each import uses. + test("a source you wrote takes the same import lock as the built-in", async () => { + const { record, file } = exportRun("auth0", auth0Rows); + const lock = importLockFile({ + sha256: sha256File(file), + instanceId: keyInstanceId(baseOptions.secretKey), + }); + fs.writeFileSync(lock, "1"); + try { + await expect( + run({ ...noSource, source: ownAuth0Source(), input: record.id }), + ).rejects.toThrow(/importing this file into this instance right now/); + } finally { + fs.rmSync(lock, { force: true }); + } + }); + + // A Firebase export's digests need the project's parameters, whatever maps it. + test("a source you wrote over a Firebase export gets its hash parameters", async () => { + const { record } = exportRun( + "firebase", + [{ localId: "f1", email: "f@x.dev", passwordHash: "HASH", passwordSalt: "SALT" }], + { + firebase: { + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }, + }, + ); + const own = `./own-firebase-${Date.now()}.ts`; + fs.writeFileSync( + path.join(workDir, own), + `export default { + key: "myfirebase", + label: "My Firebase", + transformer: { localId: "userId", email: "email", passwordHash: "passwordHash", passwordSalt: "salt" }, + carries: { + passwords: { level: "yes", note: "scrypt." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + defaults: { passwordHasher: "scrypt_firebase" }, + postTransform: (user, context) => { + const c = context.firebaseHashConfig; + if (!c) throw new Error("no hash parameters"); + user.password = [user.passwordHash, user.salt, c.base64_signer_key, c.base64_salt_separator, c.rounds, c.mem_cost].join("$"); + delete user.passwordHash; + delete user.salt; + }, + };`, + ); + + await run({ ...noSource, source: own, input: record.id }); + + const created = requests.find((r) => r.url.endsWith("/v1/users")); + expect(created?.body).toMatchObject({ + password_hasher: "scrypt_firebase", + password_digest: "HASH$SALT$SIGNER$Bw==$8$14", + }); + const imported = listRuns(runsDir()).find((r) => r.kind === "import"); + expect(imported?.firebaseHash).toMatch(/^[0-9a-f]{64}$/); + }); + + // The provider rows come from the raw file, whatever maps it. + test("a source you wrote over a Supabase export still checks its providers", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + enterprise_sso: { enabled: true }, + }, + }); + const { record } = exportRun("supabase", [ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["discord"]}', + }, + ]); + const own = `./own-supabase-${Date.now()}.ts`; + fs.writeFileSync( + path.join(workDir, own), + `export default { + key: "mysupabase", + label: "My Supabase", + transformer: { id: "userId", email: "email" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + };`, + ); + + await run({ ...noSource, source: own, input: record.id, dryRun: true }); + + expect(captured.err).toContain("only signs in with Discord, which is not enabled in Clerk"); + }); + + // Without its own `socialProviders`, a source you wrote counts the export's. + test("a source you wrote over an Auth0 export still counts its social providers", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true } }, + enterprise_sso: { enabled: true }, + }, + }); + const { record } = exportRun("auth0", [ + { user_id: "google-oauth2|g1", email: "g@x.dev", email_verified: true }, + ]); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id, dryRun: true }); + + expect(captured.err).toContain("1 user signed in with Google, which is not enabled in Clerk"); + }); + + // A file from a newer CLI names a platform this one doesn't know; yours maps it. + test("a source you wrote maps an export from a platform this CLI doesn't know", async () => { + const { record } = exportRun("okta", auth0Rows); + + await run({ ...noSource, source: ownAuth0Source(), input: record.id }); + + const created = requests.filter((r) => r.url.endsWith("/v1/users")); + expect(created).toHaveLength(1); + }); + + test("the printed command keeps a source you wrote in place of the envelope's", async () => { + const { record } = exportRun("auth0", auth0Rows); + const own = ownAuth0Source(); + + const error = (await run({ ...noSource, source: own, input: record.id, yes: false }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.examples?.[0]?.command).toContain( + `clerk migrate import ${record.id} --source ${own}`, + ); + }); + test("refuses a run ID that is not an export", async () => { await run(baseOptions); const [imported] = listRuns(runsDir()); @@ -2034,6 +2256,80 @@ describe("run", () => { fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); expect(await exitCodeOf(run(baseOptions))).toBe(EXIT_CODE.USAGE); + // A new run would send the same external IDs alongside it. + expect(await exitCodeOf(run({ ...baseOptions, newRun: true }))).toBe(EXIT_CODE.USAGE); + }); + + // The scan of runs alone has a gap: a second process can pass it before + // either has written a run. The import's own lock closes it. + test("refuses while another process holds this import's lock, with no run yet", async () => { + // Keyed by the key, so it holds whether or not `GET /v1/instance` answers. + const lock = importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + instanceId: keyInstanceId(baseOptions.secretKey), + }); + // PID 1 is always alive, and never this test. + fs.writeFileSync(lock, "1"); + try { + await expect(run(baseOptions)).rejects.toThrow( + /importing this file into this instance right now/, + ); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); + }); + + // A different --runs-dir is a different record of the same job. + test("refuses while another process holds this import's lock under another runs folder", async () => { + const lock = importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + instanceId: keyInstanceId(baseOptions.secretKey), + }); + fs.writeFileSync(lock, "1"); + try { + await expect( + run({ ...baseOptions, runsDir: path.join(workDir, "other-runs") }), + ).rejects.toThrow(/importing this file into this instance right now/); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + }); + + // A second key for the same instance has its own stand-in ID. + test("refuses while another key's process holds this instance's lock", async () => { + const lock = importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + instanceId: "ins_1", + }); + fs.writeFileSync(lock, "1"); + try { + await expect(run(baseOptions)).rejects.toThrow( + /importing this file into this instance right now/, + ); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + // The key's own lock, taken first, is let go when the second is refused. + const own = importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + instanceId: keyInstanceId(baseOptions.secretKey), + }); + expect(fs.existsSync(own)).toBe(false); + }); + + test("lets go of the import's locks when the run ends", async () => { + await run(baseOptions); + const lockFor = (instanceId: string) => + importLockFile({ + sha256: sha256File(path.join(workDir, "export.json")), + instanceId, + }); + expect(fs.existsSync(lockFor(keyInstanceId(baseOptions.secretKey)))).toBe(false); + expect(fs.existsSync(lockFor("ins_1"))).toBe(false); }); // An edited file is a different job. @@ -2094,6 +2390,351 @@ describe("run", () => { }); }); + describe("--source ", () => { + const CUSTOM = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email", given: "firstName", pw: "password" }, + carries: { + passwords: { level: "yes", note: "bcrypt." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { if (!user.firstName) delete user.firstName; }, + };`; + + let customFile: string; + let customCounter = 0; + + beforeEach(() => { + // A fresh filename each time: dynamic import() caches by URL, so reusing + // one would silently return a previous test's module. + customFile = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, customFile), CUSTOM); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { account_ref: "mp_1", contact_email: "a@x.dev", given: "Ada", pw: BCRYPT }, + { account_ref: "mp_2", contact_email: "b@x.dev", given: "", pw: BCRYPT }, + ]), + ); + }); + + afterEach(() => { + __resetCustomSourcesForTesting(); + }); + + const created = () => requests.filter((r) => r.url.endsWith("/v1/users")); + + // The registered key isn't something --source accepts; the path is. + test("the printed command names the source's path, not its key", async () => { + const error = (await run({ + input: "export.json", + source: customFile, + secretKey: "sk_test_x", + json: true, + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.examples?.[0]?.command).toBe( + `clerk migrate import export.json --source ${customFile} --secret-key --json --yes`, + ); + }); + + // The checks reject the users a run under another source created, so the + // refusal names that run, as it does for one of the same source. + test("a refusal names the earlier run of another source that holds the users", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const [first] = listRuns(runsDir()); + const other = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, other), + CUSTOM.replace('key: "myplatform"', 'key: "otherplatform"'), + ); + stubClerk({ + existing: [ + { id: "user_1", external_id: "mp_1" }, + { id: "user_2", external_id: "mp_2" }, + ], + }); + + const error = (await run({ + input: "export.json", + source: other, + newRun: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.message).toContain(`2 of them were created by run ${first!.id}`); + expect(error.examples?.map((example) => example.description)).toContain( + `Undo run ${first!.id} first`, + ); + }); + + test("imports through a user-authored source", async () => { + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_1", + "mp_2", + ]); + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("source from"); + }); + + test("applies the custom source's defaults and postTransform", async () => { + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ first_name: "Ada", password_hasher: "bcrypt" }); + // postTransform dropped the empty given name rather than sending "". + expect("first_name" in (bodies[1] ?? {})).toBe(false); + }); + + // An edited source misses the live run's `sourceHash`, but both would send + // the same external IDs. + test("refuses while a run of this file with an earlier version of the source is live", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + // PID 1 is always alive, and never this test. + fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); + + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + await expect( + run({ input: "export.json", source: edited, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/importing this file right now in another process/); + expect(created()).toHaveLength(0); + }); + + /** Rewrites the only run as one a crash stopped, and returns its ID. */ + const interruptOnly = (): string => { + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + return first!.id; + }; + + test("continues an interrupted run when the source is unchanged", async () => { + stubClerk({ failing: new Set(["mp_2"]) }); + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const id = interruptOnly(); + requests = []; + process.exitCode = 0; + stubClerk(); + + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_2", + ]); + expect(captured.err).toContain(`Continuing run ${id}`); + expect(listRuns(runsDir())).toHaveLength(1); + }); + + // Starting over would bring the users the earlier run created back as conflicts. + test("refuses an edited source while an earlier version's run is interrupted, naming it", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const id = interruptOnly(); + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + + const error = (await run({ + input: "export.json", + source: edited, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain( + `Run ${id} imported this file into this instance with an earlier version`, + ); + expect(error.message).toContain("--new-run"); + expect(created()).toHaveLength(0); + }); + + // The newest run being undone doesn't clear an older one that still holds users. + test("refuses an edited source while any earlier version's run is unfinished, not just the newest", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const older = interruptOnly(); + const second = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, second), `${CUSTOM}\n// second`); + await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + const newer = listRuns(runsDir()).find((record) => record.id !== older)!; + fs.writeFileSync( + path.join(runsDir(), newer.id, "run.json"), + JSON.stringify({ ...readRun(runsDir(), newer.id), status: "undone" }), + ); + const third = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, third), `${CUSTOM}\n// third`); + requests = []; + + const error = (await run({ + input: "export.json", + source: third, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.message).toContain( + `Run ${older} imported this file into this instance with an earlier version`, + ); + expect(error.message).toContain(`clerk migrate undo ${older} --secret-key `); + expect(created()).toHaveLength(0); + }); + + // The newer, undone run is of this same version: undone, it starts a new + // run, so the older interrupted one has to be checked too. + test("refuses while an earlier version's run is unfinished behind an undone run of this version", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const older = interruptOnly(); + const second = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, second), `${CUSTOM}\n// second`); + await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + const newer = listRuns(runsDir()).find((record) => record.id !== older)!; + fs.writeFileSync( + path.join(runsDir(), newer.id, "run.json"), + JSON.stringify({ ...readRun(runsDir(), newer.id), status: "undone" }), + ); + requests = []; + + const error = (await run({ + input: "export.json", + source: second, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain( + `Run ${older} imported this file into this instance with an earlier version`, + ); + expect(created()).toHaveLength(0); + }); + + test("an edited source with --new-run starts a new run beside the earlier one", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + interruptOnly(); + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + + await run({ + input: "export.json", + source: edited, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + + expect(listRuns(runsDir())).toHaveLength(2); + }); + + // The loader caches by URL: an edit at the same path must still load the + // edited mappings, the ones the run's hash names. + test("an edited source at the same path maps with its edited code", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + fs.writeFileSync( + path.join(workDir, customFile), + CUSTOM.replace('given: "firstName"', 'given: "lastName"'), + ); + requests = []; + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ last_name: "Ada" }); + expect("first_name" in (bodies[0] ?? {})).toBe(false); + }); + + // An edited source is a different source, so the run records which one. + test("records the custom source's content hash on the run", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + + const [record] = listRuns(runsDir()); + expect(record?.source).toBe("myplatform"); + expect(record?.sourceHash).toMatch(/^[0-9a-f]{64}$/); + }); + + test("an unknown built-in key is a usage error listing the valid ones", async () => { + await expect( + run({ input: "export.json", source: "okta", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/Unknown source "okta". Valid sources: clerk, auth0/); + expect(created()).toHaveLength(0); + }); + + test("fails before any request when the file is not there", async () => { + await expect( + run({ + input: "export.json", + source: "./nope.ts", + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/No source file at/); + expect(requests).toHaveLength(0); + }); + + test("fails before any request when the file is malformed", async () => { + const bad = `./bad-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, bad), + `export default { key: "x", label: "X", transformer: {} };`, + ); + + const error = (await run({ + input: "export.json", + source: bad, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + expect(error.message).toMatch(/no source field maps to `userId`/); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(requests).toHaveLength(0); + }); + + test("still requires a file", async () => { + await expect(run({ source: customFile, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /needs the file to import/, + ); + }); + }); + describe("per-platform imports", () => { /** One realistic record per platform, in that platform's export shape. */ const PLATFORMS: [string, unknown, string][] = [ diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 27d853609..ee6afbb75 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -61,6 +61,7 @@ import { continueRun, latestUserLines, listRuns, + lockImport, readRun, readUserLines, resolveRunsDir, @@ -111,8 +112,14 @@ import { link } from "../link/index.ts"; export type MigrateRunOptions = { /** An export file, or the ID of the export run that wrote one. */ input?: string; - /** A built-in source key. */ + /** A built-in source key, or the path to a source you wrote. */ source?: string; + /** Content hash of a custom `--source`, set once it is loaded. */ + sourceHash?: string; + /** `--source` as typed, for printed commands: a custom source's path. Not a flag. */ + sourceArg?: string; + /** The source an export file named, when a custom `--source` replaced it. Not a flag. */ + envelopeSource?: string; /** The file to read, once `input` is resolved. Not a flag. */ file?: string; requirePassword?: boolean; @@ -140,7 +147,7 @@ export type MigrateRunOptions = { * Validates what a run needs before anything is read or sent. * * `--source` has already been resolved to a registered key by the time this - * runs. + * runs, custom sources included. * * @returns The source key and file path, both guaranteed present. */ @@ -150,7 +157,7 @@ export function validateRunOptions(options: MigrateRunOptions): { } { if (!options.source) { throwUsageError( - `Missing --source. Valid values: ${sourceKeys().join(", ")}.`, + `Missing --source. Valid values: ${sourceKeys().join(", ")}, or the path to a source you wrote.`, undefined, ERROR_CODE.USAGE_ERROR, [ @@ -375,32 +382,54 @@ function throwChangedExport(runId: string, file: string): never { } /** - * Resolves `--source` to a registered key. + * Resolves `--source` to a registered key, loading a custom source from its + * path so the rest of the run treats it exactly like a built-in. * * @throws UsageError for an unknown key. */ async function applySource(options: MigrateRunOptions): Promise { if (!options.source) return options; const resolved = await resolveSource(options.source); - return { ...options, source: resolved.key }; + if (resolved.path && !options.json) { + log.info(`Loaded the \`${resolved.key}\` source from ${options.source}.`); + } + return { + ...options, + source: resolved.key, + sourceArg: options.source, + ...(resolved.hash ? { sourceHash: resolved.hash } : {}), + }; } /** * The source an export file names for itself, checked against the one the - * flags name. + * flags name. A source you wrote replaces it: it reads the same users, in the + * platform's own shape, and maps them as it says. * - * @throws UsageError when the two disagree: importing an Auth0 export through - * the Supabase mapping would create users with the wrong fields. + * @throws UsageError when two built-in sources disagree: importing an Auth0 + * export through the Supabase mapping would create users with the wrong + * fields. */ function applyEnvelope( options: MigrateRunOptions, envelope: ExportEnvelope | undefined, ): MigrateRunOptions { if (!envelope) return options; + // A built-in name is a claim about where the file came from, and a wrong + // one is a mistake. A path is a choice to map it differently, say to bring + // across fields the built-in mapping leaves out. + if (options.sourceHash) { + if (!options.json) { + log.info( + `Mapping this ${envelope.source} export with ${options.sourceArg}, in place of the built-in \`${envelope.source}\` source.`, + ); + } + return { ...options, envelopeSource: envelope.source }; + } if (options.source && options.source !== envelope.source) { throwUsageError( `The file was exported from ${envelope.source}, but --source names ${options.source}. ` + - "Drop --source: the file already says where it came from.", + "Drop --source: the file already says where it came from. To map its users differently, pass the path to a source you wrote.", ); } // `--source` goes through the registry; the file's own word must too, or a @@ -416,6 +445,54 @@ function applyEnvelope( // --- Continuing an earlier run --------------------------------------------- +/** + * Refuses while another process is importing this file, with this source key, + * into this instance. Checked before any resume matching, and under + * --new-run too: an edited custom source (another `sourceHash`) or a new run + * would otherwise send the same external IDs alongside it. + */ +function assertNoActiveImport( + runsDir: string, + match: { sha256: string; instanceId: string; keyInstanceId?: string }, +): void { + const active = listRuns(runsDir).find( + (record) => sameFileAndInstance(record, match) && runState(runsDir, record) === "running", + ); + if (active) { + throwUsageError( + inUseMessage(runsDir, active.id, "importing this file", "Wait for it to finish."), + ); + } +} + +/** + * An import of this file into this instance, by its ID or its key's stand-in + * ID, under any source. Whichever source maps them, its users are the same + * people, so another source's run, or another version of one, holds them too. + */ +function sameFileAndInstance( + record: RunRecord, + match: { sha256: string; instanceId: string; keyInstanceId?: string }, +): boolean { + return ( + record.kind === "import" && + record.file?.sha256 === match.sha256 && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId) + ); +} + +/** True when a run's users are gone: it is marked undone, or an undo of it finished. */ +function undoneInFull(runsDir: string, record: RunRecord): boolean { + return ( + runState(runsDir, record) === "undone" || + listRuns(runsDir).some( + (undo) => + undo.kind === "undo" && undo.undoes === record.id && runState(runsDir, undo) === "complete", + ) + ); +} + /** How this import relates to earlier runs of the same file, source and instance. */ type ResumeCase = | { kind: "new" } @@ -423,8 +500,8 @@ type ResumeCase = | { kind: "complete"; record: RunRecord }; /** - * Finds the latest import run of this file (by sha256), this source and this - * instance, and decides what a re-run does: + * Finds the latest import run of this file (by sha256), this source (and a + * custom source's hash) and this instance, and decides what a re-run does: * * - none, or undone: a new run * - interrupted: continue it, skipping the users it created @@ -434,13 +511,16 @@ type ResumeCase = * @throws UsageError when that run is still running in another process, or * has an undo that never finished: some of its users are gone and some are * not, so neither continuing it nor starting over is safe. Also when Clerk - * did not confirm the instance and a run of this file names a real one. + * did not confirm the instance and a run of this file names a real one, or + * when the latest run of this file was made with an earlier version of a + * custom source and was not undone. */ function findResume( runsDir: string, match: { sha256: string; source: string; + sourceHash?: string; instanceId: string; /** The key's stand-in ID, for a run recorded while Clerk could not name the instance. */ keyInstanceId: string; @@ -449,18 +529,43 @@ function findResume( carry: string[], ): ResumeCase { const runs = listRuns(runsDir); - const ofFile = importsOfFile(runs, match); - const latest = ofFile.find( - (record) => - record.target.instanceId === match.instanceId || - record.target.instanceId === match.keyInstanceId, + // Only a run of this same source, and the same version of a custom one, + // is continued. + const latest = importsOfFile(runs, match).find( + (record) => sameFileAndInstance(record, match) && record.sourceHash === match.sourceHash, ); - if (!latest) { + // An undone exact match starts a new run, so it checks the others too. + if (!latest || undoneInFull(runsDir, latest)) { + // Another source's run of this file, or an earlier version of this one, + // sent the same users: starting over beside it would reject every user it + // created and leave it unfinished, so that takes --new-run. Any such run + // not undone, not just the newest: an older interrupted run behind a + // newer undone one still holds its users. + const earlier = runs.find( + (record) => sameFileAndInstance(record, match) && !undoneInFull(runsDir, record), + ); + if (earlier) { + const how = + earlier.source !== match.source + ? `with the \`${earlier.source}\` source` + : earlier.sourceHash !== match.sourceHash + ? "with an earlier version of this source" + : "too, and was not undone"; + throwUsageError( + `Run ${earlier.id} imported this file into this instance ${how}, so the users it created would come back as conflicts. Nothing was imported.\n` + + `Pass --new-run to start a new run beside it, or undo it first with \`${undoCommand(earlier.id, carry)}\`.`, + ); + } // Clerk did not name the instance (GET /v1/instance failed), so a run of // this file recorded under a real ID can be neither matched nor ruled out. // Starting over would reject every user it created and leave it unfinished. if (isStandInInstanceId(match.instanceId)) { - const unconfirmed = ofFile.find((record) => isClerkInstanceId(record.target.instanceId)); + const unconfirmed = runs.find( + (record) => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + isClerkInstanceId(record.target.instanceId), + ); if (unconfirmed) { throwUsageError( `Clerk did not confirm which instance this key addresses, so this import cannot tell whether run ${unconfirmed.id} is for it. ` + @@ -668,7 +773,9 @@ function commandFor( ) { const input = fromExport ?? options.input ?? options.file ?? ""; const parts = ["clerk migrate import", quoteArg(input)]; - if (!fromExport && options.source) parts.push("--source", quoteArg(options.source)); + const source = options.sourceArg ?? options.source; + // An export names its own source, unless a source you wrote replaced it. + if ((!fromExport || options.envelopeSource) && source) parts.push("--source", quoteArg(source)); if (options.newRun) parts.push("--new-run"); parts.push(...importFlags(options)); // Names, not `<…>`: pasted as is, a shell reads `` as a redirect. @@ -697,14 +804,13 @@ function rejectSummary(checks: ImportChecks, json = false): string { } /** - * The earlier import run, of this source into this instance, that created the + * The earlier import run into this instance, of any source, that created the * users the checks reject as already there: how many of them, and whether it * ran this same file. Fixing a skipped row changes the file, and the changed * file's run meets them; so does the same file run with `--new-run`. */ function earlierRunHolding( runsDir: string, - source: string, instanceId: string, checks: ImportChecks, sha256: string, @@ -714,7 +820,8 @@ function earlierRunHolding( ); if (already.size === 0) return undefined; for (const record of listRuns(runsDir)) { - if (record.kind !== "import" || record.source !== source) continue; + // Any source: findResume and the import lock match on file and instance. + if (record.kind !== "import") continue; if (record.target.instanceId !== instanceId || runState(runsDir, record) === "undone") continue; let count = 0; for (const line of latestUserLines(runsDir, record.id).values()) { @@ -784,6 +891,20 @@ function recordRejects(run: Run, checks: ImportChecks): void { } export async function run(rawOptions: MigrateRunOptions): Promise { + // Released on a return or a refusal. A Ctrl-C can kill the process first; + // the next import reclaims a lock whose PID is gone. + const lock: ImportLock = {}; + try { + await runImport(rawOptions, lock); + } finally { + lock.release?.(); + } +} + +/** The import's identity lock, taken part-way through and released by {@link run}. */ +type ImportLock = { release?: () => void }; + +async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promise { // A mistyped --source fails before any sign-in or link. let options = await applySource(rawOptions); await ensureImportTarget(options); @@ -814,15 +935,19 @@ export async function run(rawOptions: MigrateRunOptions): Promise { // The flags every suggested command needs to reach this runs folder and instance. const carry = carryFlags(options); + // The platform the file came from. An export through a source you wrote is + // still that platform's: Firebase digests need the project's parameters, and + // Supabase's raw rows hold its users' providers. + const platform = options.envelopeSource ?? source; // A flag is typed for this run, so it wins over the export file. A variable // is ambient, and may be left over from another project, so it only fills in // for an export file without the parameters: with one, a partial flag set is // refused rather than completed from the variables. - const fromEnvelope = source === "firebase" ? envelope?.firebase : undefined; + const fromEnvelope = platform === "firebase" ? envelope?.firebase : undefined; const fromFlagsAtAll = FIREBASE_FLAGS.some(([key]) => options[key] !== undefined); const fromFlags = fromFlagsAtAll || !fromEnvelope - ? resolveFirebaseHashConfig(options, source, fromEnvelope ? {} : process.env) + ? resolveFirebaseHashConfig(options, platform, fromEnvelope ? {} : process.env) : undefined; let firebaseHashConfig = fromFlags ?? fromEnvelope; let configFrom = fromFlags @@ -831,7 +956,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { : "the CLERK_FIREBASE_* variables" : "the export file"; // `-y` is "import without prompting": it gets the usage error naming the flags. - if (source === "firebase" && !firebaseHashConfig && !options.yes && canPrompt(options)) { + if (platform === "firebase" && !firebaseHashConfig && !options.yes && canPrompt(options)) { firebaseHashConfig = await promptForFirebaseHashConfig(); configFrom = "the parameters entered"; // A suggested command runs with --yes, which never asks: it names the flags. @@ -874,9 +999,37 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (expectedSha256 !== undefined && sha256 !== expectedSha256) throwChanged(filePath); const runsDir = await resolveRunsDir(options.runsDir); + // Held from before the checks to the end of the run: the scan below + // alone leaves a gap a second process can pass through before either + // one writes a run. + // Two locks: the key's stand-in ID, which the same key takes when + // `GET /v1/instance` fails, and the instance's ID when Clerk named it, + // which a second key for the instance takes. + // ponytail: a second key whose lookup failed takes neither; the run scan + // below is all that stops it. A lock on the key's instance needs Clerk. + if (!options.dryRun) { + const releases: (() => void)[] = []; + try { + for (const instanceId of new Set([keyInstanceId(secretKey), target.instanceId])) { + releases.push(lockImport({ sha256, instanceId })); + } + } catch (error) { + for (const release of releases) release(); + throw error; + } + lock.release = () => { + for (const release of releases) release(); + }; + } + assertNoActiveImport(runsDir, { + sha256, + instanceId: target.instanceId, + keyInstanceId: keyInstanceId(secretKey), + }); const match = { sha256, source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), instanceId: target.instanceId, keyInstanceId: keyInstanceId(secretKey), }; @@ -992,7 +1145,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { } let supabaseRows: Record[] | undefined; - if (source === "supabase") { + if (platform === "supabase") { try { supabaseRows = await readSupabaseRows(file); } catch (error) { @@ -1046,7 +1199,11 @@ export async function run(rawOptions: MigrateRunOptions): Promise { failures, unknownFields: loaded.unknownFields, ...(supabaseRows ? { supabaseRows } : {}), - socialProviders: getSource(source).socialProviders, + // A source you wrote without its own counts the export platform's, + // when this CLI knows the platform: yours may map one it doesn't. + socialProviders: + getSource(source).socialProviders ?? + (sourceKeys().includes(platform) ? getSource(platform).socialProviders : undefined), settings, existingUsers, maxAllowedUsers, @@ -1107,7 +1264,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { if (refused) { if (options.json) preview({ refused: true }); - const earlier = earlierRunHolding(runsDir, source, target.instanceId, checks, sha256); + const earlier = earlierRunHolding(runsDir, target.instanceId, checks, sha256); const undoEarlier = earlier ? undoCommand(earlier.id, carry) : undefined; const earlierRun = options.newRun && earlier @@ -1248,8 +1405,10 @@ export async function run(rawOptions: MigrateRunOptions): Promise { kind: "import", target, source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), file: { path: filePath, sha256 }, ...(input.fromExport ? { fromExport: input.fromExport } : {}), + ...(options.envelopeSource ? { envelopeSource: options.envelopeSource } : {}), ...(firebaseHash ? { firebaseHash } : {}), flags, }); diff --git a/packages/cli-core/src/commands/migrate/runs.test.ts b/packages/cli-core/src/commands/migrate/runs.test.ts index eb2c4b6cb..d9b541333 100644 --- a/packages/cli-core/src/commands/migrate/runs.test.ts +++ b/packages/cli-core/src/commands/migrate/runs.test.ts @@ -406,6 +406,17 @@ describe("runs ", () => { expect(next[0]).toContain("to retry the users it didn't finish"); }); + // The record has the custom source's hash, not its path. + test("a partial import from a source you wrote names the source file to pass", async () => { + const record = partialImport({ source: "acme", sourceHash: "abc123" }); + + await runs(record.id, { runsDir, json: true }); + + const { next } = JSON.parse(captured.out) as { next: string[] }; + expect(next[0]).toContain(`clerk migrate import ${quoteArg(usersFile())} --source SOURCE_FILE`); + expect(next[0]).toContain("with SOURCE_FILE replaced by the path to the source you wrote"); + }); + test("an unknown ID is a usage error", async () => { await expect(runs("nope", { runsDir })).rejects.toThrow(/No run `nope`/); }); diff --git a/packages/cli-core/src/commands/migrate/runs.ts b/packages/cli-core/src/commands/migrate/runs.ts index 70660eaa1..427c7fe93 100644 --- a/packages/cli-core/src/commands/migrate/runs.ts +++ b/packages/cli-core/src/commands/migrate/runs.ts @@ -124,16 +124,19 @@ function continueCommand(record: RunRecord, carry: string[]): string | undefined const input = record.fromExport ?? record.file?.path; if (!input) return undefined; const parts = ["clerk migrate import", quoteArg(input)]; - if (!record.fromExport && record.source) parts.push("--source", quoteArg(record.source)); + // A source you wrote is a file the record knows only by its hash. A name, + // not `<…>`: pasted as is, a shell reads `` as a redirect. + if (record.sourceHash) parts.push("--source", "SOURCE_FILE"); + else if (!record.fromExport && record.source) parts.push("--source", quoteArg(record.source)); return withFlags(parts.join(" "), [...importFlags(record.flags), ...carry, "--yes"]); } /** * Whether importing the record's file again continues this run, rather than - * starting another or refusing: no newer import of that file and source may - * be for the same instance, the file is unchanged, and no undo of it exists. - * An unfinished undo makes an import refuse; a finished one that never marked - * the run makes it start a new one. + * starting another or refusing: no newer import of that file and source (and + * version of a source you wrote) may be for the same instance, the file is + * unchanged, and no undo of it exists. An unfinished undo makes an import + * refuse; a finished one that never marked the run makes it start a new one. */ function continuesThisRun(runsDir: string, record: RunRecord): boolean { const file = record.file; @@ -146,8 +149,8 @@ function continuesThisRun(runsDir: string, record: RunRecord): boolean { id === record.target.instanceId || isStandInInstanceId(id) || isStandInInstanceId(record.target.instanceId); - const newest = importsOfFile(runs, { sha256: file.sha256, source: record.source }).find((other) => - mayBeSameInstance(other.target.instanceId), + const newest = importsOfFile(runs, { sha256: file.sha256, source: record.source }).find( + (other) => other.sourceHash === record.sourceHash && mayBeSameInstance(other.target.instanceId), ); if (newest?.id !== record.id) return false; if (undosOf(runs, record.id).length > 0) return false; @@ -179,7 +182,11 @@ function nextCommands( // Continuing is what a stopped run is for: the same file, source and // target pick it up where it left off. const why = state === "interrupted" ? "continue it" : "retry the users it didn't finish"; - return [`Run \`${again}\` to ${why}`, undo]; + // The record knows a source you wrote only by its hash. + const placeholder = record.sourceHash + ? ", with SOURCE_FILE replaced by the path to the source you wrote" + : ""; + return [`Run \`${again}\` to ${why}${placeholder}`, undo]; } if (record.kind === "undo" && state !== "complete" && record.undoes) { return [`Run \`${undoCommand(record.undoes, carry)}\` to retry the users that failed`]; diff --git a/packages/cli-core/src/commands/migrate/sources/list.test.ts b/packages/cli-core/src/commands/migrate/sources/list.test.ts new file mode 100644 index 000000000..e452580f3 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.test.ts @@ -0,0 +1,189 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { list, wrapText } from "./list.ts"; +import { ACCOUNT_LINKING_URL, sources } from "./registry.ts"; + +const captured = useCaptureLog(); + +/** `log.info` highlights backticked spans and the levels are coloured. */ +const plain = () => Bun.stripANSI(captured.err).replace(/\s+/g, " "); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-sources-"))); + process.chdir(workDir); + fs.writeFileSync( + path.join(workDir, "custom.ts"), + `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", mail: "email", full_name: "name" }, + carries: { + passwords: { level: "partial", note: "Only for accounts made after 2020." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "yes", note: "All of it." }, + }, + };`, + ); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("sources", () => { + test.each([...sources])("lists $key with what it carries", async (entry) => { + await list(undefined); + const row = plain().match(new RegExp(`${entry.key} (\\w+) (\\w+) (\\w+)`)); + expect(row?.slice(1)).toEqual([ + entry.carries.passwords.level, + entry.carries.mfa.level, + entry.carries.metadata.level, + ]); + }); + + // No social column: every source shares one note. + test("prints the account-linking note once", async () => { + await list(undefined); + expect(captured.err.split(ACCOUNT_LINKING_URL)).toHaveLength(2); + }); + + // A compiled binary has no source tree to grep, so the way to extend it has + // to be discoverable from the list itself. + test("says how to add one", async () => { + await list(undefined); + expect(captured.err).toContain("--source ./my-source.ts"); + expect(captured.err).toContain("README.md#custom-sources"); + }); + + test("--json lists every built-in with its carries on stdout", async () => { + await list(undefined, { json: true }); + + const parsed = JSON.parse(captured.out) as { + sources: { key: string; carries: unknown }[]; + account_linking: string; + }; + expect(parsed.sources.map((entry) => entry.key)).toEqual(sources.map((entry) => entry.key)); + expect(parsed.sources[0]?.carries).toEqual(sources[0]?.carries); + expect(parsed.account_linking).toContain(ACCOUNT_LINKING_URL); + }); +}); + +describe("sources ", () => { + test("shows where each field lands, how to export, and what comes across", async () => { + await list("supabase"); + + expect(plain()).toContain("Export with `clerk migrate export supabase`"); + expect(plain()).toContain("encrypted_password → password"); + expect(plain()).toContain("Passwords yes"); + expect(captured.err).toContain(ACCOUNT_LINKING_URL); + }); + + // Supabase detects its hasher per user, so a source with a fixed one shows it. + test("shows a source's fixed field values", async () => { + await list("firebase"); + expect(plain()).toContain('passwordHasher is always "scrypt_firebase"'); + }); + + test("shows a source you wrote, by its path", async () => { + await list("./custom.ts"); + + expect(plain()).toContain("myplatform My Platform (custom — ./custom.ts)"); + expect(plain()).toContain("mail → email"); + expect(plain()).toContain("Only for accounts made after 2020."); + }); + + // `name` reads like a Clerk field and isn't one: copied from a built-in into + // a source you wrote, it was dropped with no word until the import's checks. + test("marks a target that isn't a Clerk field", async () => { + await list("authjs"); + expect(plain()).toContain("name → name not a Clerk field; the source handles it"); + expect(plain()).not.toContain("email → email not a Clerk field"); + + captured.clear(); + await list("./custom.ts"); + expect(plain()).toContain( + "full_name → name not a Clerk field: Clerk won't store it unless postTransform moves it", + ); + }); + + test("--json returns the detail on stdout", async () => { + await list("auth0", { json: true }); + + expect(JSON.parse(captured.out)).toMatchObject({ + key: "auth0", + export_command: "clerk migrate export auth0", + fields: { user_id: "userId" }, + // The fixed defaults the detail view prints, as the README says. + defaults: { passwordHasher: "bcrypt" }, + carries: { passwords: { level: "partial" } }, + }); + }); + + test("an unknown source is a usage error naming the valid ones", async () => { + const error = (await list("okta").catch((caught: unknown) => caught)) as CliError; + expect(error).toBeInstanceOf(CliError); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Valid sources: clerk, auth0"); + }); +}); + +describe("human-mode frame", () => { + let originalMode: string | undefined; + + beforeAll(() => { + originalMode = process.env.CLERK_MODE; + process.env.CLERK_MODE = "human"; + }); + + afterAll(() => { + if (originalMode === undefined) delete process.env.CLERK_MODE; + else process.env.CLERK_MODE = originalMode; + }); + + // Reading a static registry is not a run: there is no progress to bracket, + // and the gutter's `│` would sit in front of every wrapped line. + test("prints no intro/outro gutter", async () => { + await list(undefined); + + expect(captured.err).not.toContain("┌"); + expect(captured.err).not.toContain("└"); + }); + + test("--json stays on stdout only", async () => { + await list(undefined, { json: true }); + + expect(() => JSON.parse(captured.out)).not.toThrow(); + expect(captured.err).toBe(""); + }); +}); + +describe("wrapText", () => { + test("breaks on whitespace within the width", () => { + expect(wrapText("one two three four", 9)).toEqual(["one two", "three", "four"]); + }); + + // `log.info` pairs backticks per line, so a span split across two lines + // leaves one unmatched backtick on each and colours the wrong half of both. + test("never breaks inside a backticked span", () => { + const lines = wrapText("Assumes an export of `SELECT id, name FROM users`.", 30); + + expect(lines).toContain("`SELECT id, name FROM users`."); + for (const line of lines) { + expect((line.match(/`/g) ?? []).length % 2).toBe(0); + } + }); + + test("gives an over-long word its own line rather than dropping it", () => { + expect(wrapText("short supercalifragilistic", 8)).toEqual(["short", "supercalifragilistic"]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts new file mode 100644 index 000000000..af69263d5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -0,0 +1,215 @@ +/** + * `clerk migrate sources [source]` — which platforms an import can read, and + * what each one brings across. + * + * A compiled binary's users have no source tree to read the mappings in, so + * this is where they are shown. `sources` alone lists every source with its + * passwords, MFA and metadata at a glance. `sources ` shows one in + * full: what each field becomes, how to export from it, and its caveats. + */ + +import { bold, cyan, dim, green, red, yellow } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { exportPlatforms } from "../export/registry.ts"; +import { SCHEMA_FIELDS } from "../lib/transform.ts"; +import type { Carry, CarryLevel, SourceEntry } from "../types.ts"; +import { CUSTOM_SOURCES_URL } from "./load-custom.ts"; +import { ACCOUNT_LINKING_NOTE, resolveSource, sources as builtIns } from "./registry.ts"; + +export type SourcesOptions = { + json?: boolean; +}; + +const KINDS = [ + ["passwords", "Passwords"], + ["mfa", "MFA"], + ["metadata", "Metadata"], +] as const; + +/** + * Capped, not just measured: a description that rewrapped differently on every + * terminal makes two runs of the same command look like different output. 80 is + * the same width `--help` lays itself out at. + */ +const MAX_WIDTH = 80; + +function outputWidth(): number { + return Math.min(process.stderr.columns || MAX_WIDTH, MAX_WIDTH); +} + +/** + * A run of non-space characters, except that a backticked span counts as one + * character run even when it contains spaces. Keeps `SELECT a, b FROM users` + * whole: `log.info` pairs backticks per line, so a span broken across two lines + * leaves an unmatched backtick on each and colours the wrong half of both. + */ +const WORD = /(?:`[^`]*`|\S)+/g; + +/** + * Wraps on whitespace. Safe to measure raw because the backtick spans + * `log.info` highlights keep their backticks — the colour it adds is invisible + * to width, and nothing here is coloured before wrapping. + */ +export function wrapText(text: string, width: number): string[] { + const lines: string[] = []; + let line = ""; + + for (const word of text.match(WORD) ?? []) { + if (!line) line = word; + else if (line.length + 1 + word.length <= width) line += ` ${word}`; + else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + + return lines; +} + +function levelMark(level: CarryLevel): string { + if (level === "yes") return green("yes"); + if (level === "partial") return yellow("partial"); + return red("no"); +} + +/** The command that writes a file this source reads, when there is one. */ +function exportCommand(entry: SourceEntry): string | undefined { + const platform = exportPlatforms.find((candidate) => candidate.sourceKey === entry.key); + return platform ? `clerk migrate export ${platform.key}` : undefined; +} + +function toJson(entry: SourceEntry, custom?: string) { + return { + key: entry.key, + label: entry.label, + description: entry.description, + ...(custom ? { path: custom } : {}), + carries: entry.carries, + export_command: exportCommand(entry) ?? null, + fields: entry.transformer, + defaults: entry.defaults ?? {}, + caveats: entry.caveats ?? [], + }; +} + +function printList(width: number): void { + for (const line of wrapText( + "A source maps one platform's export onto the fields Clerk imports. A file from " + + "`clerk migrate export` names its own; anything else takes `--source `.", + width, + )) { + log.info(line); + } + log.blank(); + + const keyWidth = Math.max(...builtIns.map((entry) => entry.key.length)) + 2; + log.info( + bold( + `${"SOURCE".padEnd(keyWidth)}${"PASSWORDS".padEnd(11)}${"MFA".padEnd(9)}${"METADATA".padEnd(10)}`, + ), + ); + for (const entry of builtIns) { + const cell = (carry: Carry, pad: number) => + levelMark(carry.level) + " ".repeat(Math.max(1, pad - carry.level.length)); + log.info( + `${cyan(entry.key.padEnd(keyWidth))}${cell(entry.carries.passwords, 11)}` + + `${cell(entry.carries.mfa, 9)}${levelMark(entry.carries.metadata.level)}`, + ); + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); + log.blank(); + log.info("Run `clerk migrate sources ` for what each field becomes."); + log.info("Migrating from something else? Write a source and pass --source ./my-source.ts."); + log.info(`How to write one: ${CUSTOM_SOURCES_URL}`); +} + +function printDetail(entry: SourceEntry, width: number, custom?: string): void { + log.info(`${cyan(bold(entry.key))} ${entry.label}${custom ? dim(` (custom — ${custom})`) : ""}`); + for (const line of wrapText(entry.description, width)) log.info(line); + + const command = exportCommand(entry); + if (command) { + log.blank(); + log.info(`${bold("Export with")} \`${command}\``); + } + + log.blank(); + log.info(bold("What comes across")); + for (const [kind, label] of KINDS) { + const carry = entry.carries[kind]; + log.info(` ${label.padEnd(10)}${levelMark(carry.level)}`); + for (const line of wrapText(carry.note, width - 4)) log.info(` ${dim(line)}`); + } + + const fields = Object.entries(entry.transformer); + const fromWidth = Math.max(...fields.map(([from]) => from.length)) + 2; + log.blank(); + log.info(bold("Where each field lands")); + for (const [from, to] of fields) { + // A built-in maps some columns to keys its postTransform consumes. In a + // source you wrote, the same is a column Clerk drops unless yours moves it. + const note = SCHEMA_FIELDS.has(to) + ? "" + : custom + ? yellow(" not a Clerk field: Clerk won't store it unless postTransform moves it") + : dim(" not a Clerk field; the source handles it"); + log.info(` ${from.padEnd(fromWidth)}→ ${to}${note}`); + } + if (entry.defaults) { + for (const [field, value] of Object.entries(entry.defaults)) { + log.info(dim(` ${field} is always ${JSON.stringify(value)}`)); + } + } + + if (entry.caveats?.length) { + log.blank(); + log.info(bold("Caveats")); + for (const caveat of entry.caveats) { + const [first, ...rest] = wrapText(caveat, width - 4); + log.info(` - ${first ?? ""}`); + for (const line of rest) log.info(` ${line}`); + } + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); +} + +export async function list(key: string | undefined, options: SourcesOptions = {}): Promise { + const width = outputWidth(); + + if (key === undefined) { + if (options.json) { + log.data( + JSON.stringify( + { + sources: builtIns.map((entry) => toJson(entry)), + account_linking: ACCOUNT_LINKING_NOTE, + }, + null, + 2, + ), + ); + return; + } + printList(width); + return; + } + + // No gutter: this reads a static registry, it does not run anything. + const resolved = await resolveSource(key); + if (options.json) { + log.data( + JSON.stringify( + { ...toJson(resolved.entry, resolved.path), account_linking: ACCOUNT_LINKING_NOTE }, + null, + 2, + ), + ); + return; + } + printDetail(resolved.entry, width, resolved.path ? key : undefined); +} diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts new file mode 100644 index 000000000..af604ab25 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts @@ -0,0 +1,279 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { loadCustomSource, validateSource } from "./load-custom.ts"; +import { __resetCustomSourcesForTesting, resolveSource } from "./registry.ts"; + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-custom-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +afterEach(() => { + __resetCustomSourcesForTesting(); +}); + +/** + * Writes a source file with a unique name. + * + * Names must not repeat: a dynamic `import()` caches by URL, so reusing one + * would silently return the previous test's module. + */ +function writeSource(source: string, ext = "ts"): string { + const name = `custom-${counter++}.${ext}`; + fs.writeFileSync(path.join(workDir, name), source); + return `./${name}`; +} + +const VALID = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, +};`; + +describe("loadCustomSource", () => { + test("loads a user-authored TypeScript source", async () => { + const entry = await loadCustomSource(writeSource(VALID)); + + expect(entry).toMatchObject({ + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId", contact_email: "email" }, + }); + }); + + test("loads plain JavaScript too", async () => { + const entry = await loadCustomSource(writeSource(VALID, "js")); + expect(entry.key).toBe("myplatform"); + }); + + // The file is the user's own code; the CLI must transpile whatever they wrote. + test("transpiles TypeScript syntax the runtime has to strip", async () => { + const entry = await loadCustomSource( + writeSource(` + interface Entry { key: string; label: string; transformer: Record; carries: unknown } + const mapping = { my_id: "userId" } as const; + const custom: Entry = { key: "tsplatform", label: "TS", transformer: { ...mapping }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } }; + export default custom satisfies Entry; + `), + ); + expect(entry.key).toBe("tsplatform"); + }); + + test("carries the optional hooks through", async () => { + const entry = await loadCustomSource( + writeSource(`export default { + key: "hooked", label: "Hooked", + transformer: { id: "userId" }, + carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { user.firstName = "set"; }, + };`), + ); + + expect(entry.defaults).toEqual({ passwordHasher: "bcrypt" }); + const user: Record = {}; + entry.postTransform?.(user, {}); + expect(user.firstName).toBe("set"); + }); + + test("supplies a description when the author omitted one", async () => { + const entry = await loadCustomSource( + writeSource( + `export default { key: "bare", label: "Bare", transformer: { id: "userId" }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } };`, + ), + ); + expect(entry.description).toBe("Custom source"); + }); + + test("reports a path that is not there", async () => { + await expect(loadCustomSource("./nope.ts")).rejects.toThrow(/No source file at/); + }); + + test("reports a directory given instead of a file", async () => { + fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); + await expect(loadCustomSource("./adir")).rejects.toThrow(/is a directory/); + }); + + // `--source` hashes the file before loading it, and reading a directory + // threw a raw EISDIR ahead of the usage error. + test("--source reports a directory as a usage error, not EISDIR", async () => { + fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); + await expect(resolveSource("./adir")).rejects.toThrow(/is a directory/); + }); + + test("reports a file that does not parse, quoting the syntax error", async () => { + await expect(loadCustomSource(writeSource("export default { key: ,,, }"))).rejects.toThrow( + /Could not load/, + ); + }); + + test("reports a file that throws while loading", async () => { + await expect( + loadCustomSource(writeSource(`throw new Error("boom"); export default {};`)), + ).rejects.toThrow(/Could not load .*boom/s); + }); + + test("points at a named export when the default is missing", async () => { + const file = writeSource( + `export const myPlatform = { key: "x", label: "X", transformer: { a: "userId" } };`, + ); + + await expect(loadCustomSource(file)).rejects.toThrow( + /has no default export.*`myPlatform`.*did you mean `export default`/s, + ); + }); + + test("reports a missing default with no named exports to suggest", async () => { + await expect(loadCustomSource(writeSource("const unused = 1;"))).rejects.toThrow( + /has no default export\.$/m, + ); + }); +}); + +describe("validateSource", () => { + const valid = { + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + }; + + test("accepts a minimal valid entry", () => { + expect(validateSource(valid, "f.ts").key).toBe("myplatform"); + }); + + // The old migration-tool's docs say nothing about this file, and an agent + // with only the CLI learns the format from wherever the error points. + test("points a rejection at how to write a source", () => { + try { + validateSource(null, "f.ts"); + throw new Error("expected a rejection"); + } catch (error) { + expect((error as CliError).docsUrl).toContain("README.md#custom-sources"); + } + }); + + test.each([ + ["a null default export", null, /is null, not an object/], + ["a number default export", 42, /is number, not an object/], + ["a string default export", "nope", /is string, not an object/], + ])("rejects %s", (_label, value, expected) => { + expect(() => validateSource(value, "f.ts")).toThrow(expected); + }); + + test.each([ + ["key", { ...valid, key: undefined }], + ["key", { ...valid, key: "" }], + ["key", { ...valid, key: " " }], + ["key", { ...valid, key: 7 }], + ["label", { ...valid, label: undefined }], + ["label", { ...valid, label: "" }], + ])("rejects a bad %s naming the field", (field, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test("rejects a non-string description", () => { + expect(() => validateSource({ ...valid, description: 7 }, "f.ts")).toThrow( + /`description` must be a string/, + ); + }); + + test.each([ + ["missing", { ...valid, transformer: undefined }], + ["null", { ...valid, transformer: null }], + ["an array", { ...valid, transformer: [] }], + ["a string", { ...valid, transformer: "id" }], + ])("rejects a transformer mapping that is %s", (_label, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(/`transformer`|`transformer\./); + }); + + test("names the offending entry when a mapping target is not a field name", () => { + expect(() => + validateSource({ ...valid, transformer: { account_ref: "userId", bad: 7 } }, "f.ts"), + ).toThrow(/`transformer.bad` must map to a Clerk field name, got number/); + }); + + // Without it the import runs to completion and creates every user with no + // external_id — which is what makes a migration reversible. + test("rejects a mapping with no userId target", () => { + expect(() => validateSource({ ...valid, transformer: { a: "email" } }, "f.ts")).toThrow( + /no source field maps to `userId`/, + ); + }); + + test.each([ + ["defaults", { ...valid, defaults: "nope" }], + ["preTransform", { ...valid, preTransform: "nope" }], + ["postTransform", { ...valid, postTransform: 7 }], + ])("rejects a %s of the wrong type", (field, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + // `migrate sources` wraps each caveat as text: a number crashed it, and a + // string printed one bullet per character. + test.each([ + ["a non-string caveat", [42]], + ["a string instead of a list", "text"], + ])("rejects caveats holding %s", (_label, caveats) => { + expect(() => validateSource({ ...valid, caveats }, "f.ts")).toThrow( + /`caveats` must be an array of strings/, + ); + }); + + test.each([["clerk"], ["auth0"], ["supabase"]])( + "rejects %s, which would shadow a built-in", + (key) => { + expect(() => + validateSource({ ...valid, key }, "f.ts", ["clerk", "auth0", "supabase"]), + ).toThrow(/already a built-in source/); + }, + ); + + // What a source brings across is the first thing `sources` shows, and only + // its author knows it. + test("requires carries", () => { + expect(() => validateSource({ ...valid, carries: undefined }, "f.ts")).toThrow( + /`carries` must say what the source brings across/, + ); + }); + + test.each([ + ["a level that is not yes, no or partial", { level: "maybe", note: "x" }], + ["no note", { level: "yes" }], + ])("rejects a carries entry with %s", (_label, mfa) => { + expect(() => validateSource({ ...valid, carries: { ...valid.carries, mfa } }, "f.ts")).toThrow( + /`carries.mfa` must be/, + ); + }); + + test("names the file in every message, so the author knows which one", () => { + expect(() => validateSource({}, "./their-file.ts")).toThrow(/\.\/their-file\.ts/); + }); + + test("raises CliError, so the global handler formats it", () => { + expect(() => validateSource({}, "f.ts")).toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts new file mode 100644 index 000000000..b9c6e3959 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -0,0 +1,207 @@ +/** + * Loading a user-authored source at runtime. + * + * In the standalone migration-tool, supporting a new platform meant adding a + * file to `src/transformers/` and one line to the registry — the user had the + * source tree. A compiled binary has neither a source tree to edit nor a way + * for an end user to rebuild it, so `--source ` restores that + * extensibility by importing a file from the user's own project instead. + * + * **Verified before this was built on:** a `bun build --compile` executable can + * `import()` an arbitrary external `.ts` file at runtime, including TypeScript + * that needs transpiling. Bun's transpiler is part of the runtime, not only the + * bundler. Confirmed with a throwaway compiled binary on darwin-arm64, + * linux-arm64 (glibc), linux-arm64-musl and linux-x64. + * + * The file is user-supplied code the CLI executes, so its shape is validated + * up front and rejected with a specific message rather than crashing deep in + * the transform pipeline on a missing field. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import type { CarryLevel, SourceEntry } from "../types.ts"; + +/** How to write a source. The README ships nowhere, so link the published copy. */ +export const CUSTOM_SOURCES_URL = + "https://github.com/clerk/cli/blob/main/packages/cli-core/src/commands/migrate/README.md#custom-sources"; + +const LEVELS: readonly CarryLevel[] = ["yes", "no", "partial"]; + +function invalid(problem: string, file: string): never { + throwUsageError(`${file} is not a valid source: ${problem}`, CUSTOM_SOURCES_URL); +} + +/** + * Checks a loaded value against the registry entry shape. + * + * Every failure names the specific field and what was wrong with it — the + * author is writing this file by hand against a shape they cannot see. + * + * @param file - Path as the user typed it, for the error message. + * @param reservedKeys - The built-in keys, which a custom source may not reuse. + */ +export function validateSource( + value: unknown, + file: string, + reservedKeys: readonly string[] = [], +): SourceEntry { + if (value === null || typeof value !== "object") { + invalid(`the default export is ${value === null ? "null" : typeof value}, not an object`, file); + } + + const entry = value as Record; + + for (const field of ["key", "label"] as const) { + if (typeof entry[field] !== "string" || entry[field].trim().length === 0) { + invalid(`\`${field}\` must be a non-empty string`, file); + } + } + + if (entry.description !== undefined && typeof entry.description !== "string") { + invalid("`description` must be a string when present", file); + } + + // Arrays are objects, and an author who wrote `transformer: []` should hear + // that rather than the downstream "no field maps to userId". + if ( + entry.transformer === null || + typeof entry.transformer !== "object" || + Array.isArray(entry.transformer) + ) { + invalid("`transformer` must be an object mapping source fields to Clerk fields", file); + } + + const mapping = entry.transformer as Record; + for (const [source, target] of Object.entries(mapping)) { + if (typeof target !== "string" || target.length === 0) { + invalid( + `\`transformer.${source}\` must map to a Clerk field name, got ${typeof target}`, + file, + ); + } + } + + // Without this the import runs to completion and creates every user with no + // external_id, which is what makes a migration re-runnable and reversible. + if (!Object.values(mapping).includes("userId")) { + invalid( + "no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id", + file, + ); + } + + if ( + entry.defaults !== undefined && + (entry.defaults === null || typeof entry.defaults !== "object" || Array.isArray(entry.defaults)) + ) { + invalid("`defaults` must be an object when present", file); + } + + // What a source brings across is the first thing `sources` shows, and the + // author is the only one who knows it. + const carries = entry.carries as Record | undefined; + if (!carries || typeof carries !== "object" || Array.isArray(carries)) { + invalid( + "`carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note }", + file, + ); + } + for (const kind of ["passwords", "mfa", "metadata"] as const) { + const carry = carries[kind] as { level?: unknown; note?: unknown } | undefined; + if (!carry || !LEVELS.includes(carry.level as CarryLevel) || typeof carry.note !== "string") { + invalid( + `\`carries.${kind}\` must be { level: "yes" | "no" | "partial", note: string }`, + file, + ); + } + } + + for (const hook of ["preTransform", "postTransform"] as const) { + if (entry[hook] !== undefined && typeof entry[hook] !== "function") { + invalid(`\`${hook}\` must be a function when present`, file); + } + } + + if ( + entry.caveats !== undefined && + (!Array.isArray(entry.caveats) || entry.caveats.some((caveat) => typeof caveat !== "string")) + ) { + invalid("`caveats` must be an array of strings when present", file); + } + + if (reservedKeys.includes(entry.key as string)) { + invalid( + `\`key\` is "${String(entry.key)}", which is already a built-in source. Choose another key`, + file, + ); + } + + return { + ...(entry as unknown as SourceEntry), + description: (entry.description as string | undefined) ?? "Custom source", + }; +} + +/** + * Imports and validates a user-authored source. + * + * @throws CliError when the path is missing, the module fails to load, or the + * exported value does not match the registry entry shape. + */ +export async function loadCustomSource( + file: string, + reservedKeys: readonly string[] = [], + /** + * The file's content hash. The module loader caches by URL, so an edited + * file at the same path would load the old module; the hash in the URL + * loads the version that was hashed. + */ + version?: string, +): Promise { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No source file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: CUSTOM_SOURCES_URL, + }); + } + if (fs.statSync(resolved).isDirectory()) { + throwUsageError(`${resolved} is a directory, not a source file.`); + } + + let module: Record; + try { + // A file URL rather than a bare path: an absolute POSIX path happens to + // work, but a Windows path (`C:\...`) is not a valid import specifier. + // Bun keys its module cache by specifier and keeps a `file://` URL's + // module whatever its query, but a plain path's query loads afresh. + // ponytail: Windows keeps the URL form, so an edit loaded twice in one + // process reuses the first; each CLI run is a process of its own. + const specifier = + version && process.platform !== "win32" + ? `${resolved}?v=${version}` + : Bun.pathToFileURL(resolved).href; + module = (await import(specifier)) as Record; + } catch (error) { + throwUsageError( + `Could not load ${file}: ${(error as Error).message}\n` + + "The file must be valid JavaScript or TypeScript that this CLI can import.", + CUSTOM_SOURCES_URL, + ); + } + + if (module.default === undefined) { + // Point at what they probably meant rather than just restating the rule. + const named = Object.keys(module).filter((key) => key !== "default"); + const hint = + named.length > 0 + ? ` Found named export${named.length === 1 ? "" : "s"} ${named.map((n) => `\`${n}\``).join(", ")} — did you mean \`export default\`?` + : ""; + throwUsageError(`${file} has no default export.${hint}`, CUSTOM_SOURCES_URL); + } + + return validateSource(module.default, file, reservedKeys); +} diff --git a/packages/cli-core/src/commands/migrate/sources/registry.ts b/packages/cli-core/src/commands/migrate/sources/registry.ts index 25d376e71..6612eb612 100644 --- a/packages/cli-core/src/commands/migrate/sources/registry.ts +++ b/packages/cli-core/src/commands/migrate/sources/registry.ts @@ -1,12 +1,16 @@ /** * Source registry. * - * `migrate import` resolves `--source` here. + * `migrate import` resolves `--source` here, and `migrate sources` lists what + * each one brings across. * * To add a platform: create `sources/.ts` exporting a `SourceEntry`, * then add it to the array below. */ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; import { throwUsageError } from "../../../lib/errors.ts"; import type { SourceEntry } from "../types.ts"; import auth0Source from "./auth0.ts"; @@ -14,6 +18,7 @@ import authjsSource from "./authjs.ts"; import betterAuthSource from "./betterauth.ts"; import clerkSource from "./clerk.ts"; import firebaseSource from "./firebase.ts"; +import { loadCustomSource } from "./load-custom.ts"; import supabaseSource from "./supabase.ts"; import workosSource from "./workos.ts"; @@ -41,36 +46,91 @@ export const ACCOUNT_LINKING_NOTE = "Social sign-ins are not copied. Enable the same providers in Clerk, and a user who signs in " + `with one is linked to their imported account by verified email. See ${ACCOUNT_LINKING_URL}`; +/** + * A source loaded from a user's `--source ` for this invocation. + * + * Kept beside the built-ins rather than pushed into them, so the shipped list + * is never mutated. One invocation loads at most one, so this holding a + * single entry is the normal case; the array shape just avoids a special case + * in the lookups. + */ +const customSources: SourceEntry[] = []; + +export function registerCustomSource(entry: SourceEntry): void { + // The latest load of a key wins: an edited source replaces its old self. + const earlier = customSources.findIndex((source) => source.key === entry.key); + if (earlier >= 0) customSources.splice(earlier, 1); + customSources.push(entry); +} + +/** Test-only: drops anything a previous test registered. */ +export function __resetCustomSourcesForTesting(): void { + customSources.length = 0; +} + +/** Built-ins plus whatever `--source ` loaded. */ +export function allSources(): SourceEntry[] { + return [...sources, ...customSources]; +} + /** The built-in keys, for tab-completion and error messages. */ export function sourceKeys(): string[] { return sources.map((entry) => entry.key); } /** - * Looks up a source by key. + * Looks up a source by key, custom ones included. * * @throws Error when no source is registered under that key. */ export function getSource(key: string): SourceEntry { - const source = sources.find((entry) => entry.key === key); + const source = allSources().find((entry) => entry.key === key); if (!source) { throw new Error(`Source not found for key: ${key}`); } return source; } -/** A resolved `--source`. */ -export type ResolvedSource = { key: string; entry: SourceEntry }; +/** A `--source` value that names a file rather than a built-in key. */ +export function isSourcePath(value: string): boolean { + return /^(\.\.?\/|\/)/.test(value) || /\.(ts|js|mjs)$/.test(value); +} + +/** A resolved `--source`: its key, and for a custom one, the file's content hash. */ +export type ResolvedSource = { key: string; entry: SourceEntry; path?: string; hash?: string }; /** - * Resolves `--source` to a built-in source. + * Resolves `--source`: a built-in key, or a path to a source you wrote. + * + * A custom source is keyed by a hash of its file as well as its key, so an + * edited source is a different source when a re-run decides whether to + * continue an earlier import. * * @throws UsageError for an unknown key, listing the valid ones. */ export async function resolveSource(value: string): Promise { + if (isSourcePath(value)) { + const resolved = path.resolve(process.cwd(), value); + // Hashed first, and loaded by that hash: the record names the very code + // that maps the users, even if the file was edited since a last load. + // Only a file: loadCustomSource reports a missing path or a directory. + const hash = fs.statSync(resolved, { throwIfNoEntry: false })?.isFile() + ? createHash("sha256").update(fs.readFileSync(resolved)).digest("hex") + : undefined; + const entry = await loadCustomSource(value, sourceKeys(), hash); + registerCustomSource(entry); + return { key: entry.key, entry, path: resolved, hash }; + } + const entry = sources.find((candidate) => candidate.key === value); if (!entry) { - throwUsageError(`Unknown source "${value}". Valid sources: ${sourceKeys().join(", ")}.`); + throwUsageError( + `Unknown source "${value}". Valid sources: ${sourceKeys().join(", ")}.\n` + + "For a platform with no built-in, pass the path to a source you wrote, e.g. --source ./my-source.ts.", + undefined, + undefined, + [{ command: "clerk migrate sources", description: "List the built-in sources" }], + ); } return { key: entry.key, entry }; } diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index e3ad5c39d..1a4be8998 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -5,8 +5,8 @@ import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; import { loadUsersFromFile, normalizeUserData, transformUsers } from "../lib/transform.ts"; import type { FirebaseHashConfig, User } from "../types.ts"; -import { getSource, sourceKeys, sources } from "./registry.ts"; import { isAuthJsVerified } from "./authjs.ts"; +import { getSource, isSourcePath, sourceKeys, sources } from "./registry.ts"; import { isVerified, toIsoDate } from "./shared.ts"; const FIREBASE_HASH: FirebaseHashConfig = { @@ -79,6 +79,19 @@ describe("registry", () => { }); }); +describe("isSourcePath", () => { + test.each([["./mine.ts"], ["../up/mine.js"], ["/abs/mine.mjs"], ["mine.ts"], ["dir/mine.js"]])( + "%s is a path", + (value) => { + expect(isSourcePath(value)).toBe(true); + }, + ); + + test.each([["clerk"], ["betterauth"], ["okta"]])("%s is a key", (value) => { + expect(isSourcePath(value)).toBe(false); + }); +}); + describe("isVerified", () => { // A CSV export stringifies everything, so the boolean style must read // "false" as false. Treating it as truthy would mark unconfirmed addresses diff --git a/packages/cli-core/src/test/integration/completion.test.ts b/packages/cli-core/src/test/integration/completion.test.ts index 822704218..d27b152d9 100644 --- a/packages/cli-core/src/test/integration/completion.test.ts +++ b/packages/cli-core/src/test/integration/completion.test.ts @@ -239,12 +239,10 @@ describe("generateCompletions", () => { const saved = process.env.CLERK_EXPERIMENTAL; process.env.CLERK_EXPERIMENTAL = "migrate"; try { - const names = generateCompletions(createProgram(), [ - "migrate", - "import", - "--source", - "", - ]).completions.map((c) => c.name); + const result = generateCompletions(createProgram(), ["migrate", "import", "--source", ""]); + // A source you wrote is a path, so the shell's file completion stays on. + expect(result.directive).toBe(0); + const names = result.completions.map((c) => c.name); expect(names).toEqual([ "clerk", "auth0", @@ -334,6 +332,21 @@ describe("generateCompletions", () => { expect(names).toContain("settings"); }); + test("migrate sources: suggests the built-in sources", () => { + const saved = process.env.CLERK_EXPERIMENTAL; + process.env.CLERK_EXPERIMENTAL = "migrate"; + try { + const result = generateCompletions(createProgram(), ["migrate", "sources", ""]); + expect(result.directive).toBe(0); + const names = result.completions.map((c) => c.name); + expect(names).toContain("supabase"); + expect(names).toContain("workos"); + } finally { + if (saved === undefined) delete process.env.CLERK_EXPERIMENTAL; + else process.env.CLERK_EXPERIMENTAL = saved; + } + }); + test("open dashboard: filters subpaths by prefix", () => { const names = completionNames("open", "dashboard", "u"); expect(names).toContain("users");