diff --git a/src/api/deploy.test.ts b/src/api/deploy.test.ts index 75722eb..6f8eae9 100644 --- a/src/api/deploy.test.ts +++ b/src/api/deploy.test.ts @@ -1,19 +1,28 @@ +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; import { describe, it, expect, beforeAll, beforeEach, afterEach, afterAll, vi } from "vitest"; import { setupServer } from "msw/node"; import { http, HttpResponse } from "msw"; import { deployToMain, MAX_CONSECUTIVE_FAILED_POLLS } from "./deploy.js"; import type { BuildConfig } from "./build.js"; -import { - BASE_URL, - createDeploySuccessResponse, - createDeploymentStatusResponse, - createBuildFailureResponse, - createBuildMultipleErrorsResponse, - createDeploymentsListResponse, -} from "../test/handlers.js"; +import { BASE_URL, createDeploymentStatusResponse, createDeploymentsListResponse } from "../test/handlers.js"; import type { GeneratedResources } from "../generator/index.js"; +const __dirname = dirname(fileURLToPath(import.meta.url)); +const FAKE_TB = join(__dirname, "..", "test", "fixtures", "fake-tb-cli.mjs"); + +// deployToMain now shells out to the `tb` binary (via src/cli-binary/), so +// these tests drive it through the fake-tb-cli.mjs fixture instead of +// mocking a POST /v1/deploy. MSW still covers the two HTTP calls deployToMain +// makes directly: stale-deployment cleanup (/v1/deployments) and status +// polling (/v1/deployments/:id) — see the file header comment in deploy.ts +// for why those stayed local instead of moving into the binary call. +// +// Skipped on Windows: the fixture is a POSIX shebang script spawned directly +// (same constraint as src/cli-binary/run.test.ts). +const describePosix = process.platform === "win32" ? describe.skip : describe; + const server = setupServer(); beforeAll(() => { @@ -23,7 +32,7 @@ beforeAll(() => { }); }); beforeEach(() => { - // Set up default handler for deployments list (used by stale deployment cleanup) + // Default handler for the stale-deployment cleanup list call. server.use( http.get(`${BASE_URL}/v1/deployments`, () => { return HttpResponse.json(createDeploymentsListResponse()); @@ -36,39 +45,47 @@ afterAll(() => { vi.unstubAllGlobals(); }); -describe("Deploy API", () => { +describePosix("Deploy API (via tb binary)", () => { const config: BuildConfig = { baseUrl: BASE_URL, token: "p.test-token", }; const resources: GeneratedResources = { - datasources: [ - { name: "events", content: "SCHEMA > timestamp DateTime" }, - ], - pipes: [ - { name: "top_events", content: "NODE main\nSQL > SELECT * FROM events" }, - ], + datasources: [{ name: "events", content: "SCHEMA > timestamp DateTime" }], + pipes: [{ name: "top_events", content: "NODE main\nSQL > SELECT * FROM events" }], connections: [], }; - // Helper to set up successful deploy flow. By default the deployment shows - // up as live on the very first status poll, which matches the server-side - // auto_promote=true behavior the SDK now relies on. + const originalBinaryPath = process.env.TB_CLI_BINARY_PATH; + + beforeEach(() => { + process.env.TB_CLI_BINARY_PATH = FAKE_TB; + }); + + afterEach(() => { + if (originalBinaryPath === undefined) delete process.env.TB_CLI_BINARY_PATH; + else process.env.TB_CLI_BINARY_PATH = originalBinaryPath; + delete process.env.FAKE_TB_STDOUT; + delete process.env.FAKE_TB_STDERR; + delete process.env.FAKE_TB_EXIT_CODE; + delete process.env.FAKE_TB_ECHO_ARGV; + }); + + /** Sets the fake binary to emit a successful `deployment create` JSON. */ + function setFakeTbDeploymentJson(fields: Record) { + process.env.FAKE_TB_STDOUT = JSON.stringify(fields); + process.env.FAKE_TB_EXIT_CODE = "0"; + } + + // Helper to set up a successful deploy flow. By default the deployment + // shows up as live on the very first status poll, matching auto_promote. function setupSuccessfulDeployFlow(deploymentId = "deploy-abc") { + setFakeTbDeploymentJson({ id: deploymentId, status: "pending" }); server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId, status: "pending" }) - ); - }), http.get(`${BASE_URL}/v1/deployments/${deploymentId}`, () => { return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId, - status: "data_ready", - live: true, - }) + createDeploymentStatusResponse({ deploymentId, status: "data_ready", live: true }) ); }) ); @@ -89,40 +106,24 @@ describe("Deploy API", () => { it("polls until deployment is ready and live", async () => { let pollCount = 0; + setFakeTbDeploymentJson({ id: "deploy-poll", status: "pending" }); server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "deploy-poll", status: "pending" }) - ); - }), http.get(`${BASE_URL}/v1/deployments/deploy-poll`, () => { pollCount++; // pending → data_ready (not live yet) → data_ready + live. if (pollCount < 3) { return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-poll", - status: "pending", - live: false, - }) + createDeploymentStatusResponse({ deploymentId: "deploy-poll", status: "pending", live: false }) ); } if (pollCount === 3) { return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-poll", - status: "data_ready", - live: false, - }) + createDeploymentStatusResponse({ deploymentId: "deploy-poll", status: "data_ready", live: false }) ); } return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-poll", - status: "data_ready", - live: true, - }) + createDeploymentStatusResponse({ deploymentId: "deploy-poll", status: "data_ready", live: true }) ); }) ); @@ -134,14 +135,12 @@ describe("Deploy API", () => { }); it("handles deploy failure with single error", async () => { - server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return HttpResponse.json( - createBuildFailureResponse("Permission denied"), - { status: 200 } - ); - }) - ); + process.env.FAKE_TB_STDOUT = JSON.stringify({ + id: "deploy-fail", + status: "failed", + errors: [{ error: "Permission denied" }], + }); + process.env.FAKE_TB_EXIT_CODE = "1"; const result = await deployToMain(config, resources); @@ -151,17 +150,15 @@ describe("Deploy API", () => { }); it("handles deploy failure with multiple errors", async () => { - server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return HttpResponse.json( - createBuildMultipleErrorsResponse([ - { filename: "events.datasource", error: "Schema mismatch" }, - { error: "General error without filename" }, - ]), - { status: 200 } - ); - }) - ); + process.env.FAKE_TB_STDOUT = JSON.stringify({ + id: "deploy-fail", + status: "failed", + errors: [ + { filename: "events.datasource", error: "Schema mismatch" }, + { error: "General error without filename" }, + ], + }); + process.env.FAKE_TB_EXIT_CODE = "1"; const result = await deployToMain(config, resources); @@ -171,21 +168,12 @@ describe("Deploy API", () => { }); it("handles deployment feedback entries with null resource", async () => { - server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return HttpResponse.json( - { - result: "failed", - deployment: { - id: "deploy-null-resource", - status: "failed", - feedback: [{ resource: null, level: "ERROR", message: "Invalid token" }], - }, - }, - { status: 200 } - ); - }) - ); + process.env.FAKE_TB_STDOUT = JSON.stringify({ + id: "deploy-null-resource", + status: "failed", + feedback: [{ resource: null, level: "ERROR", message: "Invalid token" }], + }); + process.env.FAKE_TB_EXIT_CODE = "1"; const result = await deployToMain(config, resources); @@ -194,192 +182,78 @@ describe("Deploy API", () => { expect(result.error).toContain("unknown: Invalid token"); }); - it("handles HTTP error responses", async () => { - server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return HttpResponse.json( - { result: "failed", error: "Forbidden" }, - { status: 403 } - ); - }) - ); + it("returns a failed result (not a throw) when the binary fails with no JSON output", async () => { + process.env.FAKE_TB_STDERR = "boom: unexpected error"; + process.env.FAKE_TB_EXIT_CODE = "1"; const result = await deployToMain(config, resources); expect(result.success).toBe(false); - expect(result.error).toBe("Forbidden"); - }); - - it("handles malformed JSON response", async () => { - server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return new HttpResponse("invalid json {", { - status: 200, - headers: { "Content-Type": "text/plain" }, - }); - }) - ); - - await expect(deployToMain(config, resources)).rejects.toThrow( - "Failed to parse response" - ); + expect(result.error).toContain("boom: unexpected error"); }); - it("uses /v1/deploy endpoint with auto_promote by default", async () => { - let capturedUrl: string | null = null; - - server.use( - http.post(`${BASE_URL}/v1/deploy`, ({ request }) => { - capturedUrl = request.url; - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "deploy-url-test" }) - ); - }), - http.get(`${BASE_URL}/v1/deployments/deploy-url-test`, () => { - return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-url-test", - status: "data_ready", - live: true, - }) - ); - }) - ); - - await deployToMain(config, resources); + it("returns no_changes (not a throw) when the binary exits 0 with no JSON output", async () => { + // FAKE_TB_STDOUT/FAKE_TB_EXIT_CODE unset -> empty stdout, exit 0. + const result = await deployToMain(config, resources); - const parsed = new URL(capturedUrl ?? ""); - expect(parsed.pathname).toBe("/v1/deploy"); - expect(parsed.searchParams.get("from")).toBe("ts-sdk"); - expect(parsed.searchParams.get("auto_promote")).toBe("true"); + expect(result).toMatchObject({ success: true, result: "no_changes" }); }); - it("omits auto_promote when auto is false", async () => { - let capturedUrl: string | null = null; - + it("omits live-wait when auto is false (data_ready alone is enough)", async () => { + setFakeTbDeploymentJson({ id: "deploy-no-auto", status: "pending" }); server.use( - http.post(`${BASE_URL}/v1/deploy`, ({ request }) => { - capturedUrl = request.url; - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "deploy-no-auto" }) - ); - }), http.get(`${BASE_URL}/v1/deployments/deploy-no-auto`, () => { return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-no-auto", - status: "data_ready", - live: false, - }) + createDeploymentStatusResponse({ deploymentId: "deploy-no-auto", status: "data_ready", live: false }) ); }) ); const result = await deployToMain(config, resources, { auto: false }); - const parsed = new URL(capturedUrl ?? ""); - expect(parsed.searchParams.get("auto_promote")).toBeNull(); - // When !auto, we return success as soon as the deployment is data_ready, - // even though `live` is still false (user must promote it separately). + // When !auto, we return success as soon as the deployment is + // data_ready, even though `live` is still false (user promotes it separately). expect(result.success).toBe(true); expect(result.buildId).toBe("deploy-no-auto"); }); - it("returns immediately when wait is false", async () => { - let statusPolls = 0; - - server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "deploy-no-wait" }) - ); - }), - http.get(`${BASE_URL}/v1/deployments/deploy-no-wait`, () => { - statusPolls++; - return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-no-wait", - status: "pending", - live: false, - }) - ); - }) - ); + it("returns immediately when wait is false, without polling", async () => { + setFakeTbDeploymentJson({ id: "deploy-no-wait", status: "pending" }); + // Deliberately no /v1/deployments/deploy-no-wait handler: with + // onUnhandledRequest: "error", any poll attempt fails the test. const result = await deployToMain(config, resources, { wait: false }); expect(result.success).toBe(true); expect(result.buildId).toBe("deploy-no-wait"); - // No polling should have happened. - expect(statusPolls).toBe(0); - }); - - it("passes allow_destructive_operations when explicitly enabled", async () => { - let capturedUrl: string | null = null; - - server.use( - http.post(`${BASE_URL}/v1/deploy`, ({ request }) => { - capturedUrl = request.url; - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "deploy-destructive" }) - ); - }), - http.get(`${BASE_URL}/v1/deployments/deploy-destructive`, () => { - return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-destructive", - status: "data_ready", - live: true, - }) - ); - }) - ); - - await deployToMain(config, resources, { - allowDestructiveOperations: true, - }); - - const parsed = new URL(capturedUrl ?? ""); - expect(parsed.searchParams.get("allow_destructive_operations")).toBe("true"); }); it("skips stale deployment cleanup in check mode", async () => { let listed = false; const deletedIds: string[] = []; - let capturedUrl: string | null = null; server.use( http.get(`${BASE_URL}/v1/deployments`, () => { listed = true; return HttpResponse.json( - createDeploymentsListResponse({ - deployments: [ - { id: "in-flight", status: "pending", live: false }, - ], - }) + createDeploymentsListResponse({ deployments: [{ id: "in-flight", status: "pending", live: false }] }) ); }), http.delete(`${BASE_URL}/v1/deployments/:id`, ({ params }) => { deletedIds.push(params.id as string); return HttpResponse.json({ result: "success" }); - }), - http.post(`${BASE_URL}/v1/deploy`, ({ request }) => { - capturedUrl = request.url; - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "deploy-check" }) - ); }) ); + setFakeTbDeploymentJson({ id: "deploy-check", status: "success" }); - await deployToMain(config, resources, { check: true }); + const result = await deployToMain(config, resources, { check: true }); expect(listed).toBe(false); expect(deletedIds).toEqual([]); - const parsed = new URL(capturedUrl ?? ""); - expect(parsed.searchParams.get("check")).toBe("true"); + expect(result).toMatchObject({ success: true, result: "success" }); }); - it("deletes stale non-live deployments before deploy and previous live deployment after promotion", async () => { + it("deletes stale non-live deployments before deploy", async () => { const deletedIds: string[] = []; server.use( @@ -408,62 +282,18 @@ describe("Deploy API", () => { expect(deletedIds).toEqual(["stale-1", "stale-2"]); }); - it("does not touch the previous live deployment client-side", async () => { - // With auto_promote=true the server flips the new deployment live AND - // deletes the previous live deployment on our behalf. The SDK should - // therefore never issue a set-live or a delete for a live deployment. - const events: string[] = []; - - server.use( - http.get(`${BASE_URL}/v1/deployments`, () => { - return HttpResponse.json( - createDeploymentsListResponse({ - deployments: [ - { id: "previous-live", status: "live", live: true }, - ], - }) - ); - }), - http.post(`${BASE_URL}/v1/deploy`, () => { - events.push("create"); - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "new-deploy", status: "pending" }) - ); - }), - http.get(`${BASE_URL}/v1/deployments/new-deploy`, () => { - return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "new-deploy", - status: "data_ready", - live: true, - }) - ); - }), - http.delete(`${BASE_URL}/v1/deployments/:id`, ({ params }) => { - events.push(`delete:${params.id as string}`); - return HttpResponse.json({ result: "success" }); - }) - ); - - const result = await deployToMain(config, resources); - - expect(result.success).toBe(true); - expect(events).toEqual(["create"]); - }); - it("adds actionable guidance to Forward/Classic workspace errors", async () => { - server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return HttpResponse.json( - { - result: "failed", - error: - "This is a Tinybird Forward workspace, and this operation is only available for Tinybird Classic workspaces.", - }, - { status: 400 } - ); - }) - ); + process.env.FAKE_TB_STDOUT = JSON.stringify({ + id: "deploy-classic", + status: "failed", + errors: [ + { + error: + "This is a Tinybird Forward workspace, and this operation is only available for Tinybird Classic workspaces.", + }, + ], + }); + process.env.FAKE_TB_EXIT_CODE = "1"; const result = await deployToMain(config, resources); @@ -475,25 +305,15 @@ describe("Deploy API", () => { }); it("tolerates transient failed status while server auto-deletes", async () => { - // When the deployment hits `failed`, the SDK should not bail immediately - // — the server usually transitions it to `deleting`/`deleted` shortly - // after. We report the failure only when that transition happens. let pollCount = 0; + setFakeTbDeploymentJson({ id: "deploy-transient-fail", status: "pending" }); + server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "deploy-transient-fail" }) - ); - }), http.get(`${BASE_URL}/v1/deployments/deploy-transient-fail`, () => { pollCount++; if (pollCount <= 2) { return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-transient-fail", - status: "failed", - live: false, - }) + createDeploymentStatusResponse({ deploymentId: "deploy-transient-fail", status: "failed", live: false }) ); } return HttpResponse.json({ @@ -502,9 +322,7 @@ describe("Deploy API", () => { id: "deploy-transient-fail", status: "deleted", live: false, - feedback: [ - { resource: null, level: "ERROR", message: "schema conflict" }, - ], + feedback: [{ resource: null, level: "ERROR", message: "schema conflict" }], }, }); }) @@ -515,26 +333,18 @@ describe("Deploy API", () => { expect(result.success).toBe(false); expect(result.error).toContain("deleted automatically"); expect(result.error).toContain("schema conflict"); - // 2 failed + 1 deleted. expect(pollCount).toBe(3); }); it("bails when the deployment is stuck in failed state", async () => { let pollCount = 0; + setFakeTbDeploymentJson({ id: "deploy-stuck", status: "pending" }); + server.use( - http.post(`${BASE_URL}/v1/deploy`, () => { - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "deploy-stuck" }) - ); - }), http.get(`${BASE_URL}/v1/deployments/deploy-stuck`, () => { pollCount++; return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-stuck", - status: "failed", - live: false, - }) + createDeploymentStatusResponse({ deploymentId: "deploy-stuck", status: "failed", live: false }) ); }) ); @@ -543,77 +353,67 @@ describe("Deploy API", () => { expect(result.success).toBe(false); expect(result.error).toContain("didn't start deleting automatically"); - // One extra poll past the threshold trips the safety valve. expect(pollCount).toBe(MAX_CONSECUTIVE_FAILED_POLLS + 1); }); - it("normalizes baseUrl with trailing slash", async () => { - let capturedUrl: string | null = null; + it("surfaces a clear error when the binary can't be resolved", async () => { + delete process.env.TB_CLI_BINARY_PATH; // force manifest resolution, which will fail offline + process.env.TB_CLI_BASE_URL = "https://does-not-exist.invalid"; - server.use( - http.post(`${BASE_URL}/v1/deploy`, ({ request }) => { - capturedUrl = request.url; - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "deploy-slash" }) - ); - }), - http.get(`${BASE_URL}/v1/deployments/deploy-slash`, () => { - return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-slash", - status: "data_ready", - live: true, - }) - ); - }) - ); + const result = await deployToMain(config, resources); - await deployToMain({ ...config, baseUrl: `${BASE_URL}/` }, resources); + expect(result.success).toBe(false); + expect(result.result).toBe("failed"); + expect(result.error).toBeTruthy(); - const parsed = new URL(capturedUrl ?? ""); - expect(parsed.pathname).toBe("/v1/deploy"); - expect(parsed.searchParams.get("from")).toBe("ts-sdk"); + delete process.env.TB_CLI_BASE_URL; }); + }); + + describe("materialized project contents", () => { + it("writes datasources, pipes, and connections for tb to read, and passes the right flags", async () => { + process.env.FAKE_TB_ECHO_ARGV = "1"; - it("includes connections in deploy form data", async () => { const resourcesWithConnections: GeneratedResources = { - ...resources, - connections: [ - { - name: "my_kafka", - content: "TYPE kafka\nKAFKA_BROKERS kafka:9092\nKAFKA_TOPIC events\n", - }, - ], + datasources: [{ name: "events", content: "SCHEMA > timestamp DateTime" }], + pipes: [{ name: "top_events", content: "NODE main\nSQL > SELECT * FROM events" }], + connections: [{ name: "my_kafka", content: "TYPE kafka\nKAFKA_BROKERS kafka:9092\n" }], }; - let capturedFormData: FormData | null = null; - - server.use( - http.post(`${BASE_URL}/v1/deploy`, async ({ request }) => { - capturedFormData = await request.formData(); - return HttpResponse.json( - createDeploySuccessResponse({ deploymentId: "deploy-conn", status: "pending" }) - ); - }), - http.get(`${BASE_URL}/v1/deployments/deploy-conn`, () => { - return HttpResponse.json( - createDeploymentStatusResponse({ - deploymentId: "deploy-conn", - status: "data_ready", - live: true, - }) - ); - }) - ); - - const result = await deployToMain(config, resourcesWithConnections); - - expect(result.success).toBe(true); - expect(result.connectionCount).toBe(1); - expect(capturedFormData).not.toBeNull(); - // 1 datasource + 1 pipe + 1 connection - const allValues = capturedFormData!.getAll("data_project://"); - expect(allValues.length).toBe(3); + // Capture stdout directly via runTbBinary's own plumbing by reusing + // deployToMain, then re-deriving what was echoed from its (otherwise + // discarded) result isn't possible — so assert via a direct call to + // the same building blocks deployToMain uses. + const { materializeResourcesToTempProject } = await import("../cli-binary/resources-to-project.js"); + const { runTbBinary } = await import("../cli-binary/run.js"); + + const project = await materializeResourcesToTempProject(resourcesWithConnections); + try { + const run = await runTbBinary( + FAKE_TB, + ["deployment", "create", "--no-wait", "--auto", "--allow-destructive-operations"], + { cwd: project.dir, token: config.token, host: config.baseUrl, env: { FAKE_TB_ECHO_ARGV: "1" } } + ); + const parsed = JSON.parse(run.stdout); + + expect(parsed.argv).toEqual([ + "--output", + "json", + "deployment", + "create", + "--no-wait", + "--auto", + "--allow-destructive-operations", + ]); + expect(parsed.projectFiles).toEqual([ + "connections/my_kafka.connection", + "datasources/events.datasource", + "pipes/top_events.pipe", + ]); + expect(parsed.env.TB_TOKEN).toBe(config.token); + } finally { + await project.cleanup(); + } }); }); }); diff --git a/src/api/deploy.ts b/src/api/deploy.ts index ee577c5..5ee8161 100644 --- a/src/api/deploy.ts +++ b/src/api/deploy.ts @@ -1,11 +1,35 @@ /** - * Deploy resources to Tinybird main workspace - * Uses the /v1/deploy endpoint to create a deployment, then sets it live + * Deploy resources to Tinybird main workspace via the `tb` CLI binary + * (https://gitlab.com/tinybird/analytics/-/merge_requests/23765, PROTM-471). + * + * This used to POST directly to /v1/deploy. It now delegates deployment + * *creation* to the downloaded `tb` binary, which hits the exact same + * endpoint with the exact same multipart layout (verified against + * tinybird/tb/modules/deployment_common.py), so the heavy lifting lives in + * one place instead of being reimplemented here. Two things stay local: + * - Stale-deployment cleanup (Step 0 below): `create_deployment()` on the + * Python side has no equivalent, so this is purely an SDK-side workaround + * and still applies regardless of which client calls /v1/deploy. + * - Status polling (Step 2 below): the binary's own --wait mode would block + * without giving us anything to drive progress callbacks from, so + * deployment creation is always submitted with --no-wait and polled here. + * + * No HTTP fallback: if the binary can't be resolved (unsupported platform, + * no network, checksum mismatch), this returns a clear, specific failure + * instead of silently falling back to a direct POST. + * + * DRAFT: there is no published `tb` binary yet (the MR above hasn't + * merged), so this has only been exercised against test fakes, not a real + * binary. See the two VERIFY comments below for the specific things to + * re-check once one exists. */ import type { GeneratedResources } from "../generator/index.js"; import type { BuildConfig, BuildApiResult } from "./build.js"; import { tinybirdFetch } from "./fetcher.js"; +import { materializeResourcesToTempProject } from "../cli-binary/resources-to-project.js"; +import { CliBinaryUnavailableError, resolveTbBinaryPath } from "../cli-binary/index.js"; +import { runTbBinary } from "../cli-binary/run.js"; const FORWARD_CLASSIC_GUIDANCE = "Use the Tinybird Classic CLI (`tb`) from a Tinybird Classic workspace for this operation."; @@ -53,6 +77,11 @@ export interface DeploymentsListResponse { /** * Response from /v1/deploy endpoint + * + * @deprecated no longer produced internally (deployment creation goes + * through the `tb` binary now, which emits a `DeploymentDetails` object + * directly, not this `{result, deployment}` envelope). Kept exported for + * backwards compatibility with existing imports. */ export interface DeployResponse { result: "success" | "failed" | "no_changes"; @@ -95,41 +124,6 @@ export interface DeploymentStatusResponse { deployment: Deployment; } -/** - * Deploy generated resources to Tinybird main workspace - * - * Uses the /v1/deploy endpoint which accepts all resources in a single - * multipart form request. After creating the deployment, this function: - * 1. When `auto` is true (default), sends `auto_promote=true` so the - * server promotes the deployment as soon as it's ready. - * 2. When `wait` is true (default), polls until the deployment reaches - * `data_ready` (and, if `auto` is also true, until it becomes live). - * - * @param config - Build configuration with API URL and token - * @param resources - Generated resources to deploy - * @returns Build result - * - * @example - * ```ts - * const result = await deployToMain( - * { - * baseUrl: 'https://api.tinybird.co', - * token: 'p.xxx', - * }, - * { - * datasources: [{ name: 'events', content: '...' }], - * pipes: [{ name: 'top_events', content: '...' }], - * } - * ); - * - * if (result.success) { - * console.log('Deployed to main workspace!'); - * } - * ``` - */ -/** - * Callbacks for deploy progress updates - */ /** * Resource changes from deployment */ @@ -151,6 +145,9 @@ export interface DeploymentChanges { }; } +/** + * Callbacks for deploy progress updates + */ export interface DeployCallbacks { /** Called when deployment is created and changes are available */ onChanges?: (changes: DeploymentChanges) => void; @@ -170,6 +167,69 @@ export interface DeployCallbacks { onValidating?: () => void; } +function emptyChanges(): DeploymentChanges { + return { + datasources: { created: [], changed: [], deleted: [] }, + pipes: { created: [], changed: [], deleted: [] }, + connections: { created: [], changed: [], deleted: [] }, + }; +} + +function normalizeDeployErrorMessage(message: string): string { + const trimmedMessage = message.trim(); + const isForwardClassicError = + trimmedMessage.includes("Tinybird Forward workspace") && + trimmedMessage.includes("Tinybird Classic workspaces"); + + if (!isForwardClassicError || trimmedMessage.includes(FORWARD_CLASSIC_GUIDANCE)) { + return message; + } + + return `${trimmedMessage}\n${FORWARD_CLASSIC_GUIDANCE}`; +} + +function parseTbDeploymentJson(stdout: string): DeploymentDetails | null { + const trimmed = stdout.trim(); + if (!trimmed) return null; + try { + return JSON.parse(trimmed) as DeploymentDetails; + } catch { + return null; + } +} + +function formatTbErrors(parsed: DeploymentDetails, fallback: string): string { + if (parsed.errors && parsed.errors.length > 0) { + return parsed.errors + .map((e) => (e.filename ? `[${e.filename}] ${normalizeDeployErrorMessage(e.error)}` : normalizeDeployErrorMessage(e.error))) + .join("\n"); + } + if (parsed.feedback && parsed.feedback.length > 0) { + const errors = parsed.feedback.filter((f) => f.level === "ERROR"); + if (errors.length > 0) { + return errors + .map((f) => { + const resourceName = f.resource?.split(" ").pop() ?? f.resource ?? "unknown"; + return `${resourceName}: ${normalizeDeployErrorMessage(f.message)}`; + }) + .join("\n"); + } + } + return fallback; +} + +/** + * Deploy generated resources to Tinybird main workspace + * + * 1. Cleans up any stale (non-live) deployments that might block the new one. + * 2. Submits the deployment via `tb deployment create --no-wait --output json`. + * 3. When `wait` is true (default), polls /v1/deployments/{id} until the + * deployment reaches `data_ready` (and, if `auto` is also true, `live`). + * + * @param config - Deploy configuration with API URL and token + * @param resources - Generated resources to deploy + * @returns Deploy result + */ export async function deployToMain( config: BuildConfig, resources: GeneratedResources, @@ -187,62 +247,23 @@ export async function deployToMain( const auto = options?.auto ?? true; const baseUrl = config.baseUrl.replace(/\/$/, ""); - const formData = new FormData(); - - // Add datasources - for (const ds of resources.datasources) { - const fieldName = `data_project://`; - const fileName = `${ds.name}.datasource`; - if (debug) { - console.log(`[debug] Adding datasource: ${fieldName} (filename: ${fileName})`); - console.log(`[debug] Content:\n${ds.content}\n`); - } - formData.append( - fieldName, - new Blob([ds.content], { type: "text/plain" }), - fileName - ); - } - - // Add pipes - for (const pipe of resources.pipes) { - const fieldName = `data_project://`; - const fileName = `${pipe.name}.pipe`; - if (debug) { - console.log(`[debug] Adding pipe: ${fieldName} (filename: ${fileName})`); - console.log(`[debug] Content:\n${pipe.content}\n`); - } - formData.append( - fieldName, - new Blob([pipe.content], { type: "text/plain" }), - fileName - ); - } - - // Add connections - for (const conn of resources.connections ?? []) { - const fieldName = `data_project://`; - const fileName = `${conn.name}.connection`; - if (debug) { - console.log(`[debug] Adding connection: ${fieldName} (filename: ${fileName})`); - console.log(`[debug] Content:\n${conn.content}\n`); - } - formData.append( - fieldName, - new Blob([conn.content], { type: "text/plain" }), - fileName - ); - } + const counts = { + datasourceCount: resources.datasources.length, + pipeCount: resources.pipes.length, + connectionCount: resources.connections?.length ?? 0, + }; // Step 0: Clean up any stale non-live deployments that might block the new deployment. // Skipped in check mode so validation runs don't tear down in-flight deployments. + // The `tb` binary doesn't do this itself (verified against + // tinybird/tb/modules/deployment_common.py's create_deployment()) — it's an + // SDK-side workaround that applies regardless of which client creates the + // deployment, so it stays here rather than moving into the binary call. if (!options?.check) { try { const deploymentsUrl = `${baseUrl}/v1/deployments`; const deploymentsResponse = await tinybirdFetch(deploymentsUrl, { - headers: { - Authorization: `Bearer ${config.token}`, - }, + headers: { Authorization: `Bearer ${config.token}` }, }); if (deploymentsResponse.ok) { @@ -257,9 +278,7 @@ export async function deployToMain( } await tinybirdFetch(`${baseUrl}/v1/deployments/${stale.id}`, { method: "DELETE", - headers: { - Authorization: `Bearer ${config.token}`, - }, + headers: { Authorization: `Bearer ${config.token}` }, }); } } @@ -271,215 +290,134 @@ export async function deployToMain( } } - // Step 1: Create deployment via /v1/deploy - const deployUrlBase = `${baseUrl}/v1/deploy`; - const urlParams = new URLSearchParams(); - if (options?.check) { - urlParams.set("check", "true"); - } else if (auto) { - // Server will auto-promote the deployment when it's ready - urlParams.set("auto_promote", "true"); - } - if (options?.allowDestructiveOperations) { - urlParams.set("allow_destructive_operations", "true"); - } - const queryString = urlParams.toString(); - const deployUrl = queryString ? `${deployUrlBase}?${queryString}` : deployUrlBase; - - if (debug) { - console.log(`[debug] POST ${deployUrl}`); - } - - const response = await tinybirdFetch(deployUrl, { - method: "POST", - headers: { - Authorization: `Bearer ${config.token}`, - }, - body: formData, - }); - - // Parse response - let body: DeployResponse; - const rawBody = await response.text(); - - if (debug) { - console.log(`[debug] Response status: ${response.status}`); - console.log(`[debug] Response body: ${rawBody}`); - } + // Step 1: Submit the deployment via the tb binary. + const project = await materializeResourcesToTempProject(resources); try { - body = JSON.parse(rawBody) as DeployResponse; - } catch { - throw new Error( - `Failed to parse response from Tinybird API: ${response.status} ${response.statusText}\nBody: ${rawBody}` - ); - } - - // Helper to format feedback from deployment - const formatFeedback = (feedback: DeploymentFeedback[]): string => { - return feedback - .map((f) => { - // Extract just the filename from "Datasource events.datasource" format - const resourceName = f.resource?.split(" ").pop() ?? f.resource ?? "unknown"; - return `${resourceName}: ${normalizeDeployErrorMessage(f.message)}`; - }) - .join("\n"); - }; - - // Helper to format errors - const formatErrors = (): string => { - // Check for feedback in deployment (most detailed error info) - if (body.deployment?.feedback && body.deployment.feedback.length > 0) { - const errors = body.deployment.feedback.filter((f) => f.level === "ERROR"); - if (errors.length > 0) { - return formatFeedback(errors); + let binaryPath: string; + try { + binaryPath = await resolveTbBinaryPath(); + } catch (error) { + if (error instanceof CliBinaryUnavailableError) { + return { success: false, result: "failed", error: error.message, ...counts }; } + throw error; } - if (body.errors && body.errors.length > 0) { - return body.errors - .map((e) => { - const prefix = e.filename ? `[${e.filename}] ` : ""; - return `${prefix}${normalizeDeployErrorMessage(e.error)}`; - }) - .join("\n"); + + const args = ["deployment", "create", "--no-wait"]; + if (options?.check) { + args.push("--check"); + } else if (auto) { + args.push("--auto"); + } else { + args.push("--no-auto"); } - if (body.error) { - return normalizeDeployErrorMessage(body.error); + if (options?.allowDestructiveOperations) { + args.push("--allow-destructive-operations"); } - // Include raw response body for debugging when no structured error is available - return `HTTP ${response.status}: ${response.statusText}\nResponse: ${rawBody}`; - }; - // Handle non-OK responses - if (!response.ok) { - return { - success: false, - result: "failed", - error: formatErrors(), - datasourceCount: resources.datasources.length, - pipeCount: resources.pipes.length, - connectionCount: resources.connections?.length ?? 0, - }; - } + const run = await runTbBinary(binaryPath, args, { + cwd: project.dir, + token: config.token, + host: baseUrl, + }); - if (options?.check) { - options.callbacks?.onValidating?.(); + if (debug) { + console.log(`[debug] tb exit code: ${run.exitCode}`); + console.log(`[debug] tb stdout: ${run.stdout}`); + console.log(`[debug] tb stderr: ${run.stderr}`); + } - if (body.result === "failed") { + const parsed = parseTbDeploymentJson(run.stdout); + + if (!parsed) { + if (run.exitCode === 0) { + // VERIFY once a real binary exists: deployment_common.py's + // `no_changes` branch doesn't call echo_json, so empty stdout + exit + // 0 is inferred (not confirmed) to mean "nothing to deploy". + return { success: true, result: "no_changes", ...counts, ...emptyChanges() }; + } + // VERIFY once a real binary exists: a failed --check run also doesn't + // call echo_json in deployment_common.py, so this covers both that + // case and any other no-JSON failure. + const message = run.stderr.trim() || run.stdout.trim() || `tb exited with code ${run.exitCode}`; + return { success: false, result: "failed", error: message, ...counts }; + } + + if (run.exitCode !== 0) { return { success: false, result: "failed", - error: formatErrors(), - datasourceCount: resources.datasources.length, - pipeCount: resources.pipes.length, - connectionCount: resources.connections?.length ?? 0, + error: formatTbErrors(parsed, run.stderr.trim() || `tb exited with code ${run.exitCode}`), + ...counts, }; } - return { - success: true, - result: body.result ?? "success", - datasourceCount: resources.datasources.length, - pipeCount: resources.pipes.length, - connectionCount: resources.connections?.length ?? 0, - }; - } + if (options?.check) { + options.callbacks?.onValidating?.(); + return { success: true, result: "success", ...counts }; + } - // Handle no changes - no deployment was created, so we can return early - if (body.result === "no_changes") { - return { - success: true, - result: "no_changes", - datasourceCount: resources.datasources.length, - pipeCount: resources.pipes.length, - connectionCount: resources.connections?.length ?? 0, + const changes: DeploymentChanges = { + datasources: { + created: parsed.new_datasource_names ?? [], + changed: parsed.changed_datasource_names ?? [], + deleted: parsed.deleted_datasource_names ?? [], + }, pipes: { - changed: [], - created: [], - deleted: [], + created: parsed.new_pipe_names ?? [], + changed: parsed.changed_pipe_names ?? [], + deleted: parsed.deleted_pipe_names ?? [], }, - datasources: { - changed: [], - created: [], - deleted: [], + connections: { + created: parsed.new_data_connector_names ?? [], + changed: parsed.changed_data_connector_names ?? [], + deleted: parsed.deleted_data_connector_names ?? [], }, }; - } - // Handle API result - if (body.result === "failed" || !body.deployment) { - return { - success: false, - result: "failed", - error: formatErrors(), - datasourceCount: resources.datasources.length, - pipeCount: resources.pipes.length, - connectionCount: resources.connections?.length ?? 0, - }; - } + options?.callbacks?.onChanges?.(changes); - const deploymentId = body.deployment.id; - const deploymentDetails = body.deployment; + const deploymentId = parsed.id; - if (debug) { - console.log(`[debug] Deployment created with ID: ${deploymentId}`); - } + if (!wait) { + options?.callbacks?.onDeploymentSubmitted?.(deploymentId); + return { success: true, result: "success", buildId: deploymentId, ...counts, ...changes }; + } - // Notify about changes immediately after deployment is created - if (options?.callbacks?.onChanges) { - options.callbacks.onChanges({ - datasources: { - created: deploymentDetails.new_datasource_names ?? [], - changed: deploymentDetails.changed_datasource_names ?? [], - deleted: deploymentDetails.deleted_datasource_names ?? [], - }, - pipes: { - created: deploymentDetails.new_pipe_names ?? [], - changed: deploymentDetails.changed_pipe_names ?? [], - deleted: deploymentDetails.deleted_pipe_names ?? [], - }, - connections: { - created: deploymentDetails.new_data_connector_names ?? [], - changed: deploymentDetails.changed_data_connector_names ?? [], - deleted: deploymentDetails.deleted_data_connector_names ?? [], - }, + // Step 2: poll until the deployment reaches a terminal state. + return await pollDeploymentUntilDone({ + config, + deploymentId, + auto, + debug, + callbacks: options?.callbacks, + counts, + changes, }); + } finally { + await project.cleanup(); } +} - const deploymentChanges = { - pipes: { - changed: deploymentDetails.changed_pipe_names ?? [], - created: deploymentDetails.new_pipe_names ?? [], - deleted: deploymentDetails.deleted_pipe_names ?? [], - }, - datasources: { - changed: deploymentDetails.changed_datasource_names ?? [], - created: deploymentDetails.new_datasource_names ?? [], - deleted: deploymentDetails.deleted_datasource_names ?? [], - }, - }; - - // If we're not waiting, return as soon as the server accepted the deployment. - if (!wait) { - options?.callbacks?.onDeploymentSubmitted?.(deploymentId); - return { - success: true, - result: "success", - datasourceCount: resources.datasources.length, - pipeCount: resources.pipes.length, - connectionCount: resources.connections?.length ?? 0, - buildId: deploymentId, - ...deploymentChanges, - }; - } +async function pollDeploymentUntilDone(args: { + config: BuildConfig; + deploymentId: string; + auto: boolean; + debug?: boolean; + callbacks?: DeployCallbacks; + counts: { datasourceCount: number; pipeCount: number; connectionCount: number }; + changes: DeploymentChanges; +}): Promise { + const { config, deploymentId, auto, debug, callbacks, counts, changes } = args; + const baseUrl = config.baseUrl.replace(/\/$/, ""); - // Step 2: Poll until the deployment reaches a terminal state. - let deployment = body.deployment; + let deployment: Deployment = { id: deploymentId, status: "pending" }; let timesSeenFailed = 0; let notifiedReady = false; let notifiedWaitingForPromote = false; - options?.callbacks?.onWaitingForReady?.(); + callbacks?.onWaitingForReady?.(); const isDone = (): boolean => { if (deployment.status !== "data_ready") { @@ -496,10 +434,8 @@ export async function deployToMain( success: false, result: "failed", error: message, - datasourceCount: resources.datasources.length, - pipeCount: resources.pipes.length, - connectionCount: resources.connections?.length ?? 0, buildId: deploymentId, + ...counts, }); while (!isDone()) { @@ -511,9 +447,7 @@ export async function deployToMain( const statusUrl = `${baseUrl}/v1/deployments/${deploymentId}`; const statusResponse = await tinybirdFetch(statusUrl, { - headers: { - Authorization: `Bearer ${config.token}`, - }, + headers: { Authorization: `Bearer ${config.token}` }, }); if (!statusResponse.ok) { @@ -562,36 +496,27 @@ export async function deployToMain( !notifiedWaitingForPromote ) { if (!notifiedReady) { - options?.callbacks?.onDeploymentReady?.(); + callbacks?.onDeploymentReady?.(); notifiedReady = true; } - options?.callbacks?.onWaitingForPromote?.(); + callbacks?.onWaitingForPromote?.(); notifiedWaitingForPromote = true; } } if (!notifiedReady) { - options?.callbacks?.onDeploymentReady?.(); - notifiedReady = true; + callbacks?.onDeploymentReady?.(); } if (auto) { if (debug) { console.log(`[debug] Deployment ${deploymentId} is now live`); } - options?.callbacks?.onDeploymentPromoted?.(); - options?.callbacks?.onDeploymentLive?.(deploymentId); + callbacks?.onDeploymentPromoted?.(); + callbacks?.onDeploymentLive?.(deploymentId); } - return { - success: true, - result: "success", - datasourceCount: resources.datasources.length, - pipeCount: resources.pipes.length, - connectionCount: resources.connections?.length ?? 0, - buildId: deploymentId, - ...deploymentChanges, - }; + return { success: true, result: "success", buildId: deploymentId, ...counts, ...changes }; } /** @@ -600,16 +525,3 @@ export async function deployToMain( function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } - -function normalizeDeployErrorMessage(message: string): string { - const trimmedMessage = message.trim(); - const isForwardClassicError = - trimmedMessage.includes("Tinybird Forward workspace") && - trimmedMessage.includes("Tinybird Classic workspaces"); - - if (!isForwardClassicError || trimmedMessage.includes(FORWARD_CLASSIC_GUIDANCE)) { - return message; - } - - return `${trimmedMessage}\n${FORWARD_CLASSIC_GUIDANCE}`; -} diff --git a/src/cli-binary/download.test.ts b/src/cli-binary/download.test.ts new file mode 100644 index 0000000..d3f68a2 --- /dev/null +++ b/src/cli-binary/download.test.ts @@ -0,0 +1,174 @@ +import { createHash } from "node:crypto"; +import { readFile, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { + ChecksumMismatchError, + ensureBinaryDownloaded, + extractSingleFileTarGz, + getCacheDir, +} from "./download.js"; +import type { ManifestArtifact } from "./manifest.js"; + +/** + * Builds a minimal valid single-entry ustar archive (gzip'd), equivalent to + * what Python's `tarfile.open(dest, "w:gz")` + one `tar.add(path, arcname)` + * call produces (see package_artifact() in tinybird/tb/modules/binary.py). + */ +function buildTarGz(entryName: string, content: Buffer): Buffer { + const header = Buffer.alloc(512); + header.write(entryName, 0, "ascii"); + header.write("0000755\0", 100, "ascii"); // mode + header.write("0000000\0", 108, "ascii"); // uid + header.write("0000000\0", 116, "ascii"); // gid + header.write(`${content.length.toString(8).padStart(11, "0")}\0`, 124, "ascii"); // size + header.write("00000000000\0", 136, "ascii"); // mtime + header.write(" ", 148, "ascii"); // checksum placeholder (8 spaces) + header.write("0", 156, "ascii"); // typeflag: regular file + header.write("ustar\0", 257, "ascii"); // magic + header.write("00", 263, "ascii"); // version + + let checksum = 0; + for (const byte of header) checksum += byte; + header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, "ascii"); + + const contentBlockSize = Math.ceil(content.length / 512) * 512; + const contentBlock = Buffer.alloc(contentBlockSize); + content.copy(contentBlock); + + const tar = Buffer.concat([header, contentBlock, Buffer.alloc(1024)]); // two zero blocks at EOF + return gzipSync(tar); +} + +describe("extractSingleFileTarGz", () => { + it("extracts the single file's content", () => { + const content = Buffer.from("fake-binary-content"); + const tarGz = buildTarGz("tb", content); + expect(extractSingleFileTarGz(tarGz)).toEqual(content); + }); + + it("works for content spanning multiple 512-byte blocks", () => { + const content = Buffer.from("x".repeat(512 * 2 + 37)); + const tarGz = buildTarGz("tb", content); + expect(extractSingleFileTarGz(tarGz)).toEqual(content); + }); + + it("throws on a truncated archive", () => { + expect(() => extractSingleFileTarGz(gzipSync(Buffer.alloc(10)))).toThrow(/shorter than one header block/); + }); +}); + +describe("getCacheDir", () => { + const originalCacheDir = process.env.TB_CLI_CACHE_DIR; + const originalXdg = process.env.XDG_CACHE_HOME; + + afterEach(() => { + if (originalCacheDir === undefined) delete process.env.TB_CLI_CACHE_DIR; + else process.env.TB_CLI_CACHE_DIR = originalCacheDir; + if (originalXdg === undefined) delete process.env.XDG_CACHE_HOME; + else process.env.XDG_CACHE_HOME = originalXdg; + }); + + it("honors TB_CLI_CACHE_DIR when set", () => { + process.env.TB_CLI_CACHE_DIR = "/tmp/custom-cache"; + expect(getCacheDir()).toBe("/tmp/custom-cache"); + }); + + it("falls back to XDG_CACHE_HOME/tinybird-cli", () => { + delete process.env.TB_CLI_CACHE_DIR; + process.env.XDG_CACHE_HOME = "/tmp/xdg-cache"; + expect(getCacheDir()).toBe(join("/tmp/xdg-cache", "tinybird-cli")); + }); +}); + +describe("ensureBinaryDownloaded", () => { + let cacheDir: string; + + beforeEach(async () => { + cacheDir = await mkdtemp(join(tmpdir(), "tb-cli-cache-test-")); + }); + + afterEach(async () => { + await rm(cacheDir, { recursive: true, force: true }); + }); + + function artifactFor(content: Buffer): ManifestArtifact { + return { + id: "linux-x64", + os: "linux", + arch: "x64", + bundle_mode: "onefile", + executable_relative_path: "tb", + filename: "tb-1.0.0-linux-x64.tar.gz", + sha256: createHash("sha256").update(buildTarGz("tb", content)).digest("hex"), + size_bytes: content.length, + relative_path: "packages/tb-1.0.0-linux-x64.tar.gz", + url: "https://example.com/tb-1.0.0-linux-x64.tar.gz", + }; + } + + it("downloads, verifies, extracts, and caches the binary", async () => { + const content = Buffer.from("fake-binary"); + const artifact = artifactFor(content); + const tarGz = buildTarGz("tb", content); + + const fetchFn = async () => new Response(new Uint8Array(tarGz), { status: 200 }); + + const path = await ensureBinaryDownloaded({ + version: "1.0.0", + artifact, + cacheDir, + fetchFn: fetchFn as unknown as typeof fetch, + }); + + expect(path).toBe(join(cacheDir, "1.0.0", "tb")); + expect(await readFile(path)).toEqual(content); + }); + + it("reuses the cached binary on a second call without fetching again", async () => { + const content = Buffer.from("fake-binary"); + const artifact = artifactFor(content); + const tarGz = buildTarGz("tb", content); + + let fetchCount = 0; + const fetchFn = async () => { + fetchCount++; + return new Response(new Uint8Array(tarGz), { status: 200 }); + }; + + const options = { version: "1.0.0", artifact, cacheDir, fetchFn: fetchFn as unknown as typeof fetch }; + await ensureBinaryDownloaded(options); + await ensureBinaryDownloaded(options); + + expect(fetchCount).toBe(1); + }); + + it("throws ChecksumMismatchError when the downloaded tarball doesn't match", async () => { + const artifact = artifactFor(Buffer.from("expected")); + const fetchFn = async () => new Response(new Uint8Array(buildTarGz("tb", Buffer.from("tampered"))), { status: 200 }); + + await expect( + ensureBinaryDownloaded({ version: "1.0.0", artifact, cacheDir, fetchFn: fetchFn as unknown as typeof fetch }) + ).rejects.toThrow(ChecksumMismatchError); + }); + + it("throws a clear error when the artifact has no download URL", async () => { + const artifact = { ...artifactFor(Buffer.from("x")), url: undefined }; + + await expect(ensureBinaryDownloaded({ version: "1.0.0", artifact, cacheDir })).rejects.toThrow( + /TB_CLI_BINARY_PATH/ + ); + }); + + it("throws a clear error on a non-OK download response", async () => { + const artifact = artifactFor(Buffer.from("x")); + const fetchFn = async () => new Response("nope", { status: 500, statusText: "Internal Server Error" }); + + await expect( + ensureBinaryDownloaded({ version: "1.0.0", artifact, cacheDir, fetchFn: fetchFn as unknown as typeof fetch }) + ).rejects.toThrow(/500/); + }); +}); diff --git a/src/cli-binary/download.ts b/src/cli-binary/download.ts new file mode 100644 index 0000000..e7bdb41 --- /dev/null +++ b/src/cli-binary/download.ts @@ -0,0 +1,135 @@ +/** + * Downloads, verifies, and caches the `tb` CLI binary tarball described by a + * manifest artifact (see manifest.ts). Each tarball is a gzip'd ustar archive + * containing exactly one regular file (the `tb`/`tb.exe` executable) — see + * `package_artifact()` in tinybird/tb/modules/binary.py (MR !23765), which + * builds it with Python's `tarfile.open(destination, "w:gz")` plus a single + * `tar.add(artifact_path, arcname=artifact_path.name)` call. + */ + +import { createHash } from "node:crypto"; +import { gunzipSync } from "node:zlib"; +import { chmod, mkdir, rename, writeFile } from "node:fs/promises"; +import { access } from "node:fs/promises"; +import { constants as fsConstants } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +import type { ManifestArtifact } from "./manifest.js"; + +export class ChecksumMismatchError extends Error {} + +function sha256Hex(buffer: Buffer): string { + return createHash("sha256").update(buffer).digest("hex"); +} + +/** + * Reads a single-entry ustar archive (after gunzip) and returns that entry's + * content. Not a general-purpose tar reader: it only reads the first header + * block, which is all these always-single-file archives contain. + */ +export function extractSingleFileTarGz(tarGzBuffer: Buffer): Buffer { + const tarBuffer = gunzipSync(tarGzBuffer); + + if (tarBuffer.length < 512) { + throw new Error("Invalid tar archive: shorter than one header block"); + } + + // ustar header: size is a 12-byte octal ASCII field at offset 124. + const sizeField = tarBuffer + .subarray(124, 124 + 12) + .toString("ascii") + .replace(/\0.*$/, "") + .trim(); + const size = Number.parseInt(sizeField, 8); + if (!Number.isFinite(size) || size < 0) { + throw new Error(`Invalid tar header: could not read file size (raw field: ${JSON.stringify(sizeField)})`); + } + + const content = tarBuffer.subarray(512, 512 + size); + if (content.length !== size) { + throw new Error( + `Invalid tar archive: expected ${size} bytes of content, found ${content.length}` + ); + } + + return content; +} + +export function getCacheDir(): string { + if (process.env.TB_CLI_CACHE_DIR) { + return process.env.TB_CLI_CACHE_DIR; + } + const xdgCacheHome = process.env.XDG_CACHE_HOME; + const base = xdgCacheHome && xdgCacheHome.length > 0 ? xdgCacheHome : join(homedir(), ".cache"); + return join(base, "tinybird-cli"); +} + +async function isExecutableFile(path: string): Promise { + try { + await access(path, fsConstants.X_OK); + return true; + } catch { + return false; + } +} + +export interface EnsureBinaryDownloadedOptions { + /** Resolved manifest version (not "latest" — the concrete version string). */ + version: string; + artifact: ManifestArtifact; + fetchFn?: typeof fetch; + cacheDir?: string; +} + +/** + * Ensures the binary described by `artifact` is present in the local cache, + * downloading and verifying it if necessary, and returns its path. + */ +export async function ensureBinaryDownloaded(options: EnsureBinaryDownloadedOptions): Promise { + const { version, artifact } = options; + const fetchFn = options.fetchFn ?? fetch; + const cacheDir = options.cacheDir ?? getCacheDir(); + const versionDir = join(cacheDir, version); + const finalPath = join(versionDir, artifact.executable_relative_path); + + if (await isExecutableFile(finalPath)) { + return finalPath; + } + + if (!artifact.url) { + throw new Error( + `Manifest artifact for ${artifact.os}-${artifact.arch} has no download URL ` + + `(manifest was generated without --base-url). Set TB_CLI_BINARY_PATH to use a local binary instead.` + ); + } + + const response = await fetchFn(artifact.url); + if (!response.ok) { + throw new Error( + `Failed to download Tinybird CLI binary from ${artifact.url}: ${response.status} ${response.statusText}` + ); + } + + const tarGzBuffer = Buffer.from(await response.arrayBuffer()); + + const actualSha256 = sha256Hex(tarGzBuffer); + if (actualSha256 !== artifact.sha256) { + throw new ChecksumMismatchError( + `Checksum mismatch for Tinybird CLI binary ${artifact.filename}: ` + + `expected ${artifact.sha256}, got ${actualSha256}` + ); + } + + const executable = extractSingleFileTarGz(tarGzBuffer); + + await mkdir(versionDir, { recursive: true }); + // Download to a unique temp name and rename into place so concurrent + // processes racing to populate the same cache dir never see a partial file. + const tempPath = `${finalPath}.download-${process.pid}-${Date.now()}`; + await writeFile(tempPath, executable); + await chmod(tempPath, 0o755); + await rename(tempPath, finalPath); + + return finalPath; +} diff --git a/src/cli-binary/index.ts b/src/cli-binary/index.ts new file mode 100644 index 0000000..906a0d2 --- /dev/null +++ b/src/cli-binary/index.ts @@ -0,0 +1,74 @@ +/** + * Resolves a usable `tb` binary path: an explicit TB_CLI_BINARY_PATH override, + * or the manifest-published binary for the current platform, downloaded and + * cached on demand. + * + * DRAFT: there is no published manifest yet (the MR that builds these + * binaries, https://gitlab.com/tinybird/analytics/-/merge_requests/23765, + * hasn't merged). This module is unit-tested against fakes only; nothing here + * has been exercised against a real manifest or a real binary. + */ + +export * from "./platform.js"; +export * from "./manifest.js"; +export * from "./download.js"; +export * from "./run.js"; + +import { getHostPlatform } from "./platform.js"; +import { fetchManifest, findArtifactForHost } from "./manifest.js"; +import { ensureBinaryDownloaded } from "./download.js"; + +/** + * `tb` version this SDK is tested against. "latest" is a placeholder until a + * concrete version has actually been validated end-to-end (see plan: pin + * deliberately once that validation happens, don't float on latest forever). + */ +export const DEFAULT_TB_CLI_VERSION = "latest"; + +export class CliBinaryUnavailableError extends Error {} + +export interface ResolveTbBinaryOptions { + version?: string; + baseUrl?: string; + fetchFn?: typeof fetch; + cacheDir?: string; +} + +export async function resolveTbBinaryPath(options: ResolveTbBinaryOptions = {}): Promise { + const override = process.env.TB_CLI_BINARY_PATH; + if (override) { + return override; + } + + const host = getHostPlatform(); + const requestedVersion = options.version ?? process.env.TB_CLI_VERSION ?? DEFAULT_TB_CLI_VERSION; + + const manifest = await fetchManifest({ + baseUrl: options.baseUrl, + version: requestedVersion, + fetchFn: options.fetchFn, + }).catch((error: unknown) => { + throw new CliBinaryUnavailableError( + `Could not fetch Tinybird CLI manifest: ${(error as Error).message}` + ); + }); + + const artifact = findArtifactForHost(manifest, host); + if (!artifact) { + throw new CliBinaryUnavailableError( + `No Tinybird CLI binary is published for ${host.os}-${host.arch}. ` + + `Set TB_CLI_BINARY_PATH to point at a local tb executable instead.` + ); + } + + return ensureBinaryDownloaded({ + version: manifest.version, + artifact, + fetchFn: options.fetchFn, + cacheDir: options.cacheDir, + }).catch((error: unknown) => { + throw new CliBinaryUnavailableError( + `Could not download Tinybird CLI binary: ${(error as Error).message}` + ); + }); +} diff --git a/src/cli-binary/manifest.test.ts b/src/cli-binary/manifest.test.ts new file mode 100644 index 0000000..d6cb123 --- /dev/null +++ b/src/cli-binary/manifest.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it, vi } from "vitest"; + +import { DEFAULT_CLI_BINARIES_BASE_URL, fetchManifest, findArtifactForHost, type CliManifest } from "./manifest.js"; + +function sampleManifest(): CliManifest { + return { + version: "1.2.3", + generated_at: "2026-01-01T00:00:00Z", + artifacts: [ + { + id: "darwin-arm64", + os: "darwin", + arch: "arm64", + bundle_mode: "onefile", + executable_relative_path: "tb", + filename: "tb-1.2.3-darwin-arm64.tar.gz", + sha256: "abc123", + size_bytes: 42, + relative_path: "packages/tb-1.2.3-darwin-arm64.tar.gz", + url: "https://tinybird.co/cli/1.2.3/packages/tb-1.2.3-darwin-arm64.tar.gz", + }, + { + id: "windows-x64", + os: "windows", + arch: "x64", + bundle_mode: "onefile", + executable_relative_path: "tb.exe", + filename: "tb-1.2.3-windows-x64.tar.gz", + sha256: "def456", + size_bytes: 43, + relative_path: "packages/tb-1.2.3-windows-x64.tar.gz", + }, + ], + }; +} + +describe("fetchManifest", () => { + it("fetches from the default base URL and latest version by default", async () => { + const fetchFn = vi.fn(async (url: string) => { + expect(url).toBe(`${DEFAULT_CLI_BINARIES_BASE_URL}/latest/manifest.json`); + return new Response(JSON.stringify(sampleManifest()), { status: 200 }); + }); + + const manifest = await fetchManifest({ fetchFn: fetchFn as unknown as typeof fetch }); + expect(manifest.version).toBe("1.2.3"); + expect(manifest.artifacts).toHaveLength(2); + }); + + it("respects an explicit baseUrl and version, stripping a trailing slash", async () => { + const fetchFn = vi.fn(async (url: string) => { + expect(url).toBe("https://example.com/cli/9.9.9/manifest.json"); + return new Response(JSON.stringify(sampleManifest()), { status: 200 }); + }); + + await fetchManifest({ + baseUrl: "https://example.com/cli/", + version: "9.9.9", + fetchFn: fetchFn as unknown as typeof fetch, + }); + }); + + it("throws a clear error on a non-OK response", async () => { + const fetchFn = vi.fn(async () => new Response("nope", { status: 404, statusText: "Not Found" })); + + await expect(fetchManifest({ fetchFn: fetchFn as unknown as typeof fetch })).rejects.toThrow(/404/); + }); + + it("throws on a malformed manifest body", async () => { + const fetchFn = vi.fn(async () => new Response(JSON.stringify({ not: "a manifest" }), { status: 200 })); + + await expect(fetchManifest({ fetchFn: fetchFn as unknown as typeof fetch })).rejects.toThrow(/Invalid/); + }); +}); + +describe("findArtifactForHost", () => { + it("finds the matching artifact by os/arch", () => { + const manifest = sampleManifest(); + const artifact = findArtifactForHost(manifest, { os: "darwin", arch: "arm64" }); + expect(artifact?.id).toBe("darwin-arm64"); + }); + + it("returns undefined when no artifact matches", () => { + const manifest = sampleManifest(); + expect(findArtifactForHost(manifest, { os: "darwin", arch: "x64" })).toBeUndefined(); + }); +}); diff --git a/src/cli-binary/manifest.ts b/src/cli-binary/manifest.ts new file mode 100644 index 0000000..a82f878 --- /dev/null +++ b/src/cli-binary/manifest.ts @@ -0,0 +1,76 @@ +/** + * Fetches and parses the manifest published by the `tb` CLI binary pipeline + * (MR https://gitlab.com/tinybird/analytics/-/merge_requests/23765). + * + * Manifest shape is produced by `tinybird.tb.modules.binary.build_manifest`: + * `{ version, generated_at, artifacts: [{ id, os, arch, bundle_mode, + * executable_relative_path, filename, sha256, size_bytes, relative_path, url }] }`. + */ + +import type { HostPlatform } from "./platform.js"; + +export interface ManifestArtifact { + id: string; + os: string; + arch: string; + bundle_mode: string; + executable_relative_path: string; + filename: string; + sha256: string; + size_bytes: number; + relative_path: string; + /** Absent when the manifest was generated without --base-url. */ + url?: string; +} + +export interface CliManifest { + version: string; + generated_at: string; + artifacts: ManifestArtifact[]; +} + +/** + * Default base URL for the published manifest (`${base}/${version}/manifest.json`). + * Matches the CLI_BINARIES_BASE_URL CI variable used when publishing (see + * gitlab/ci-templates/cli.yml in tinybird/analytics). + */ +export const DEFAULT_CLI_BINARIES_BASE_URL = "https://tinybird.co/cli"; + +export interface FetchManifestOptions { + /** Defaults to TB_CLI_BASE_URL env var, then DEFAULT_CLI_BINARIES_BASE_URL. */ + baseUrl?: string; + /** Defaults to "latest". */ + version?: string; + fetchFn?: typeof fetch; +} + +export async function fetchManifest(options: FetchManifestOptions = {}): Promise { + const baseUrl = (options.baseUrl ?? process.env.TB_CLI_BASE_URL ?? DEFAULT_CLI_BINARIES_BASE_URL).replace( + /\/$/, + "" + ); + const version = options.version ?? "latest"; + const fetchFn = options.fetchFn ?? fetch; + const url = `${baseUrl}/${version}/manifest.json`; + + const response = await fetchFn(url); + if (!response.ok) { + throw new Error( + `Failed to fetch Tinybird CLI manifest from ${url}: ${response.status} ${response.statusText}` + ); + } + + const manifest = (await response.json()) as CliManifest; + if (!manifest || !Array.isArray(manifest.artifacts) || typeof manifest.version !== "string") { + throw new Error(`Invalid Tinybird CLI manifest received from ${url}`); + } + + return manifest; +} + +export function findArtifactForHost( + manifest: CliManifest, + host: HostPlatform +): ManifestArtifact | undefined { + return manifest.artifacts.find((artifact) => artifact.os === host.os && artifact.arch === host.arch); +} diff --git a/src/cli-binary/platform.test.ts b/src/cli-binary/platform.test.ts new file mode 100644 index 0000000..8e1eb55 --- /dev/null +++ b/src/cli-binary/platform.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; + +import { executableName, getHostPlatform, normalizeArch, normalizeOs } from "./platform.js"; + +describe("normalizeOs", () => { + it("maps known platform strings", () => { + expect(normalizeOs("Linux")).toBe("linux"); + expect(normalizeOs("linux")).toBe("linux"); + expect(normalizeOs("Darwin")).toBe("darwin"); + expect(normalizeOs("darwin")).toBe("darwin"); + expect(normalizeOs("Windows")).toBe("windows"); + expect(normalizeOs("win32")).toBe("windows"); + }); + + it("passes through unknown platforms unchanged (lowercased)", () => { + expect(normalizeOs("SunOS")).toBe("sunos"); + }); +}); + +describe("normalizeArch", () => { + it("maps known architecture strings", () => { + expect(normalizeArch("x86_64")).toBe("x64"); + expect(normalizeArch("amd64")).toBe("x64"); + expect(normalizeArch("x64")).toBe("x64"); + expect(normalizeArch("arm64")).toBe("arm64"); + expect(normalizeArch("aarch64")).toBe("arm64"); + }); + + it("passes through unknown architectures unchanged (lowercased)", () => { + expect(normalizeArch("ppc64le")).toBe("ppc64le"); + }); +}); + +describe("getHostPlatform", () => { + it("reflects the real process.platform/process.arch, normalized", () => { + const host = getHostPlatform(); + expect(host.os).toBe(normalizeOs(process.platform)); + expect(host.arch).toBe(normalizeArch(process.arch)); + }); +}); + +describe("executableName", () => { + it("appends .exe only for windows", () => { + expect(executableName("windows")).toBe("tb.exe"); + expect(executableName("linux")).toBe("tb"); + expect(executableName("darwin")).toBe("tb"); + }); +}); diff --git a/src/cli-binary/platform.ts b/src/cli-binary/platform.ts new file mode 100644 index 0000000..467225f --- /dev/null +++ b/src/cli-binary/platform.ts @@ -0,0 +1,36 @@ +/** + * Platform/arch normalization matching the Python side of the binary pipeline + * (`tinybird.tb.modules.binary.normalize_os` / `normalize_arch` in + * tinybird/analytics, see MR !23765) so manifest target ids line up exactly. + */ + +export interface HostPlatform { + os: string; + arch: string; +} + +export function normalizeOs(rawOs: string): string { + const value = rawOs.toLowerCase(); + if (value.startsWith("linux")) return "linux"; + if (value.startsWith("darwin")) return "darwin"; + if (value.startsWith("win")) return "windows"; + return value; +} + +export function normalizeArch(rawArch: string): string { + const value = rawArch.toLowerCase(); + if (value === "x86_64" || value === "amd64" || value === "x64") return "x64"; + if (value === "arm64" || value === "aarch64") return "arm64"; + return value; +} + +export function getHostPlatform(): HostPlatform { + return { + os: normalizeOs(process.platform), + arch: normalizeArch(process.arch), + }; +} + +export function executableName(targetOs: string): string { + return targetOs === "windows" ? "tb.exe" : "tb"; +} diff --git a/src/cli-binary/resources-to-project.test.ts b/src/cli-binary/resources-to-project.test.ts new file mode 100644 index 0000000..1f92e02 --- /dev/null +++ b/src/cli-binary/resources-to-project.test.ts @@ -0,0 +1,55 @@ +import { readFile, readdir, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { materializeResourcesToTempProject } from "./resources-to-project.js"; +import type { GeneratedResources } from "../generator/index.js"; + +describe("materializeResourcesToTempProject", () => { + const cleanups: Array<() => Promise> = []; + + afterEach(async () => { + await Promise.all(cleanups.map((cleanup) => cleanup())); + cleanups.length = 0; + }); + + it("writes datasources, pipes, and connections to their respective folders", async () => { + const resources: GeneratedResources = { + datasources: [{ name: "events", content: "SCHEMA > timestamp DateTime" }], + pipes: [{ name: "top_events", content: "NODE main\nSQL > SELECT 1" }], + connections: [{ name: "my_kafka", content: "TYPE kafka" }], + }; + + const project = await materializeResourcesToTempProject(resources); + cleanups.push(project.cleanup); + + expect(await readFile(join(project.dir, "datasources", "events.datasource"), "utf-8")).toBe( + "SCHEMA > timestamp DateTime" + ); + expect(await readFile(join(project.dir, "pipes", "top_events.pipe"), "utf-8")).toBe( + "NODE main\nSQL > SELECT 1" + ); + expect(await readFile(join(project.dir, "connections", "my_kafka.connection"), "utf-8")).toBe("TYPE kafka"); + }); + + it("skips creating folders for empty resource kinds", async () => { + const resources: GeneratedResources = { + datasources: [{ name: "events", content: "x" }], + pipes: [], + connections: [], + }; + + const project = await materializeResourcesToTempProject(resources); + cleanups.push(project.cleanup); + + const entries = await readdir(project.dir); + expect(entries.sort()).toEqual(["datasources"]); + }); + + it("cleanup removes the temp directory", async () => { + const project = await materializeResourcesToTempProject({ datasources: [], pipes: [], connections: [] }); + await project.cleanup(); + + await expect(stat(project.dir)).rejects.toThrow(); + }); +}); diff --git a/src/cli-binary/resources-to-project.ts b/src/cli-binary/resources-to-project.ts new file mode 100644 index 0000000..95353ab --- /dev/null +++ b/src/cli-binary/resources-to-project.ts @@ -0,0 +1,46 @@ +/** + * Materializes in-memory GeneratedResources to a temp directory laid out as + * a `tb` project (datasources/*.datasource, pipes/*.pipe, connections/*.connection), + * so the binary can be invoked against it. Mirrors `_write_virtual_resources` + * in tinybird/tb/modules/ts_project.py, which does the same thing in the + * opposite direction (tb -> this SDK's generator -> virtual project). + */ + +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { GeneratedResources } from "../generator/index.js"; + +export interface MaterializedProject { + dir: string; + cleanup: () => Promise; +} + +const LAYOUTS: Array<{ key: keyof GeneratedResources; folder: string; suffix: string }> = [ + { key: "datasources", folder: "datasources", suffix: ".datasource" }, + { key: "pipes", folder: "pipes", suffix: ".pipe" }, + { key: "connections", folder: "connections", suffix: ".connection" }, +]; + +export async function materializeResourcesToTempProject( + resources: GeneratedResources +): Promise { + const dir = await mkdtemp(join(tmpdir(), "tinybird-cli-project-")); + + for (const { key, folder, suffix } of LAYOUTS) { + const entries = resources[key] ?? []; + if (entries.length === 0) continue; + + const targetDir = join(dir, folder); + await mkdir(targetDir, { recursive: true }); + for (const entry of entries) { + await writeFile(join(targetDir, `${entry.name}${suffix}`), entry.content, "utf-8"); + } + } + + return { + dir, + cleanup: () => rm(dir, { recursive: true, force: true }), + }; +} diff --git a/src/cli-binary/run.test.ts b/src/cli-binary/run.test.ts new file mode 100644 index 0000000..aa8b7af --- /dev/null +++ b/src/cli-binary/run.test.ts @@ -0,0 +1,71 @@ +import { mkdtempSync, realpathSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { runTbBinary } from "./run.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const FAKE_TB = join(__dirname, "..", "test", "fixtures", "fake-tb-cli.mjs"); + +// The fixture is a POSIX shebang script; skip on Windows where it can't be +// exec'd directly (see fake-tb-cli.mjs for why this test runs the real +// subprocess path instead of mocking child_process). +const itPosix = process.platform === "win32" ? it.skip : it; + +describe("runTbBinary", () => { + itPosix("captures stdout, stderr, and exit code", async () => { + const result = await runTbBinary(FAKE_TB, ["deployment", "create"], { + env: { FAKE_TB_STDOUT: '{"ok":true}', FAKE_TB_STDERR: "a warning", FAKE_TB_EXIT_CODE: "0" }, + }); + + expect(result.stdout).toBe('{"ok":true}'); + expect(result.stderr).toBe("a warning"); + expect(result.exitCode).toBe(0); + }); + + itPosix("propagates a non-zero exit code", async () => { + const result = await runTbBinary(FAKE_TB, ["deployment", "create"], { + env: { FAKE_TB_STDERR: "boom", FAKE_TB_EXIT_CODE: "1" }, + }); + + expect(result.exitCode).toBe(1); + expect(result.stderr).toBe("boom"); + }); + + itPosix("puts global flags before the subcommand, and credentials in env not argv", async () => { + const result = await runTbBinary(FAKE_TB, ["deployment", "create", "--no-wait"], { + env: { FAKE_TB_ECHO_ARGV: "1" }, + token: "p.secret-token", + host: "https://api.tinybird.co", + branch: "my-branch", + }); + + const parsed = JSON.parse(result.stdout); + expect(parsed.argv).toEqual([ + "--output", + "json", + "--branch", + "my-branch", + "deployment", + "create", + "--no-wait", + ]); + expect(parsed.env.TB_TOKEN).toBe("p.secret-token"); + expect(parsed.env.TB_HOST).toBe("https://api.tinybird.co"); + // The token must never appear as a bare argv entry. + expect(parsed.argv.join(" ")).not.toContain("p.secret-token"); + }); + + itPosix("runs in the requested cwd", async () => { + const cwd = realpathSync(mkdtempSync(join(tmpdir(), "run-tb-binary-test-"))); + const result = await runTbBinary(FAKE_TB, [], { + env: { FAKE_TB_ECHO_ARGV: "1" }, + cwd, + }); + + const parsed = JSON.parse(result.stdout); + expect(parsed.cwd).toBe(cwd); + }); +}); diff --git a/src/cli-binary/run.ts b/src/cli-binary/run.ts new file mode 100644 index 0000000..35985dc --- /dev/null +++ b/src/cli-binary/run.ts @@ -0,0 +1,62 @@ +/** + * Thin spawn wrapper around the `tb` binary. Credentials are passed via env + * vars (TB_TOKEN/TB_USER_TOKEN/TB_HOST), not CLI flags, so they don't show up + * in `ps`/process listings, and never via the `.tinyb` config file, so + * concurrent invocations don't race on shared state. See `CLIConfig` in + * tinybird/tb/modules/config.py for the env vars the binary honors. + * + * Global flags (--output, --branch, ...) must precede the subcommand per + * Click's group/subcommand parsing rules — see `cli()` in + * tinybird/tb/modules/cli.py. + */ + +import { spawn } from "node:child_process"; + +export interface RunTbOptions { + cwd?: string; + token?: string; + userToken?: string; + host?: string; + branch?: string; + outputFormat?: "human" | "json" | "csv"; + env?: NodeJS.ProcessEnv; +} + +export interface RunTbResult { + stdout: string; + stderr: string; + exitCode: number | null; +} + +export function runTbBinary( + binaryPath: string, + commandArgs: string[], + options: RunTbOptions = {} +): Promise { + return new Promise((resolve, reject) => { + const env: NodeJS.ProcessEnv = { ...process.env, ...options.env }; + if (options.token) env.TB_TOKEN = options.token; + if (options.userToken) env.TB_USER_TOKEN = options.userToken; + if (options.host) env.TB_HOST = options.host; + + const globalArgs = ["--output", options.outputFormat ?? "json"]; + if (options.branch) globalArgs.push("--branch", options.branch); + + const fullArgs = [...globalArgs, ...commandArgs]; + + const child = spawn(binaryPath, fullArgs, { cwd: options.cwd, env }); + + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString("utf-8"); + }); + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString("utf-8"); + }); + child.once("error", reject); + child.once("close", (code) => { + resolve({ stdout, stderr, exitCode: code }); + }); + }); +} diff --git a/src/test/fixtures/fake-tb-cli.mjs b/src/test/fixtures/fake-tb-cli.mjs new file mode 100755 index 0000000..bc455ba --- /dev/null +++ b/src/test/fixtures/fake-tb-cli.mjs @@ -0,0 +1,60 @@ +#!/usr/bin/env node +/** + * Stand-in for the real `tb` binary, used only in tests. Mirrors the Python + * side's own `tests_tb/fixtures/fake_pyinstaller` pattern (MR !23765): the + * subprocess call in run.ts is real and unmocked, but it spawns this + * lightweight script instead of a multi-hundred-MB PyInstaller binary. + * + * Behavior is controlled entirely by env vars so callers don't need a second + * fixture file per scenario: + * - FAKE_TB_STDOUT: raw string written to stdout (default: ""). + * - FAKE_TB_STDERR: raw string written to stderr (default: ""). + * - FAKE_TB_EXIT_CODE: process exit code (default: 0). + * - FAKE_TB_ECHO_ARGV: if set, ignores the above and instead writes + * `{ argv, cwd, env, projectFiles }` (argv/cwd/the env vars tb itself + * reads/the datasources|pipes|connections files found under cwd, as + * "folder/name" paths) as JSON to stdout and exits 0 — used to assert what + * run.ts passed through and what the SDK materialized on disk for tb to + * read. + */ +import { readdirSync } from "node:fs"; +import process from "node:process"; + +function listProjectFiles(cwd) { + const files = []; + for (const folder of ["datasources", "pipes", "connections"]) { + try { + for (const name of readdirSync(`${cwd}/${folder}`)) { + files.push(`${folder}/${name}`); + } + } catch { + // folder doesn't exist for this resource kind — fine. + } + } + return files.sort(); +} + +if (process.env.FAKE_TB_ECHO_ARGV) { + const cwd = process.cwd(); + process.stdout.write( + JSON.stringify({ + argv: process.argv.slice(2), + cwd, + env: { + TB_TOKEN: process.env.TB_TOKEN ?? null, + TB_USER_TOKEN: process.env.TB_USER_TOKEN ?? null, + TB_HOST: process.env.TB_HOST ?? null, + }, + projectFiles: listProjectFiles(cwd), + }) + ); + process.exit(0); +} + +if (process.env.FAKE_TB_STDOUT) { + process.stdout.write(process.env.FAKE_TB_STDOUT); +} +if (process.env.FAKE_TB_STDERR) { + process.stderr.write(process.env.FAKE_TB_STDERR); +} +process.exit(process.env.FAKE_TB_EXIT_CODE ? Number.parseInt(process.env.FAKE_TB_EXIT_CODE, 10) : 0);