Deploy via the tb CLI binary instead of /v1/deploy

src/cli-binary/download.test.ts

@@ -0,0 +1,174 @@
+import { createHash } from "node:crypto";
+import { readFile, mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { gzipSync } from "node:zlib";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import {
+  ChecksumMismatchError,
+  ensureBinaryDownloaded,
+  extractSingleFileTarGz,
+  getCacheDir,
+} from "./download.js";
+import type { ManifestArtifact } from "./manifest.js";
+
+/**
+ * Builds a minimal valid single-entry ustar archive (gzip'd), equivalent to
+ * what Python's `tarfile.open(dest, "w:gz")` + one `tar.add(path, arcname)`
+ * call produces (see package_artifact() in tinybird/tb/modules/binary.py).
+ */
+function buildTarGz(entryName: string, content: Buffer): Buffer {
+  const header = Buffer.alloc(512);
+  header.write(entryName, 0, "ascii");
+  header.write("0000755\0", 100, "ascii"); // mode
+  header.write("0000000\0", 108, "ascii"); // uid
+  header.write("0000000\0", 116, "ascii"); // gid
+  header.write(`${content.length.toString(8).padStart(11, "0")}\0`, 124, "ascii"); // size
+  header.write("00000000000\0", 136, "ascii"); // mtime
+  header.write("        ", 148, "ascii"); // checksum placeholder (8 spaces)
+  header.write("0", 156, "ascii"); // typeflag: regular file
+  header.write("ustar\0", 257, "ascii"); // magic
+  header.write("00", 263, "ascii"); // version
+
+  let checksum = 0;
+  for (const byte of header) checksum += byte;
+  header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, "ascii");
+
+  const contentBlockSize = Math.ceil(content.length / 512) * 512;
+  const contentBlock = Buffer.alloc(contentBlockSize);
+  content.copy(contentBlock);
+
+  const tar = Buffer.concat([header, contentBlock, Buffer.alloc(1024)]); // two zero blocks at EOF
+  return gzipSync(tar);
+}
+
+describe("extractSingleFileTarGz", () => {
+  it("extracts the single file's content", () => {
+    const content = Buffer.from("fake-binary-content");
+    const tarGz = buildTarGz("tb", content);
+    expect(extractSingleFileTarGz(tarGz)).toEqual(content);
+  });
+
+  it("works for content spanning multiple 512-byte blocks", () => {
+    const content = Buffer.from("x".repeat(512 * 2 + 37));
+    const tarGz = buildTarGz("tb", content);
+    expect(extractSingleFileTarGz(tarGz)).toEqual(content);
+  });
+
+  it("throws on a truncated archive", () => {
+    expect(() => extractSingleFileTarGz(gzipSync(Buffer.alloc(10)))).toThrow(/shorter than one header block/);
+  });
+});
+
+describe("getCacheDir", () => {
+  const originalCacheDir = process.env.TB_CLI_CACHE_DIR;
+  const originalXdg = process.env.XDG_CACHE_HOME;
+
+  afterEach(() => {
+    if (originalCacheDir === undefined) delete process.env.TB_CLI_CACHE_DIR;
+    else process.env.TB_CLI_CACHE_DIR = originalCacheDir;
+    if (originalXdg === undefined) delete process.env.XDG_CACHE_HOME;
+    else process.env.XDG_CACHE_HOME = originalXdg;
+  });
+
+  it("honors TB_CLI_CACHE_DIR when set", () => {
+    process.env.TB_CLI_CACHE_DIR = "/tmp/custom-cache";
+    expect(getCacheDir()).toBe("/tmp/custom-cache");
+  });
+
+  it("falls back to XDG_CACHE_HOME/tinybird-cli", () => {
+    delete process.env.TB_CLI_CACHE_DIR;
+    process.env.XDG_CACHE_HOME = "/tmp/xdg-cache";
+    expect(getCacheDir()).toBe(join("/tmp/xdg-cache", "tinybird-cli"));
+  });
+});
+
+describe("ensureBinaryDownloaded", () => {
+  let cacheDir: string;
+
+  beforeEach(async () => {
+    cacheDir = await mkdtemp(join(tmpdir(), "tb-cli-cache-test-"));
+  });
+
+  afterEach(async () => {
+    await rm(cacheDir, { recursive: true, force: true });
+  });
+
+  function artifactFor(content: Buffer): ManifestArtifact {
+    return {
+      id: "linux-x64",
+      os: "linux",
+      arch: "x64",
+      bundle_mode: "onefile",
+      executable_relative_path: "tb",
+      filename: "tb-1.0.0-linux-x64.tar.gz",
+      sha256: createHash("sha256").update(buildTarGz("tb", content)).digest("hex"),
+      size_bytes: content.length,
+      relative_path: "packages/tb-1.0.0-linux-x64.tar.gz",
+      url: "https://example.com/tb-1.0.0-linux-x64.tar.gz",
+    };
+  }
+
+  it("downloads, verifies, extracts, and caches the binary", async () => {
+    const content = Buffer.from("fake-binary");
+    const artifact = artifactFor(content);
+    const tarGz = buildTarGz("tb", content);
+
+    const fetchFn = async () => new Response(new Uint8Array(tarGz), { status: 200 });
+
+    const path = await ensureBinaryDownloaded({
+      version: "1.0.0",
+      artifact,
+      cacheDir,
+      fetchFn: fetchFn as unknown as typeof fetch,
+    });
+
+    expect(path).toBe(join(cacheDir, "1.0.0", "tb"));
+    expect(await readFile(path)).toEqual(content);
+  });
+
+  it("reuses the cached binary on a second call without fetching again", async () => {
+    const content = Buffer.from("fake-binary");
+    const artifact = artifactFor(content);
+    const tarGz = buildTarGz("tb", content);
+
+    let fetchCount = 0;
+    const fetchFn = async () => {
+      fetchCount++;
+      return new Response(new Uint8Array(tarGz), { status: 200 });
+    };
+
+    const options = { version: "1.0.0", artifact, cacheDir, fetchFn: fetchFn as unknown as typeof fetch };
+    await ensureBinaryDownloaded(options);
+    await ensureBinaryDownloaded(options);
+
+    expect(fetchCount).toBe(1);
+  });
+
+  it("throws ChecksumMismatchError when the downloaded tarball doesn't match", async () => {
+    const artifact = artifactFor(Buffer.from("expected"));
+    const fetchFn = async () => new Response(new Uint8Array(buildTarGz("tb", Buffer.from("tampered"))), { status: 200 });
+
+    await expect(
+      ensureBinaryDownloaded({ version: "1.0.0", artifact, cacheDir, fetchFn: fetchFn as unknown as typeof fetch })
+    ).rejects.toThrow(ChecksumMismatchError);
+  });
+
+  it("throws a clear error when the artifact has no download URL", async () => {
+    const artifact = { ...artifactFor(Buffer.from("x")), url: undefined };
+
+    await expect(ensureBinaryDownloaded({ version: "1.0.0", artifact, cacheDir })).rejects.toThrow(
+      /TB_CLI_BINARY_PATH/
+    );
+  });
+
+  it("throws a clear error on a non-OK download response", async () => {
+    const artifact = artifactFor(Buffer.from("x"));
+    const fetchFn = async () => new Response("nope", { status: 500, statusText: "Internal Server Error" });
+
+    await expect(
+      ensureBinaryDownloaded({ version: "1.0.0", artifact, cacheDir, fetchFn: fetchFn as unknown as typeof fetch })
+    ).rejects.toThrow(/500/);
+  });
+});

src/cli-binary/download.ts

@@ -0,0 +1,135 @@
+/**
+ * Downloads, verifies, and caches the `tb` CLI binary tarball described by a
+ * manifest artifact (see manifest.ts). Each tarball is a gzip'd ustar archive
+ * containing exactly one regular file (the `tb`/`tb.exe` executable) — see
+ * `package_artifact()` in tinybird/tb/modules/binary.py (MR !23765), which
+ * builds it with Python's `tarfile.open(destination, "w:gz")` plus a single
+ * `tar.add(artifact_path, arcname=artifact_path.name)` call.
+ */
+
+import { createHash } from "node:crypto";
+import { gunzipSync } from "node:zlib";
+import { chmod, mkdir, rename, writeFile } from "node:fs/promises";
+import { access } from "node:fs/promises";
+import { constants as fsConstants } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+import type { ManifestArtifact } from "./manifest.js";
+
+export class ChecksumMismatchError extends Error {}
+
+function sha256Hex(buffer: Buffer): string {
+  return createHash("sha256").update(buffer).digest("hex");
+}
+
+/**
+ * Reads a single-entry ustar archive (after gunzip) and returns that entry's
+ * content. Not a general-purpose tar reader: it only reads the first header
+ * block, which is all these always-single-file archives contain.
+ */
+export function extractSingleFileTarGz(tarGzBuffer: Buffer): Buffer {
+  const tarBuffer = gunzipSync(tarGzBuffer);
+
+  if (tarBuffer.length < 512) {
+    throw new Error("Invalid tar archive: shorter than one header block");
+  }
+
+  // ustar header: size is a 12-byte octal ASCII field at offset 124.
+  const sizeField = tarBuffer
+    .subarray(124, 124 + 12)
+    .toString("ascii")
+    .replace(/\0.*$/, "")
+    .trim();
+  const size = Number.parseInt(sizeField, 8);
+  if (!Number.isFinite(size) || size < 0) {
+    throw new Error(`Invalid tar header: could not read file size (raw field: ${JSON.stringify(sizeField)})`);
+  }
+
+  const content = tarBuffer.subarray(512, 512 + size);
+  if (content.length !== size) {
+    throw new Error(
+      `Invalid tar archive: expected ${size} bytes of content, found ${content.length}`
+    );
+  }
+
+  return content;
+}
+
+export function getCacheDir(): string {
+  if (process.env.TB_CLI_CACHE_DIR) {
+    return process.env.TB_CLI_CACHE_DIR;
+  }
+  const xdgCacheHome = process.env.XDG_CACHE_HOME;
+  const base = xdgCacheHome && xdgCacheHome.length > 0 ? xdgCacheHome : join(homedir(), ".cache");
+  return join(base, "tinybird-cli");
+}
+
+async function isExecutableFile(path: string): Promise<boolean> {
+  try {
+    await access(path, fsConstants.X_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export interface EnsureBinaryDownloadedOptions {
+  /** Resolved manifest version (not "latest" — the concrete version string). */
+  version: string;
+  artifact: ManifestArtifact;
+  fetchFn?: typeof fetch;
+  cacheDir?: string;
+}
+
+/**
+ * Ensures the binary described by `artifact` is present in the local cache,
+ * downloading and verifying it if necessary, and returns its path.
+ */
+export async function ensureBinaryDownloaded(options: EnsureBinaryDownloadedOptions): Promise<string> {
+  const { version, artifact } = options;
+  const fetchFn = options.fetchFn ?? fetch;
+  const cacheDir = options.cacheDir ?? getCacheDir();
+  const versionDir = join(cacheDir, version);
+  const finalPath = join(versionDir, artifact.executable_relative_path);
+
+  if (await isExecutableFile(finalPath)) {
+    return finalPath;
+  }
+
+  if (!artifact.url) {
+    throw new Error(
+      `Manifest artifact for ${artifact.os}-${artifact.arch} has no download URL ` +
+        `(manifest was generated without --base-url). Set TB_CLI_BINARY_PATH to use a local binary instead.`
+    );
+  }
+
+  const response = await fetchFn(artifact.url);
+  if (!response.ok) {
+    throw new Error(
+      `Failed to download Tinybird CLI binary from ${artifact.url}: ${response.status} ${response.statusText}`
+    );
+  }
+
+  const tarGzBuffer = Buffer.from(await response.arrayBuffer());
+
+  const actualSha256 = sha256Hex(tarGzBuffer);
+  if (actualSha256 !== artifact.sha256) {
+    throw new ChecksumMismatchError(
+      `Checksum mismatch for Tinybird CLI binary ${artifact.filename}: ` +
+        `expected ${artifact.sha256}, got ${actualSha256}`
+    );
+  }
+
+  const executable = extractSingleFileTarGz(tarGzBuffer);
+
+  await mkdir(versionDir, { recursive: true });
+  // Download to a unique temp name and rename into place so concurrent
+  // processes racing to populate the same cache dir never see a partial file.
+  const tempPath = `${finalPath}.download-${process.pid}-${Date.now()}`;
+  await writeFile(tempPath, executable);
+  await chmod(tempPath, 0o755);
+  await rename(tempPath, finalPath);
+
+  return finalPath;
+}

src/cli-binary/index.ts

@@ -0,0 +1,74 @@
+/**
+ * Resolves a usable `tb` binary path: an explicit TB_CLI_BINARY_PATH override,
+ * or the manifest-published binary for the current platform, downloaded and
+ * cached on demand.
+ *
+ * DRAFT: there is no published manifest yet (the MR that builds these
+ * binaries, https://gitlab.com/tinybird/analytics/-/merge_requests/23765,
+ * hasn't merged). This module is unit-tested against fakes only; nothing here
+ * has been exercised against a real manifest or a real binary.
+ */
+
+export * from "./platform.js";
+export * from "./manifest.js";
+export * from "./download.js";
+export * from "./run.js";
+
+import { getHostPlatform } from "./platform.js";
+import { fetchManifest, findArtifactForHost } from "./manifest.js";
+import { ensureBinaryDownloaded } from "./download.js";
+
+/**
+ * `tb` version this SDK is tested against. "latest" is a placeholder until a
+ * concrete version has actually been validated end-to-end (see plan: pin
+ * deliberately once that validation happens, don't float on latest forever).
+ */
+export const DEFAULT_TB_CLI_VERSION = "latest";
+
+export class CliBinaryUnavailableError extends Error {}
+
+export interface ResolveTbBinaryOptions {
+  version?: string;
+  baseUrl?: string;
+  fetchFn?: typeof fetch;
+  cacheDir?: string;
+}
+
+export async function resolveTbBinaryPath(options: ResolveTbBinaryOptions = {}): Promise<string> {
+  const override = process.env.TB_CLI_BINARY_PATH;
+  if (override) {
+    return override;
+  }
+
+  const host = getHostPlatform();
+  const requestedVersion = options.version ?? process.env.TB_CLI_VERSION ?? DEFAULT_TB_CLI_VERSION;
+
+  const manifest = await fetchManifest({
+    baseUrl: options.baseUrl,
+    version: requestedVersion,
+    fetchFn: options.fetchFn,
+  }).catch((error: unknown) => {
+    throw new CliBinaryUnavailableError(
+      `Could not fetch Tinybird CLI manifest: ${(error as Error).message}`
+    );
+  });
+
+  const artifact = findArtifactForHost(manifest, host);
+  if (!artifact) {
+    throw new CliBinaryUnavailableError(
+      `No Tinybird CLI binary is published for ${host.os}-${host.arch}. ` +
+        `Set TB_CLI_BINARY_PATH to point at a local tb executable instead.`
+    );
+  }
+
+  return ensureBinaryDownloaded({
+    version: manifest.version,
+    artifact,
+    fetchFn: options.fetchFn,
+    cacheDir: options.cacheDir,
+  }).catch((error: unknown) => {
+    throw new CliBinaryUnavailableError(
+      `Could not download Tinybird CLI binary: ${(error as Error).message}`
+    );
+  });
+}