From 5e954f4e711957c3733df86a555e09a0dc59c24d Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Tue, 27 Oct 2026 16:07:19 -0700 Subject: [PATCH 01/13] feat(hub): add provider-neutral artifact ingestion Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- hub/.env.example | 3 + hub/docs/README.md | 1 + hub/docs/architecture.md | 6 +- hub/docs/artifact-ingestion.md | 84 +++ hub/docs/database.md | 6 +- hub/docs/identity.md | 10 +- hub/docs/logging.md | 4 +- hub/docs/prototype-authorization.md | 28 +- hub/docs/testing-and-operations.md | 4 +- hub/package.json | 1 + hub/schema/artifact-publication.prisma | 24 + .../migration.sql | 125 ++++ hub/schema/prototype-channel.prisma | 21 + hub/schema/prototype-version.prisma | 32 +- hub/schema/prototype.prisma | 11 +- hub/schema/security-audit.prisma | 2 + hub/schema/workload-credential.prisma | 20 + hub/schema/workload-publisher.prisma | 23 + hub/script/publish-artifact.mjs | 70 +++ hub/server/create-app/index.ts | 3 + .../create-service/constants.ts | 2 + .../create-service/index.db.test.ts | 595 ++++++++++++++++++ .../create-service/index.ts | 375 +++++++++++ .../create-service/types.ts | 38 ++ .../authentication/create-runtime/index.ts | 11 + .../create-service/index.db.test.ts | 118 ---- .../create-service/index.ts | 174 +++-- .../create-service/types.ts | 6 +- .../create-local-storage/index.test.ts | 67 ++ .../create-local-storage/index.ts | 96 +++ .../integrations/artifact-storage/types.ts | 10 + .../security-audits/save-audit/index.ts | 1 + .../repositories/security-audits/types.ts | 2 + hub/server/routes/index.ts | 36 +- hub/server/routes/product/index.test.ts | 14 +- hub/server/routes/product/index.ts | 26 +- hub/server/routes/publishing.test.ts | 235 +++++++ hub/server/routes/publishing/index.ts | 58 ++ hub/server/routes/types.ts | 2 + 39 files changed, 2115 insertions(+), 229 deletions(-) create mode 100644 hub/docs/artifact-ingestion.md create mode 100644 hub/schema/artifact-publication.prisma create mode 100644 hub/schema/migrations/20261027230000_artifact_ingestion/migration.sql create mode 100644 hub/schema/prototype-channel.prisma create mode 100644 hub/schema/workload-credential.prisma create mode 100644 hub/schema/workload-publisher.prisma create mode 100644 hub/script/publish-artifact.mjs create mode 100644 hub/server/domains/artifact-publishing/create-service/constants.ts create mode 100644 hub/server/domains/artifact-publishing/create-service/index.db.test.ts create mode 100644 hub/server/domains/artifact-publishing/create-service/index.ts create mode 100644 hub/server/domains/artifact-publishing/create-service/types.ts create mode 100644 hub/server/integrations/artifact-storage/create-local-storage/index.test.ts create mode 100644 hub/server/integrations/artifact-storage/create-local-storage/index.ts create mode 100644 hub/server/integrations/artifact-storage/types.ts create mode 100644 hub/server/routes/publishing.test.ts create mode 100644 hub/server/routes/publishing/index.ts diff --git a/hub/.env.example b/hub/.env.example index ec013451..b5e26553 100644 --- a/hub/.env.example +++ b/hub/.env.example @@ -20,6 +20,9 @@ DATABASE_URL=postgresql://replica:replica@127.0.0.1:5432/replica # Only database-enabled tests require this URL and CREATEDB permission. TEST_DATABASE_URL=postgresql://replica:replica@127.0.0.1:5432/postgres +# Local reference artifact storage. Production deployments should mount durable storage here. +# HUB_ARTIFACT_STORAGE_PATH=/var/lib/replica/artifacts + # GitHub user authorization is optional until both values are configured. # These credentials establish user identity and are separate from source integration. # HUB_GITHUB_AUTH_CLIENT_ID= diff --git a/hub/docs/README.md b/hub/docs/README.md index 9581e278..a8a27e53 100644 --- a/hub/docs/README.md +++ b/hub/docs/README.md @@ -11,6 +11,7 @@ Hub is being rebuilt. The [Hub overview](../README.md) describes what the baseli - [GitHub integration](github-integration.md): linked repository sources, reconciliation, webhooks, staleness, and permission mapping. - [Membership and administration](membership-and-administration.md): Hub admission, invitations, roles, lifecycle, bootstrap, and recovery. - [Prototype authorization](prototype-authorization.md): workspaces, prototypes, immutable versions, Hub overrides, synchronized baselines, and policy precedence. +- [Artifact ingestion](artifact-ingestion.md): workload publishers, the versioned manifest, storage, immutable publishing, idempotency, and promotion. - [Logging](logging.md): application events, automatic request records, and payload safety. - [Background jobs](background-jobs.md): PostgreSQL queues, workers, schedules, setup, and test isolation. - [Testing and operations](testing-and-operations.md): baseline checks and their limits. diff --git a/hub/docs/architecture.md b/hub/docs/architecture.md index 4c94dfe6..9db2665c 100644 --- a/hub/docs/architecture.md +++ b/hub/docs/architecture.md @@ -6,15 +6,15 @@ Hub contributors are rebuilding the app on `build/hub/new` before merging it int The team has chosen TypeScript, Express for the API, React and Vite for the frontend, and Vitest for tests. The old backend and the frontend code that connected to it have been removed rather than rewritten. -The API responds to a health check and provides GitHub authentication, Hub sessions, installation membership administration, Hub-native prototype authorization, and opt-in GitHub-synchronized admission and prototype baselines. The frontend displays sign-in, administrator enrollment and recovery, and a minimal membership administration page with light and dark themes. Storybook keeps the existing components and pages available for reuse. +The API responds to a health check and provides GitHub authentication, Hub sessions, installation membership administration, Hub-native prototype authorization, opt-in GitHub-synchronized admission and prototype baselines, and provider-neutral artifact publishing. The frontend displays sign-in, administrator enrollment and recovery, and a minimal membership administration page with light and dark themes. Storybook keeps the existing components and pages available for reuse. -Contributors can use PostgreSQL and Prisma locally or in the devcontainer. The [identity system](identity.md) separates Hub users, external identities, installation membership, authentication-provider configuration, sessions, authorization attempts, security records, and GitHub source-integration credentials. GitHub user authorization, Hub-owned admission, provider-neutral sessions, invitations, membership lifecycle, and administrator recovery work when user-auth credentials are configured. The [background-job foundation](background-jobs.md) provides PostgreSQL-backed queues and workers without product jobs. The [prototype authorization system](prototype-authorization.md) adds workspaces, prototypes, immutable version records, direct roles, explicit deny, synchronized GitHub baselines, and one policy evaluator. The [GitHub integration](github-integration.md) stores stable repository source records and reconciles current permissions with installation credentials. Deployment has not been built. This app does not ingest artifacts, host previews, browse source, or support reviews. +Contributors can use PostgreSQL and Prisma locally or in the devcontainer. The [identity system](identity.md) separates Hub users, external identities, installation membership, authentication-provider configuration, sessions, authorization attempts, security records, and GitHub source-integration credentials. GitHub user authorization, Hub-owned admission, provider-neutral sessions, invitations, membership lifecycle, and administrator recovery work when user-auth credentials are configured. The [background-job foundation](background-jobs.md) provides PostgreSQL-backed queues and workers without product jobs. The [prototype authorization system](prototype-authorization.md) adds workspaces, prototypes, immutable version records, direct roles, explicit deny, synchronized GitHub baselines, and one policy evaluator. The [GitHub integration](github-integration.md) stores stable repository source records and reconciles current permissions with installation credentials. The [artifact ingestion system](artifact-ingestion.md) authenticates prototype-scoped workloads, validates a shared manifest, stores bytes behind an adapter, creates immutable versions, and moves promotion pointers. Deployment has not been built. This app does not host previews, browse source, or support reviews. ## Backend API code lives under `server/`. As features are added, route handlers will read requests, call domains, and return responses. The [code conventions](backend-architecture.md) explain repositories, integrations, utilities, domains, and which layers may import others. Put these directories under `server/` when they are needed; do not add empty implementations. -`GET /api/health` returns `{ "status": "ok" }`. Authentication routes start and complete GitHub authorization, restore or revoke Hub sessions, and perform fresh verification. Administration routes list memberships and invitations, create or revoke invitations, and change membership roles or lifecycle. Product routes create and read workspaces, prototypes, and immutable versions; archive prototypes; choose Hub-managed or GitHub-synchronized prototype authorization; explain effective access; and administer direct access rules. Administration routes link repository sources, opt them into admission, inspect synchronization state, and trigger reconciliation. A verified GitHub App webhook route marks affected sources dirty or removed. Unknown paths return a JSON 404 response; invalid JSON is rejected before routing. `createApp()` builds the Express app without starting a server. `server/index.ts` composes the authentication, administration, prototype authorization, and GitHub synchronization runtime, starts the API at a local address, and schedules bounded periodic reconciliation. +`GET /api/health` returns `{ "status": "ok" }`. Authentication routes start and complete GitHub authorization, restore or revoke Hub sessions, and perform fresh verification. Administration routes list memberships and invitations, create or revoke invitations, change membership roles or lifecycle, link repository sources, opt them into admission, inspect synchronization state, and trigger reconciliation. Product routes create and read workspaces and prototypes; read immutable versions; provision workload publishers; archive prototypes; choose Hub-managed or GitHub-synchronized prototype authorization; explain effective access; and administer direct access rules. Publishing routes ingest artifacts, read idempotent status, and move `current` or `preview`. A verified GitHub App webhook route marks affected sources dirty or removed. Unknown paths return a JSON 404 response; invalid JSON is rejected before routing. `createApp()` builds the Express app without starting a server. `server/index.ts` composes the authentication, administration, prototype authorization, GitHub synchronization, and artifact publishing runtime, starts the API at a local address, and schedules bounded periodic reconciliation. The [database guide](database.md) defines the multi-file Prisma schema and shared client in integrations. The health route remains a process check, not a database readiness check. diff --git a/hub/docs/artifact-ingestion.md b/hub/docs/artifact-ingestion.md new file mode 100644 index 00000000..b2c5bdd1 --- /dev/null +++ b/hub/docs/artifact-ingestion.md @@ -0,0 +1,84 @@ +# Artifact ingestion and prototype versions + +Hub accepts one provider-neutral artifact contract from workload producers. GitHub Actions, Azure DevOps, Jenkins, local scripts, Copilot workflows, and other systems use the same API. Producers build outside Hub; Hub authenticates the workload, validates and stores the bytes, creates an immutable prototype version, records provenance, and may move a channel pointer. + +## Identity and access + +A prototype administrator creates a workload publisher with one or both actions: + +| Action | Authority | +| ------------- | ---------------------------------------------------------------------------- | +| `publish` | Publish an artifact and move the prototype's `current` or `preview` channel. | +| `read-status` | Read the result associated with that publisher's idempotency key. | + +Hub returns a bearer credential once and stores only its SHA-256 digest. The credential is scoped to one prototype through its workload publisher. It cannot create a browser session, view a prototype through user APIs, administer access, or publish to another prototype. A producer's source URL, revision, ref, pipeline, and run ID describe where a build came from; they never grant access. + +Browser users provision credentials through `POST /api/prototypes/:prototypeId/publishers`. This requires a fresh prototype-administrator session and returns the raw token only in that response. + +## Manifest contract + +`POST /api/publishing/prototypes/:prototypeId/versions` requires `Authorization: Bearer ...`, an `Idempotency-Key` header, and this JSON shape: + +```json +{ + "manifest": { + "schemaVersion": 1, + "prototypeId": "00000000-0000-0000-0000-000000000000", + "artifact": { + "format": "replica.prototype.bundle", + "version": "1", + "digest": "sha256:<64 lowercase hexadecimal characters>", + "sizeBytes": 1234, + "builtAt": "2026-10-27T20:00:00.000Z" + }, + "provenance": { + "provider": "jenkins", + "sourceUrl": "https://ci.example/jobs/prototype", + "revision": "abc123", + "ref": "refs/heads/main", + "pipeline": "prototype-build", + "runId": "451" + } + }, + "artifactBase64": "", + "promoteChannel": "preview" +} +``` + +The manifest is closed to unknown fields. Hub bounds every string, accepts only HTTP or HTTPS source URLs, limits decoded artifacts to 8 MiB in this reference layer, verifies the decoded size, and recalculates the SHA-256 digest before storing bytes. The manifest prototype ID must match the route. + +## Storage boundary + +`ArtifactStorage` separates ingestion from byte storage. The reference adapter writes content-addressed files under `HUB_ARTIFACT_STORAGE_PATH`, defaulting to `hub/.data/artifacts` for local use. Database rows store only the storage key and validated metadata, never the artifact bytes. + +The local adapter is suitable for one Hub process with a durable mounted filesystem. It does not provide replication, remote durability, encryption-key management, or multi-region consistency. A production object-store adapter must preserve the same content-addressed, immutable `put`, `read`, and `remove` boundary. + +Storage happens before the database transaction. A failed transaction can therefore leave an unreferenced content-addressed file. Hub never deletes referenced bytes automatically. Operators may remove only files that have no matching `PrototypeVersion.storageKey`; retain unreferenced files long enough to exclude in-flight transactions and backups. Version metadata has no deletion API and must not be rewritten silently. + +## Idempotency and immutability + +Hub hashes the idempotency key and stores it with a digest of the validated publication request. Retrying the same request with the same publisher and key returns the existing version. Reusing the key for different manifest or promotion input returns `idempotency_conflict`. Publishing the same content digest again to the prototype returns `artifact_already_published`. + +Publishing locks the prototype while it assigns the next sequence. Concurrent requests therefore create one ordered history. PostgreSQL rejects every update or deletion of a prototype-version row, including accidental writes outside the application. + +## Promotion and rollback + +`current` and `preview` are mutable pointers to immutable versions. A publish request may move one pointer in the same transaction that creates the version. `PUT /api/publishing/prototypes/:prototypeId/channels/:channel` moves a pointer to any version owned by that prototype. Rollback uses that endpoint to point the channel at an earlier version; it never mutates the version. + +## Status, audits, and failures + +`GET /api/publishing/prototypes/:prototypeId/status` reads the publication associated with the caller's `Idempotency-Key`. It requires `read-status`. + +Version creation, idempotency ownership, optional promotion, and the `prototype.version.publish` security audit commit in one transaction. Promotion writes `prototype.channel.promote` in its transaction. Audit failure rolls back database state. Responses expose validated version metadata and stable error codes, not credentials, storage paths, or artifact contents. Publishing routes omit headers and request payloads from structured request logs. + +## Local producer + +The local CLI proves that publishing does not depend on GitHub: + +```sh +cd hub +REPLICA_PUBLISH_TOKEN=... npm run publish:artifact -- \ + http://127.0.0.1:3001 PROTOTYPE_ID ./prototype.tar preview +``` + +Optional `REPLICA_PROVENANCE_PROVIDER`, `REPLICA_SOURCE_URL`, `REPLICA_REVISION`, `REPLICA_REF`, `REPLICA_PIPELINE`, and `REPLICA_RUN_ID` values populate provenance without changing authorization. diff --git a/hub/docs/database.md b/hub/docs/database.md index 9044c181..faf0ed19 100644 --- a/hub/docs/database.md +++ b/hub/docs/database.md @@ -1,6 +1,6 @@ # Hub database -Hub uses PostgreSQL and Prisma. The schema stores the provider-neutral [identity foundation](identity.md) and the [prototype authorization](prototype-authorization.md) product core. +Hub uses PostgreSQL and Prisma. The schema stores the provider-neutral [identity foundation](identity.md), the [prototype authorization](prototype-authorization.md) product core, and [artifact publication metadata](artifact-ingestion.md). [Background jobs](background-jobs.md) use the same database with BullMQ-owned tables and functions in the `bullmq` schema. Prisma does not own those objects. `npm run setup` applies Prisma migrations before BullMQ setup and schedule registration. @@ -57,8 +57,10 @@ GitHub synchronization uses separate records: - `GitHubSynchronizedAccess` stores one user's raw permission, mapped prototype role, admission eligibility, source, status, and last confirmation for one repository source. - `GitHubAuthorizationPolicy` stores the bounded last-known-good duration. The initial policy is one day and may not exceed seven days. - `GitHubWebhookDelivery` deduplicates verified deliveries by GitHub delivery ID. - `Prototype.authorizationMode` defaults to `hub-managed`. `sourceRepositoryId` is nullable and applies only when the mode is explicitly `github-synchronized`. Synchronized records never share rows with workspace or prototype grants, so reconciliation cannot overwrite manual Hub decisions. +`Prototype.authorizationMode` defaults to `hub-managed`. `sourceRepositoryId` is nullable and applies only when the mode is explicitly `github-synchronized`. Synchronized records never share rows with workspace or prototype grants, so reconciliation cannot overwrite manual Hub decisions. + +A database trigger rejects prototype version updates and deletions so history cannot be rewritten or silently removed. Workload credentials store token digests only. Publication rows bind a publisher-scoped idempotency-key digest to one version, and channel rows keep mutable pointers separate from immutable version metadata. Artifact bytes stay outside relational columns behind the storage boundary. Installation membership `source` distinguishes linked-source synchronization from the `legacy-synchronized` compatibility policy. Any administrator lifecycle change or assignment of Hub administration converts a synchronized membership to `manual`, so reconciliation cannot undo the Hub override or orphan administration. diff --git a/hub/docs/identity.md b/hub/docs/identity.md index f2507c05..cfafc856 100644 --- a/hub/docs/identity.md +++ b/hub/docs/identity.md @@ -18,6 +18,8 @@ Hub sessions use a random opaque cookie and store only its digest. Authenticatio `legacy-repository` preserves the previous compatibility behavior. It checks explicit collaboration on one of the first 100 private installed repositories within a ten-second total deadline and creates a `legacy-synchronized` member assignment, without requiring linked source records. Installations beyond that bound must use Hub-managed or GitHub-synchronized admission. The migration backfills pre-existing synchronized compatibility memberships to that source before linked-source synchronization begins. Linked-source reconciliation does not manage those compatibility assignments. It is not the normal synchronized authorization mode. The GitHub authentication adapter itself never reads installation credentials or performs repository authorization. +Artifact publishing uses a separate workload identity boundary. Prototype administrators issue prototype-scoped credentials with `publish` and/or `read-status`; Hub returns the raw bearer token once and stores only its digest. Workload credentials never create browser sessions or inherit user, installation, repository, or source-provider access. See [Artifact ingestion](artifact-ingestion.md). + Sign-out revokes the current database session. Every restored session requires an active installation membership. Suspension and removal therefore deny retained cookies immediately, and their administration operations delete the affected user's sessions. Fresh verification reuses GitHub authorization and must return the same external identity. An active Hub administrator membership with fresh verification may revoke all sessions; revocation also supersedes callbacks that started earlier. ## Persisted boundaries @@ -45,10 +47,10 @@ Administrator mutations require an active administrator session verified less th GitHub source access is an optional integration boundary, not the global identity boundary. Set `HUB_GITHUB_CREDENTIAL_PROVIDER` in the [environment example](../.env.example): -| Provider | Credentials | -| --- | --- | +| Provider | Credentials | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `database` (default) | Generate `HUB_GITHUB_CREDENTIAL_KEY` with `openssl rand -hex 32`. The provider encrypts GitHub integration credentials before storage and authenticates their context when reading. | -| `environment` | Set every `GITHUB_*` value, including `GITHUB_INSTALLATION_ID`. This provider is read-only and ignores database credentials. | +| `environment` | Set every `GITHUB_*` value, including `GITHUB_INSTALLATION_ID`. This provider is read-only and ignores database credentials. | Both providers currently validate an App ID, installation ID, client ID, client secret, and RSA private key. IDs are positive decimal strings. The key must contain at least 2048 bits. A deployment may use the same GitHub App for user authorization and source integration, but it supplies and rotates the two configurations independently. Set `HUB_GITHUB_WEBHOOK_SECRET` separately when enabling the webhook route. @@ -56,6 +58,6 @@ Database credentials use AES-256-GCM with contextual additional authenticated da ## Current scope -This layer adds GitHub sign-in, Hub-managed and opt-in GitHub-synchronized installation admission, invitations, member and administrator lifecycle, one-use bootstrap and recovery, session restoration, sign-out, fresh verification, global and per-member session revocation, structured security records, and the minimal frontend needed to use them. The separate [prototype authorization](prototype-authorization.md) boundary may consume a synchronized repository baseline without treating installation membership as product authority. Identity does not add artifact ingestion, production OIDC, account linking, or SCIM. +This layer adds GitHub sign-in, Hub-managed and opt-in GitHub-synchronized installation admission, invitations, member and administrator lifecycle, one-use bootstrap and recovery, session restoration, sign-out, fresh verification, global and per-member session revocation, structured security records, and the minimal frontend needed to use them. The separate [prototype authorization](prototype-authorization.md) boundary may consume a synchronized repository baseline without treating installation membership as product authority. Artifact ingestion composes with identity only when a browser administrator provisions a workload credential; publication itself does not use a browser session. Identity does not add production OIDC, account linking, or SCIM. Apply the rewritten initial migration with `npm run db:deploy`. See [Database](database.md) for setup and schema rules. diff --git a/hub/docs/logging.md b/hub/docs/logging.md index e66f3d3c..e30872e6 100644 --- a/hub/docs/logging.md +++ b/hub/docs/logging.md @@ -30,7 +30,7 @@ Each request produces one final record with `reqId`, `method`, `route`, `statusC Automatic records exclude raw URLs, headers, cookies, uploads, response bodies, and error details. JSON parsing uses Express's default 100 KiB limit; rejected bodies are not logged. The Express error handler returns generic JSON errors, or closes the connection if headers were sent, without exposing raw errors. -`createApp()` installs `omitRequestLogPayload` for authentication and session paths after request logging but before JSON parsing. This ordering covers parser failures as well as later origin checks, rate limits, and handlers: +`createApp()` installs `omitRequestLogPayload` for authentication, session, administration, product, and publishing paths after request logging but before JSON parsing. This ordering covers parser failures as well as later origin checks, rate limits, and handlers: ```ts import { omitRequestLogPayload } from "../integrations/pino/omit-request-log-payload/index.ts" @@ -40,7 +40,7 @@ app.use(["/api/auth", "/api/session"], omitRequestLogPayload) This omits `params`, `query`, and `body` while keeping request metadata. -OAuth codes, state values, CSRF values, cookies, PKCE material, provider responses, and unrelated private fields therefore never enter automatic request payload records, including when JSON is malformed or too large. +OAuth codes, state values, CSRF values, cookies, PKCE material, provider responses, workload credentials, artifact manifests, and artifact bytes therefore never enter automatic request payload records, including when JSON is malformed or too large. Automatic records never include request headers, so bearer credentials are omitted as well. ## Structured-field safety diff --git a/hub/docs/prototype-authorization.md b/hub/docs/prototype-authorization.md index 5ead8eca..1bf32fe2 100644 --- a/hub/docs/prototype-authorization.md +++ b/hub/docs/prototype-authorization.md @@ -8,20 +8,21 @@ A workspace owns prototypes. A prototype owns an ordered history of immutable ve - An active installation administrator may create a workspace. Creation also writes an explicit workspace `administrator` grant for that user in the same transaction. - A workspace `editor` or `administrator` may create a prototype. -- A prototype `editor` or `administrator` may create its next version. -- A version cannot move to another prototype or be changed after creation. -- Archiving a prototype keeps its versions readable but prevents new versions. Archive and version creation lock and re-read the prototype so overlapping requests cannot add a version after archival. +- A prototype administrator may provision a prototype-scoped workload publisher. +- An authenticated workload publisher with `publish` creates the next version through the [artifact ingestion contract](artifact-ingestion.md). +- A version cannot move to another prototype or be changed or deleted after creation. +- Archiving a prototype keeps its versions readable but prevents new versions. Archive and artifact publication lock and re-read the prototype so overlapping requests cannot add a version after archival. -Artifact payloads are not part of this layer. A version identifies immutable prototype history without storing a build, manifest, or upload. +Artifact payload validation and storage are owned by the separate [artifact ingestion layer](artifact-ingestion.md). User authorization controls who can provision the workload boundary and read versions; source provenance does not affect either decision. ## Roles -| Role | Product authority | -| --- | --- | -| `viewer` | Read the authorized workspace, prototype, and versions. | -| `commenter` | Includes viewer authority and reserves the role for future review comments. | -| `editor` | Includes commenter authority, creates prototypes within an authorized workspace, creates versions, and archives prototypes. | -| `administrator` | Includes editor authority and changes direct access rules. | +| Role | Product authority | +| --------------- | --------------------------------------------------------------------------------------------------------------------------- | +| `viewer` | Read the authorized workspace, prototype, and versions. | +| `commenter` | Includes viewer authority and reserves the role for future review comments. | +| `editor` | Includes commenter authority, creates prototypes within an authorized workspace, and archives prototypes. | +| `administrator` | Includes editor authority and changes direct access rules. | Installation membership and installation administrator status do not grant access to existing workspaces or prototypes. The workspace-creation rule is the only installation-administrator product rule in this layer. After creation, the user's explicit workspace grant supplies authority. @@ -73,8 +74,9 @@ The product API uses: - `PUT /api/prototypes/:prototypeId/authorization` - `GET /api/prototypes/:prototypeId/access/explanation` - `PUT` and `DELETE /api/prototypes/:prototypeId/access/:userId` -- `GET` and `POST /api/prototypes/:prototypeId/versions` +- `GET /api/prototypes/:prototypeId/versions` - `GET /api/prototypes/:prototypeId/versions/:versionId` +- `POST /api/prototypes/:prototypeId/publishers` List routes use an opaque UUID cursor and a limit from 1 through 100. Prototype and version cursors belong to their owning workspace or prototype; a cursor from another scope is invalid. Invalid pagination returns `422 invalid_pagination`, while malformed or missing path resources return their resource-specific `404` error. State-changing requests require the configured origin and matching CSRF cookie and header. Product responses are private and non-cacheable, request payloads are omitted from logs, and mutation routes use a bounded rate limit. @@ -82,7 +84,7 @@ Every product route restores the current session and active installation members ## Audits -Workspace creation, prototype creation and archival, version creation, role grants, explicit denies, and rule removal write structured security audits in the same transaction as the product change. A failed audit rolls back the product change. +Workspace creation, prototype creation and archival, workload publisher provisioning, role grants, explicit denies, and rule removal write structured security audits in the same transaction as the product change. Artifact publication writes its workload audit in the publication transaction. A failed audit rolls back the product change. Records identify the actor, session, authentication-provider revision, affected product records, target user when applicable, outcome, and a bounded reason. They do not include cookies, provider tokens, host secrets, request bodies, prototype artifacts, or mutable identity labels. @@ -92,4 +94,4 @@ The [GitHub integration](github-integration.md) confirms permissions with instal Repository rename changes mutable owner, name, and default-branch fields without changing the immutable source identity. Repository removal, installation removal, and permission loss update synchronized baselines only. Direct Hub overrides remain intact. -Source mutation, public links, artifact ingestion, promotion, retention, and rollback remain separate product boundaries. +Source mutation and public links remain separate product boundaries. Artifact ingestion, promotion, retention, and rollback follow the provider-neutral contract without changing user access rules. diff --git a/hub/docs/testing-and-operations.md b/hub/docs/testing-and-operations.md index 4a24ff39..900be503 100644 --- a/hub/docs/testing-and-operations.md +++ b/hub/docs/testing-and-operations.md @@ -57,12 +57,14 @@ The database command uses `--passWithNoTests` so an empty selection exits succes A dedicated migration case applies the foundation migration, seeds retained lifecycle and security records, then applies the next migration. This catches upgrade behavior that an empty final schema cannot prove, including audit-outcome backfills and lifecycle constraints. -Prototype authorization cases cover direct-rule precedence, explicit deny, no-access defaults, GitHub baseline mapping and staleness, prototype-specific restrictions, cross-prototype and cross-repository isolation, installation membership versus product authority, stale sessions, deterministic overlapping administrator changes, membership lifecycle conflicts, audit rollback, route enforcement, scalar list responses, deny-aware and terminal pagination, version ownership, archive/version races, and explainable decisions. +Prototype authorization cases cover direct-rule precedence, explicit deny, no-access defaults, GitHub baseline mapping and staleness, prototype-specific restrictions, cross-prototype and cross-repository isolation, installation membership versus product authority, stale sessions, deterministic overlapping administrator changes, membership lifecycle conflicts, audit rollback, route enforcement, scalar list responses, deny-aware and terminal pagination, version ownership, workload credential revocation, publisher suspension, archive lifecycle, and explainable decisions. Artifact publishing cases cover manifest bounds, credential-free provenance URLs, digest and size verification, scoped workload actions, token non-storage, idempotent replay and conflicts, concurrent sequence assignment, immutable rows, atomic promotion and rollback, revocation and disable races, cross-prototype isolation, audit rollback, body limits, request-log omission, and the local non-GitHub producer contract. Local storage cases reject symbolic-link roots, shards, and artifact paths. GitHub synchronization cases use mocked GitHub API responses and real database transactions. They cover transactional cache revalidation, invitation precedence during source outage, immutable-ID rename and 404 invalidation, token-404 installation removal, credential-independent webhook revocation, retryable source-generation changes, per-result confirmation ordering, active-identity filtering, bounded provider requests and apply batches, partial scans, dirty-event retry coalescing, first-enrollment and admission-failure auditing, repository and installation lifecycle, provider-response versus apply failures, scheduled failure logging, stale per-user and overlapping reconciliation or sign-in, manual suspension and audited administration preservation, compatibility membership isolation, synchronized session freshness, and rollback. The API starts reconciliation every 15 minutes and operators can trigger one repository immediately through the administration API. A source stores `dirty`, `current`, `failed`, or `removed`, its last attempt and confirmation, and a bounded error code. Provider failures leave the prior confirmation unchanged. Access fails closed after the one-day last-known-good window, even if no webhook reported the change. Monitor repeated `failed` states and stale confirmations; do not extend the window to hide an outage. +The reference artifact adapter stores content-addressed files under `HUB_ARTIFACT_STORAGE_PATH`. Give the test process a temporary absolute directory. Production deployments must mount durable storage and monitor capacity; the reference adapter has no replication or automatic garbage collection. Before deleting an unreferenced file, confirm that no `PrototypeVersion.storageKey` points to it and that no publish transaction or backup can still require it. + Database-enabled jsdom tests use the browser setup and database hooks together. They can arrange records through server code, but frontend code must not import server modules. The setup does not start an API or connect frontend requests to one. The database setup follows this lifecycle: diff --git a/hub/package.json b/hub/package.json index 19a98b76..625da5f8 100644 --- a/hub/package.json +++ b/hub/package.json @@ -47,6 +47,7 @@ "dev:api": "node --conditions=development --watch server/index.ts", "start": "node dist/server/index.js", "preview": "vite preview --host 127.0.0.1", + "publish:artifact": "node script/publish-artifact.mjs", "extension:assets:check": "node ../.github/extensions/replica-prototype/check-assets/index.mjs", "extension:build": "npm run extension:modules:build && vite build --config vite.prototype-canvas.config.mjs && node script/build-prototype-preview-bridge.mjs", "extension:modules:build": "node script/build-extension-modules.mjs", diff --git a/hub/schema/artifact-publication.prisma b/hub/schema/artifact-publication.prisma new file mode 100644 index 00000000..9b42b51f --- /dev/null +++ b/hub/schema/artifact-publication.prisma @@ -0,0 +1,24 @@ +model ArtifactPublication { + // ID columns + id String @id @default(uuid()) @db.Uuid + prototypeId String @map("prototype_id") @db.Uuid + workloadPublisherId String @map("workload_publisher_id") @db.Uuid + prototypeVersionId String @unique @map("prototype_version_id") @db.Uuid + + // Relations + prototype Prototype @relation(fields: [prototypeId], references: [id], onDelete: Restrict) + workloadPublisher WorkloadPublisher @relation(fields: [workloadPublisherId], references: [id], onDelete: Restrict) + prototypeVersion PrototypeVersion @relation(fields: [prototypeVersionId], references: [id], onDelete: Restrict) + + // Attribute columns + idempotencyKeyDigest String @map("idempotency_key_digest") + requestDigest String @map("request_digest") + + // Timestamp columns + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3) + + @@unique([workloadPublisherId, idempotencyKeyDigest]) + @@index([prototypeId]) + @@map("artifact_publications") +} diff --git a/hub/schema/migrations/20261027230000_artifact_ingestion/migration.sql b/hub/schema/migrations/20261027230000_artifact_ingestion/migration.sql new file mode 100644 index 00000000..baf29a19 --- /dev/null +++ b/hub/schema/migrations/20261027230000_artifact_ingestion/migration.sql @@ -0,0 +1,125 @@ +-- CreateTable +CREATE TABLE "workload_publishers" ( + "id" UUID NOT NULL, + "prototype_id" UUID NOT NULL, + "name" TEXT NOT NULL, + "actions" TEXT[] NOT NULL, + "active" BOOLEAN NOT NULL DEFAULT true, + "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "workload_publishers_pkey" PRIMARY KEY ("id"), + CONSTRAINT "workload_publishers_actions" CHECK ( + cardinality("actions") BETWEEN 1 AND 2 AND + "actions" <@ ARRAY['publish', 'read-status']::TEXT[] AND + "actions" <> ARRAY['publish', 'publish']::TEXT[] AND + "actions" <> ARRAY['read-status', 'read-status']::TEXT[] + ) +); + +-- CreateTable +CREATE TABLE "workload_credentials" ( + "id" UUID NOT NULL, + "workload_publisher_id" UUID NOT NULL, + "token_digest" TEXT NOT NULL, + "expires_at" TIMESTAMPTZ(3), + "revoked_at" TIMESTAMPTZ(3), + "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "workload_credentials_pkey" PRIMARY KEY ("id") +); + +-- AlterTable +ALTER TABLE "prototype_versions" +ALTER COLUMN "created_by_user_id" DROP NOT NULL, +ADD COLUMN "workload_publisher_id" UUID, +ADD COLUMN "artifact_format" TEXT, +ADD COLUMN "artifact_version" TEXT, +ADD COLUMN "content_digest" TEXT, +ADD COLUMN "size_bytes" INTEGER, +ADD COLUMN "built_at" TIMESTAMPTZ(3), +ADD COLUMN "provenance_provider" TEXT, +ADD COLUMN "provenance_source_url" TEXT, +ADD COLUMN "provenance_revision" TEXT, +ADD COLUMN "provenance_ref" TEXT, +ADD COLUMN "provenance_pipeline" TEXT, +ADD COLUMN "provenance_run_id" TEXT, +ADD COLUMN "storage_key" TEXT, +ADD CONSTRAINT "prototype_versions_single_creator" CHECK ( + ("created_by_user_id" IS NOT NULL) <> ("workload_publisher_id" IS NOT NULL) +), +ADD CONSTRAINT "prototype_versions_published_artifact" CHECK ( + "workload_publisher_id" IS NULL OR ( + "artifact_format" IS NOT NULL AND + "artifact_version" IS NOT NULL AND + "content_digest" IS NOT NULL AND + "size_bytes" IS NOT NULL AND + "built_at" IS NOT NULL AND + "provenance_provider" IS NOT NULL AND + "storage_key" IS NOT NULL + ) +); + +-- CreateTable +CREATE TABLE "artifact_publications" ( + "id" UUID NOT NULL, + "prototype_id" UUID NOT NULL, + "workload_publisher_id" UUID NOT NULL, + "prototype_version_id" UUID NOT NULL, + "idempotency_key_digest" TEXT NOT NULL, + "request_digest" TEXT NOT NULL, + "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "artifact_publications_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "prototype_channels" ( + "id" UUID NOT NULL, + "prototype_id" UUID NOT NULL, + "prototype_version_id" UUID NOT NULL, + "name" TEXT NOT NULL, + "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "prototype_channels_pkey" PRIMARY KEY ("id") +); + +-- AlterTable +ALTER TABLE "security_audits" ADD COLUMN "workload_publisher_id" UUID; + +-- CreateIndex +CREATE INDEX "workload_publishers_prototype_id_idx" ON "workload_publishers"("prototype_id"); +CREATE UNIQUE INDEX "workload_credentials_token_digest_key" ON "workload_credentials"("token_digest"); +CREATE INDEX "workload_credentials_workload_publisher_id_idx" ON "workload_credentials"("workload_publisher_id"); +CREATE INDEX "prototype_versions_workload_publisher_id_idx" ON "prototype_versions"("workload_publisher_id"); +CREATE UNIQUE INDEX "prototype_versions_prototype_id_content_digest_key" ON "prototype_versions"("prototype_id", "content_digest"); +CREATE UNIQUE INDEX "artifact_publications_prototype_version_id_key" ON "artifact_publications"("prototype_version_id"); +CREATE UNIQUE INDEX "artifact_publications_workload_publisher_id_idempotency_key_digest_key" ON "artifact_publications"("workload_publisher_id", "idempotency_key_digest"); +CREATE INDEX "artifact_publications_prototype_id_idx" ON "artifact_publications"("prototype_id"); +CREATE UNIQUE INDEX "prototype_channels_prototype_id_name_key" ON "prototype_channels"("prototype_id", "name"); +CREATE INDEX "prototype_channels_prototype_version_id_idx" ON "prototype_channels"("prototype_version_id"); +CREATE INDEX "security_audits_workload_publisher_id_idx" ON "security_audits"("workload_publisher_id"); + +-- AddForeignKey +ALTER TABLE "workload_publishers" ADD CONSTRAINT "workload_publishers_prototype_id_fkey" FOREIGN KEY ("prototype_id") REFERENCES "prototypes"("id") ON DELETE RESTRICT ON UPDATE CASCADE; +ALTER TABLE "workload_credentials" ADD CONSTRAINT "workload_credentials_workload_publisher_id_fkey" FOREIGN KEY ("workload_publisher_id") REFERENCES "workload_publishers"("id") ON DELETE RESTRICT ON UPDATE CASCADE; +ALTER TABLE "prototype_versions" ADD CONSTRAINT "prototype_versions_workload_publisher_id_fkey" FOREIGN KEY ("workload_publisher_id") REFERENCES "workload_publishers"("id") ON DELETE RESTRICT ON UPDATE CASCADE; +ALTER TABLE "artifact_publications" ADD CONSTRAINT "artifact_publications_prototype_id_fkey" FOREIGN KEY ("prototype_id") REFERENCES "prototypes"("id") ON DELETE RESTRICT ON UPDATE CASCADE; +ALTER TABLE "artifact_publications" ADD CONSTRAINT "artifact_publications_workload_publisher_id_fkey" FOREIGN KEY ("workload_publisher_id") REFERENCES "workload_publishers"("id") ON DELETE RESTRICT ON UPDATE CASCADE; +ALTER TABLE "artifact_publications" ADD CONSTRAINT "artifact_publications_prototype_version_id_fkey" FOREIGN KEY ("prototype_version_id") REFERENCES "prototype_versions"("id") ON DELETE RESTRICT ON UPDATE CASCADE; +ALTER TABLE "prototype_channels" ADD CONSTRAINT "prototype_channels_prototype_id_fkey" FOREIGN KEY ("prototype_id") REFERENCES "prototypes"("id") ON DELETE RESTRICT ON UPDATE CASCADE; +ALTER TABLE "prototype_channels" ADD CONSTRAINT "prototype_channels_prototype_version_id_fkey" FOREIGN KEY ("prototype_version_id") REFERENCES "prototype_versions"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- Immutable prototype versions may be inserted and read, but never rewritten or deleted. +CREATE FUNCTION reject_prototype_version_mutation() RETURNS trigger AS $$ +BEGIN + RAISE EXCEPTION 'prototype versions are immutable'; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER prototype_versions_immutable +BEFORE UPDATE OR DELETE ON "prototype_versions" +FOR EACH ROW EXECUTE FUNCTION reject_prototype_version_mutation(); diff --git a/hub/schema/prototype-channel.prisma b/hub/schema/prototype-channel.prisma new file mode 100644 index 00000000..b650e614 --- /dev/null +++ b/hub/schema/prototype-channel.prisma @@ -0,0 +1,21 @@ +model PrototypeChannel { + // ID columns + id String @id @default(uuid()) @db.Uuid + prototypeId String @map("prototype_id") @db.Uuid + prototypeVersionId String @map("prototype_version_id") @db.Uuid + + // Relations + prototype Prototype @relation(fields: [prototypeId], references: [id], onDelete: Restrict) + prototypeVersion PrototypeVersion @relation(fields: [prototypeVersionId], references: [id], onDelete: Restrict) + + // Attribute columns + name String + + // Timestamp columns + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3) + + @@unique([prototypeId, name]) + @@index([prototypeVersionId]) + @@map("prototype_channels") +} diff --git a/hub/schema/prototype-version.prisma b/hub/schema/prototype-version.prisma index 8a60193d..4210e28c 100644 --- a/hub/schema/prototype-version.prisma +++ b/hub/schema/prototype-version.prisma @@ -1,22 +1,40 @@ model PrototypeVersion { // ID columns - id String @id @default(uuid()) @db.Uuid - prototypeId String @map("prototype_id") @db.Uuid - createdByUserId String @map("created_by_user_id") @db.Uuid + id String @id @default(uuid()) @db.Uuid + prototypeId String @map("prototype_id") @db.Uuid + createdByUserId String? @map("created_by_user_id") @db.Uuid + workloadPublisherId String? @map("workload_publisher_id") @db.Uuid // Relations - prototype Prototype @relation(fields: [prototypeId], references: [id], onDelete: Restrict) - createdBy User @relation("PrototypeVersionCreator", fields: [createdByUserId], references: [id], onDelete: Restrict) + prototype Prototype @relation(fields: [prototypeId], references: [id], onDelete: Restrict) + createdBy User? @relation("PrototypeVersionCreator", fields: [createdByUserId], references: [id], onDelete: Restrict) + workloadPublisher WorkloadPublisher? @relation(fields: [workloadPublisherId], references: [id], onDelete: Restrict) + publication ArtifactPublication? + channels PrototypeChannel[] // Attribute columns - sequence Int - label String? + sequence Int + label String? + artifactFormat String? @map("artifact_format") + artifactVersion String? @map("artifact_version") + contentDigest String? @map("content_digest") + sizeBytes Int? @map("size_bytes") + builtAt DateTime? @map("built_at") @db.Timestamptz(3) + provenanceProvider String? @map("provenance_provider") + provenanceSourceUrl String? @map("provenance_source_url") + provenanceRevision String? @map("provenance_revision") + provenanceRef String? @map("provenance_ref") + provenancePipeline String? @map("provenance_pipeline") + provenanceRunId String? @map("provenance_run_id") + storageKey String? @map("storage_key") // Timestamp columns createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3) @@unique([prototypeId, sequence]) + @@unique([prototypeId, contentDigest]) @@index([createdByUserId]) + @@index([workloadPublisherId]) @@map("prototype_versions") } diff --git a/hub/schema/prototype.prisma b/hub/schema/prototype.prisma index 5064e50a..30dbb25a 100644 --- a/hub/schema/prototype.prisma +++ b/hub/schema/prototype.prisma @@ -5,11 +5,14 @@ model Prototype { createdByUserId String @map("created_by_user_id") @db.Uuid // Relations - workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade) - createdBy User @relation("PrototypeCreator", fields: [createdByUserId], references: [id], onDelete: Restrict) - versions PrototypeVersion[] - grants PrototypeAccessGrant[] + workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade) + createdBy User @relation("PrototypeCreator", fields: [createdByUserId], references: [id], onDelete: Restrict) + versions PrototypeVersion[] + grants PrototypeAccessGrant[] sourceRepository GitHubRepositorySource? @relation(fields: [sourceRepositoryId], references: [id], onDelete: SetNull) + publishers WorkloadPublisher[] + publications ArtifactPublication[] + channels PrototypeChannel[] // Attribute columns name String diff --git a/hub/schema/security-audit.prisma b/hub/schema/security-audit.prisma index ed0b61c4..cbc1f36a 100644 --- a/hub/schema/security-audit.prisma +++ b/hub/schema/security-audit.prisma @@ -9,6 +9,7 @@ model SecurityAudit { workspaceId String? @map("workspace_id") @db.Uuid prototypeId String? @map("prototype_id") @db.Uuid prototypeVersionId String? @map("prototype_version_id") @db.Uuid + workloadPublisherId String? @map("workload_publisher_id") @db.Uuid targetUserId String? @map("target_user_id") @db.Uuid githubRepositorySourceId String? @map("github_repository_source_id") @db.Uuid @@ -32,6 +33,7 @@ model SecurityAudit { @@index([workspaceId]) @@index([prototypeId]) @@index([prototypeVersionId]) + @@index([workloadPublisherId]) @@index([targetUserId]) @@index([githubRepositorySourceId]) @@map("security_audits") diff --git a/hub/schema/workload-credential.prisma b/hub/schema/workload-credential.prisma new file mode 100644 index 00000000..c043762b --- /dev/null +++ b/hub/schema/workload-credential.prisma @@ -0,0 +1,20 @@ +model WorkloadCredential { + // ID columns + id String @id @default(uuid()) @db.Uuid + workloadPublisherId String @map("workload_publisher_id") @db.Uuid + + // Relations + workloadPublisher WorkloadPublisher @relation(fields: [workloadPublisherId], references: [id], onDelete: Restrict) + + // Attribute columns + tokenDigest String @unique @map("token_digest") + expiresAt DateTime? @map("expires_at") @db.Timestamptz(3) + revokedAt DateTime? @map("revoked_at") @db.Timestamptz(3) + + // Timestamp columns + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3) + + @@index([workloadPublisherId]) + @@map("workload_credentials") +} diff --git a/hub/schema/workload-publisher.prisma b/hub/schema/workload-publisher.prisma new file mode 100644 index 00000000..3ebebee7 --- /dev/null +++ b/hub/schema/workload-publisher.prisma @@ -0,0 +1,23 @@ +model WorkloadPublisher { + // ID columns + id String @id @default(uuid()) @db.Uuid + prototypeId String @map("prototype_id") @db.Uuid + + // Relations + prototype Prototype @relation(fields: [prototypeId], references: [id], onDelete: Restrict) + credentials WorkloadCredential[] + versions PrototypeVersion[] + publications ArtifactPublication[] + + // Attribute columns + name String + actions String[] + active Boolean @default(true) + + // Timestamp columns + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3) + + @@index([prototypeId]) + @@map("workload_publishers") +} diff --git a/hub/script/publish-artifact.mjs b/hub/script/publish-artifact.mjs new file mode 100644 index 00000000..05d66d35 --- /dev/null +++ b/hub/script/publish-artifact.mjs @@ -0,0 +1,70 @@ +#!/usr/bin/env node + +import { createHash, randomUUID } from "node:crypto" +import { readFile, stat } from "node:fs/promises" + +function usage() { + console.error( + "Usage: REPLICA_PUBLISH_TOKEN=... node script/publish-artifact.mjs HUB_URL PROTOTYPE_ID ARTIFACT_PATH [current|preview]", + ) +} + +const [, , hubUrl, prototypeId, artifactPath, promoteChannel] = process.argv +const token = process.env.REPLICA_PUBLISH_TOKEN +if ( + !hubUrl || + !prototypeId || + !artifactPath || + !token || + (promoteChannel && !["current", "preview"].includes(promoteChannel)) +) { + usage() + process.exitCode = 2 +} else { + const bytes = await readFile(artifactPath) + const details = await stat(artifactPath) + const digest = `sha256:${createHash("sha256").update(bytes).digest("hex")}` + const response = await fetch( + new URL(`/api/publishing/prototypes/${prototypeId}/versions`, hubUrl), + { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + "idempotency-key": process.env.REPLICA_IDEMPOTENCY_KEY ?? randomUUID(), + }, + body: JSON.stringify({ + manifest: { + schemaVersion: 1, + prototypeId, + artifact: { + format: process.env.REPLICA_ARTIFACT_FORMAT ?? "replica.prototype.bundle", + version: process.env.REPLICA_ARTIFACT_VERSION ?? "1", + digest, + sizeBytes: bytes.length, + builtAt: details.mtime.toISOString(), + }, + provenance: { + provider: process.env.REPLICA_PROVENANCE_PROVIDER ?? "local-cli", + ...(process.env.REPLICA_SOURCE_URL + ? { sourceUrl: process.env.REPLICA_SOURCE_URL } + : {}), + ...(process.env.REPLICA_REVISION ? { revision: process.env.REPLICA_REVISION } : {}), + ...(process.env.REPLICA_REF ? { ref: process.env.REPLICA_REF } : {}), + ...(process.env.REPLICA_PIPELINE ? { pipeline: process.env.REPLICA_PIPELINE } : {}), + ...(process.env.REPLICA_RUN_ID ? { runId: process.env.REPLICA_RUN_ID } : {}), + }, + }, + artifactBase64: bytes.toString("base64"), + ...(promoteChannel ? { promoteChannel } : {}), + }), + }, + ) + const result = await response.text() + if (!response.ok) { + console.error(`Publish failed (${response.status}): ${result}`) + process.exitCode = 1 + } else { + process.stdout.write(`${result}\n`) + } +} diff --git a/hub/server/create-app/index.ts b/hub/server/create-app/index.ts index 22872cd7..947ade16 100644 --- a/hub/server/create-app/index.ts +++ b/hub/server/create-app/index.ts @@ -7,6 +7,7 @@ import { createRequestLogger } from "../integrations/pino/create-request-logger/ import { omitRequestLogPayload } from "../integrations/pino/omit-request-log-payload/index.ts" import { routes } from "../routes/index.ts" import type { AuthenticationRoutes } from "../routes/types.ts" +import { maximumPublishingJsonBytes } from "../domains/artifact-publishing/create-service/constants.ts" /** Give every endpoint the same request logging and safe error responses. */ export function createApp({ @@ -25,10 +26,12 @@ export function createApp({ "/api/github/webhooks", "/api/workspaces", "/api/prototypes", + "/api/publishing", ], omitRequestLogPayload, ) app.use("/api/github/webhooks", express.raw({ type: "application/json", limit: "1mb" })) + app.use("/api/publishing", express.json({ limit: maximumPublishingJsonBytes })) app.use(express.json()) routes(app, authentication) diff --git a/hub/server/domains/artifact-publishing/create-service/constants.ts b/hub/server/domains/artifact-publishing/create-service/constants.ts new file mode 100644 index 00000000..ec9f2a16 --- /dev/null +++ b/hub/server/domains/artifact-publishing/create-service/constants.ts @@ -0,0 +1,2 @@ +export const maximumArtifactBytes = 8 * 1024 * 1024 +export const maximumPublishingJsonBytes = 12 * 1024 * 1024 diff --git a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts new file mode 100644 index 00000000..0c6103b9 --- /dev/null +++ b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts @@ -0,0 +1,595 @@ +import { createHash, randomUUID } from "node:crypto" + +import { describe, expect, it, onTestFinished, vi } from "vitest" + +import { createArtifactPublishingService } from "#server/domains/artifact-publishing/create-service" +import type { ArtifactManifest } from "#server/domains/artifact-publishing/create-service/types" +import { createIdentityStore } from "#server/domains/identity/create-store" +import { createPrototypeAuthorizationService } from "#server/domains/prototype-authorization/create-service" +import type { ArtifactStorage } from "#server/integrations/artifact-storage/types" +import { digestSecret } from "#server/utils/digest-secret" + +import type { PrismaClient } from "../../../generated/prisma/client.ts" + +function artifact(content: string) { + const bytes = Buffer.from(content) + return { + bytes, + base64: bytes.toString("base64"), + digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, + } +} + +function manifest( + prototypeId: string, + publishedArtifact: ReturnType, + overrides: Partial = {}, +): ArtifactManifest { + return { + schemaVersion: 1, + prototypeId, + artifact: { + format: "replica-build", + version: "1", + digest: publishedArtifact.digest, + sizeBytes: publishedArtifact.bytes.length, + builtAt: "2026-10-27T20:00:00.000Z", + }, + provenance: { + provider: "build-system", + sourceUrl: "https://ci.example/runs/123", + revision: "abc123", + ref: "refs/heads/main", + pipeline: "publish", + runId: "123", + }, + ...overrides, + } +} + +async function database() { + vi.stubGlobal("prisma", undefined) + const { prisma } = await import("../../../integrations/prisma") + onTestFinished(() => prisma.$disconnect()) + + const stored = new Map() + const storage: ArtifactStorage = { + put(digest, bytes) { + const key = digest.replace(":", "/") + const created = !stored.has(key) + if (created) stored.set(key, bytes.slice()) + return Promise.resolve({ key, created }) + }, + read(key) { + const bytes = stored.get(key) + return bytes + ? Promise.resolve(bytes.slice()) + : Promise.reject(new Error("Artifact not found")) + }, + remove(key) { + stored.delete(key) + return Promise.resolve() + }, + } + + return { + prisma, + storage, + service: createArtifactPublishingService({ database: prisma, storage }), + } +} + +async function createPrototype(prisma: PrismaClient) { + const user = await prisma.user.create({ data: { displayName: "Publisher owner" } }) + const workspace = await prisma.workspace.create({ + data: { + name: "Workspace", + createdByUserId: user.id, + }, + }) + const prototype = await prisma.prototype.create({ + data: { + name: "Prototype", + workspaceId: workspace.id, + createdByUserId: user.id, + }, + }) + return { prototype, user, workspace } +} + +async function createAdministratorSession(prisma: PrismaClient, token: string) { + const store = createIdentityStore(prisma) + const authorizedAt = new Date() + const actor = await store.lock(async (records) => { + const user = await records.createUser({ + displayName: "Prototype administrator", + avatarUrl: null, + }) + const identity = await records.saveExternalIdentity({ + userId: user.id, + provider: "github", + issuer: "https://github.com", + subject: "publisher-administrator", + username: "publisher-administrator", + displayName: user.displayName, + avatarUrl: null, + }) + const provider = await records.createAuthenticationProviderRevision({ + key: "github", + type: "github", + enabled: true, + revision: 1, + issuer: "https://github.com", + configuration: { clientId: "github-client" }, + }) + await records.saveMembership({ + userId: user.id, + status: "active", + role: "member", + source: "manual", + }) + await records.createSession({ + userId: user.id, + externalIdentityId: identity.id, + authenticationProviderRevisionId: provider.id, + tokenDigest: digestSecret(token), + signedInAt: authorizedAt, + freshAuthorizedAt: authorizedAt, + }) + return user + }) + return { actor, store } +} + +async function createCredential( + prisma: PrismaClient, + prototypeId: string, + actions: ("publish" | "read-status")[] = ["publish", "read-status"], +) { + const credentialId = randomUUID() + const token = `rpl_pub_${credentialId}.${"t".repeat(43)}` + const publisher = await prisma.workloadPublisher.create({ + data: { + prototypeId, + name: "CI publisher", + actions, + credentials: { + create: { + id: credentialId, + tokenDigest: digestSecret(token), + }, + }, + }, + }) + return { + authorization: `Bearer ${token}`, + credentialId, + publisher, + token, + } +} + +function publicationInput( + prototypeId: string, + publishedArtifact: ReturnType, + overrides: Record = {}, +) { + return { + manifest: manifest(prototypeId, publishedArtifact), + artifactBase64: publishedArtifact.base64, + ...overrides, + } +} + +function publishedVersion(result: { version: unknown }) { + if ( + result.version === null || + typeof result.version !== "object" || + !("id" in result.version) || + typeof result.version.id !== "string" + ) { + throw new Error("Expected a published version") + } + return { id: result.version.id } +} + +describe("createArtifactPublishingService() database behavior", () => { + it.each([ + { + name: "unknown manifest field", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { ...valid.manifest, unrecognized: true }, + }), + }, + { + name: "unsupported schema version", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { ...valid.manifest, schemaVersion: 2 }, + }), + }, + { + name: "non-HTTP provenance URL", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { + ...valid.manifest, + provenance: { ...valid.manifest.provenance, sourceUrl: "file:///private/build" }, + }, + }), + }, + { + name: "different prototype identifier", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { ...valid.manifest, prototypeId: randomUUID() }, + }), + }, + ])("rejects a manifest with $name before creating a version", async ({ change }) => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const publishedArtifact = artifact("manifest-validation") + + await expect( + context.service.publish( + credential.authorization, + prototype.id, + "manifest-case", + change(publicationInput(prototype.id, publishedArtifact)), + ), + ).rejects.toMatchObject({ + status: 422, + code: "invalid_artifact_manifest", + }) + expect(await context.prisma.prototypeVersion.count()).toBe(0) + }) + + it("rejects an artifact whose decoded size differs from the manifest", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const publishedArtifact = artifact("size-mismatch") + const input = publicationInput(prototype.id, publishedArtifact) + input.manifest.artifact.sizeBytes += 1 + + await expect( + context.service.publish(credential.authorization, prototype.id, "size-case", input), + ).rejects.toMatchObject({ + status: 422, + code: "artifact_size_mismatch", + }) + expect(await context.prisma.prototypeVersion.count()).toBe(0) + }) + + it("rejects an artifact whose content digest differs from the manifest", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const publishedArtifact = artifact("digest-mismatch") + const input = publicationInput(prototype.id, publishedArtifact) + input.manifest.artifact.digest = artifact("different-content").digest + + await expect( + context.service.publish(credential.authorization, prototype.id, "digest-case", input), + ).rejects.toMatchObject({ + status: 422, + code: "artifact_digest_mismatch", + }) + expect(await context.prisma.prototypeVersion.count()).toBe(0) + }) + + it("stores only a digest of a prototype-scoped workload token", async () => { + const context = await database() + const sessionToken = "publisher-administrator-session" + const { actor, store } = await createAdministratorSession(context.prisma, sessionToken) + const workspace = await context.prisma.workspace.create({ + data: { + name: "Credential workspace", + createdByUserId: actor.id, + grants: { + create: { + userId: actor.id, + effect: "allow", + role: "administrator", + }, + }, + }, + }) + const prototype = await context.prisma.prototype.create({ + data: { + name: "Credential prototype", + workspaceId: workspace.id, + createdByUserId: actor.id, + }, + }) + const authorization = createPrototypeAuthorizationService({ + database: context.prisma, + store, + }) + const provisioned = await authorization.createPublisherCredential(sessionToken, prototype.id, { + name: "CI publisher", + actions: ["publish", "read-status"], + }) + if ( + provisioned === null || + typeof provisioned !== "object" || + !("credential" in provisioned) || + provisioned.credential === null || + typeof provisioned.credential !== "object" || + !("id" in provisioned.credential) || + typeof provisioned.credential.id !== "string" || + !("token" in provisioned.credential) || + typeof provisioned.credential.token !== "string" + ) { + throw new Error("Expected a provisioned workload credential") + } + + const saved = await context.prisma.workloadCredential.findUniqueOrThrow({ + where: { id: provisioned.credential.id }, + }) + + expect(saved.tokenDigest).toBe(digestSecret(provisioned.credential.token)) + expect(saved.tokenDigest).not.toContain(provisioned.credential.token) + expect(JSON.stringify(saved)).not.toContain(provisioned.credential.token) + }) + + it("enforces publish and read-status action scopes independently", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const publishOnly = await createCredential(context.prisma, prototype.id, ["publish"]) + const statusOnly = await createCredential(context.prisma, prototype.id, ["read-status"]) + const publishedArtifact = artifact("action-scope") + + await context.service.publish( + publishOnly.authorization, + prototype.id, + "action-scope", + publicationInput(prototype.id, publishedArtifact), + ) + + await expect( + context.service.getStatus(publishOnly.authorization, prototype.id, "action-scope"), + ).rejects.toMatchObject({ status: 403, code: "publisher_access_denied" }) + await expect( + context.service.publish( + statusOnly.authorization, + prototype.id, + "status-cannot-publish", + publicationInput(prototype.id, artifact("status-only")), + ), + ).rejects.toMatchObject({ status: 403, code: "publisher_access_denied" }) + }) + + it("rejects a credential issued for a different prototype", async () => { + const context = await database() + const first = await createPrototype(context.prisma) + const second = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, first.prototype.id) + + await expect( + context.service.publish( + credential.authorization, + second.prototype.id, + "cross-prototype", + publicationInput(second.prototype.id, artifact("isolated")), + ), + ).rejects.toMatchObject({ status: 403, code: "publisher_access_denied" }) + expect( + await context.prisma.prototypeVersion.count({ + where: { prototypeId: second.prototype.id }, + }), + ).toBe(0) + }) + + it("returns the same immutable version for an idempotent retry", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const input = publicationInput(prototype.id, artifact("idempotent")) + + const first = await context.service.publish( + credential.authorization, + prototype.id, + "same-request", + input, + ) + const replay = await context.service.publish( + credential.authorization, + prototype.id, + "same-request", + input, + ) + + expect(first).toMatchObject({ replayed: false }) + expect(replay).toMatchObject({ replayed: true, version: publishedVersion(first) }) + expect( + await context.prisma.prototypeVersion.count({ where: { prototypeId: prototype.id } }), + ).toBe(1) + expect( + await context.prisma.artifactPublication.count({ + where: { workloadPublisherId: credential.publisher.id }, + }), + ).toBe(1) + }) + + it("rejects reuse of an idempotency key for a different publication", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const publishedArtifact = artifact("idempotency-conflict") + const first = publicationInput(prototype.id, publishedArtifact) + const conflicting = publicationInput(prototype.id, publishedArtifact) + conflicting.manifest.provenance.runId = "different-run" + + await context.service.publish(credential.authorization, prototype.id, "reused-key", first) + await expect( + context.service.publish(credential.authorization, prototype.id, "reused-key", conflicting), + ).rejects.toMatchObject({ status: 409, code: "idempotency_conflict" }) + expect( + await context.prisma.prototypeVersion.count({ where: { prototypeId: prototype.id } }), + ).toBe(1) + }) + + it("serializes concurrent publications into ordered immutable versions", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + + const results = await Promise.all([ + context.service.publish( + credential.authorization, + prototype.id, + "concurrent-one", + publicationInput(prototype.id, artifact("concurrent-one")), + ), + context.service.publish( + credential.authorization, + prototype.id, + "concurrent-two", + publicationInput(prototype.id, artifact("concurrent-two")), + ), + ]) + const versions = await context.prisma.prototypeVersion.findMany({ + where: { prototypeId: prototype.id }, + orderBy: { sequence: "asc" }, + }) + + expect(results.map(({ replayed }) => replayed)).toEqual([false, false]) + expect(versions.map(({ sequence }) => sequence)).toEqual([1, 2]) + await context.prisma.$executeRawUnsafe(` + CREATE FUNCTION reject_test_prototype_version_mutation() RETURNS trigger AS $$ + BEGIN + RAISE EXCEPTION 'prototype versions are immutable'; + END; + $$ LANGUAGE plpgsql; + `) + await context.prisma.$executeRawUnsafe(` + CREATE TRIGGER test_prototype_versions_immutable + BEFORE UPDATE OR DELETE ON prototype_versions + FOR EACH ROW EXECUTE FUNCTION reject_test_prototype_version_mutation(); + `) + try { + await expect( + context.prisma.prototypeVersion.update({ + where: { id: versions[0]!.id }, + data: { label: "rewritten" }, + }), + ).rejects.toThrow(/prototype versions are immutable/i) + await expect( + context.prisma.prototypeVersion.delete({ where: { id: versions[1]!.id } }), + ).rejects.toThrow(/prototype versions are immutable/i) + } finally { + await context.prisma.$executeRawUnsafe( + "DROP TRIGGER test_prototype_versions_immutable ON prototype_versions", + ) + await context.prisma.$executeRawUnsafe( + "DROP FUNCTION reject_test_prototype_version_mutation()", + ) + } + }) + + it("promotes and rolls back a channel by moving one atomic pointer", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const first = await context.service.publish( + credential.authorization, + prototype.id, + "channel-one", + publicationInput(prototype.id, artifact("channel-one")), + ) + const second = await context.service.publish( + credential.authorization, + prototype.id, + "channel-two", + publicationInput(prototype.id, artifact("channel-two")), + ) + const firstVersion = publishedVersion(first) + const secondVersion = publishedVersion(second) + + await context.service.promote(credential.authorization, prototype.id, "current", { + versionId: firstVersion.id, + }) + await context.service.promote(credential.authorization, prototype.id, "current", { + versionId: secondVersion.id, + }) + await context.service.promote(credential.authorization, prototype.id, "preview", { + versionId: secondVersion.id, + }) + await context.service.promote(credential.authorization, prototype.id, "current", { + versionId: firstVersion.id, + }) + + expect( + await context.prisma.prototypeChannel.findMany({ + where: { prototypeId: prototype.id }, + orderBy: { name: "asc" }, + select: { name: true, prototypeVersionId: true }, + }), + ).toEqual([ + { name: "current", prototypeVersionId: firstVersion.id }, + { name: "preview", prototypeVersionId: secondVersion.id }, + ]) + expect( + await context.prisma.prototypeVersion.findMany({ + where: { prototypeId: prototype.id }, + orderBy: { sequence: "asc" }, + select: { id: true, sequence: true }, + }), + ).toEqual([ + { id: firstVersion.id, sequence: 1 }, + { id: secondVersion.id, sequence: 2 }, + ]) + }) + + it("rolls back a published version when its audit cannot be saved", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + + await context.prisma.$executeRawUnsafe(` + CREATE FUNCTION reject_version_publish_audit() RETURNS trigger AS $$ + BEGIN + IF NEW.event = 'prototype.version.publish' THEN + RAISE EXCEPTION 'audit unavailable'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + `) + await context.prisma.$executeRawUnsafe(` + CREATE TRIGGER reject_version_publish_audit + BEFORE INSERT ON security_audits + FOR EACH ROW EXECUTE FUNCTION reject_version_publish_audit(); + `) + try { + await expect( + context.service.publish( + credential.authorization, + prototype.id, + "audit-failure", + publicationInput(prototype.id, artifact("audit-failure")), + ), + ).rejects.toThrow("audit unavailable") + } finally { + await context.prisma.$executeRawUnsafe( + "DROP TRIGGER reject_version_publish_audit ON security_audits", + ) + await context.prisma.$executeRawUnsafe("DROP FUNCTION reject_version_publish_audit()") + } + + expect( + await context.prisma.prototypeVersion.count({ where: { prototypeId: prototype.id } }), + ).toBe(0) + expect( + await context.prisma.artifactPublication.count({ where: { prototypeId: prototype.id } }), + ).toBe(0) + expect( + await context.prisma.securityAudit.count({ + where: { prototypeId: prototype.id, event: "prototype.version.publish" }, + }), + ).toBe(0) + }) +}) diff --git a/hub/server/domains/artifact-publishing/create-service/index.ts b/hub/server/domains/artifact-publishing/create-service/index.ts new file mode 100644 index 00000000..e6316bb3 --- /dev/null +++ b/hub/server/domains/artifact-publishing/create-service/index.ts @@ -0,0 +1,375 @@ +import { createHash } from "node:crypto" + +import { z } from "zod" + +import type { ArtifactStorage } from "#server/integrations/artifact-storage/types" +import { saveSecurityAudit } from "#server/repositories/security-audits/save-audit" +import { digestSecret } from "#server/utils/digest-secret" +import { secretsMatch } from "#server/utils/secrets-match" +import { AuthenticationError } from "#server/utils/authentication-error" + +import { maximumArtifactBytes } from "./constants.ts" +import type { ArtifactPublishingService, PrototypeChannelName, PublisherAction } from "./types.ts" + +import type { Prisma, PrismaClient } from "../../../generated/prisma/client.ts" + +const boundedIdentifier = z.string().trim().min(1).max(255) +const manifestSchema = z + .object({ + schemaVersion: z.literal(1), + prototypeId: z.uuid(), + artifact: z.object({ + format: z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/), + version: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,31}$/), + digest: z.string().regex(/^sha256:[a-f0-9]{64}$/), + sizeBytes: z.number().int().min(0).max(maximumArtifactBytes), + builtAt: z.iso.datetime({ offset: true }), + }), + provenance: z.object({ + provider: z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/), + sourceUrl: z + .url() + .max(2_048) + .refine((value) => ["http:", "https:"].includes(new URL(value).protocol)) + .optional(), + revision: boundedIdentifier.optional(), + ref: boundedIdentifier.optional(), + pipeline: boundedIdentifier.optional(), + runId: boundedIdentifier.optional(), + }), + }) + .strict() +const publishSchema = z + .object({ + manifest: manifestSchema, + artifactBase64: z.string().max(Math.ceil((maximumArtifactBytes * 4) / 3) + 4), + promoteChannel: z.enum(["current", "preview"]).optional(), + }) + .strict() +const promotionSchema = z.object({ versionId: z.uuid() }).strict() +const channelSchema = z.enum(["current", "preview"]) +const prototypeIdSchema = z.uuid() +const idempotencyKeySchema = z + .string() + .min(1) + .max(200) + .regex(/^[\x21-\x7e]+$/) +const authorizationSchema = /^Bearer (rpl_pub_([0-9a-f-]{36})\.([A-Za-z0-9_-]{43}))$/ + +function error(status: number, code: string, message: string): never { + throw new AuthenticationError(status, code, message) +} + +function parsePrototypeId(prototypeId: string): void { + if (!prototypeIdSchema.safeParse(prototypeId).success) { + error(404, "prototype_not_found", "The prototype was not found.") + } +} + +function parseIdempotencyKey(value: string): string { + const parsed = idempotencyKeySchema.safeParse(value) + if (!parsed.success) { + error(422, "invalid_idempotency_key", "The idempotency key is invalid.") + } + return parsed.data +} + +function parseArtifactBase64(value: string): Uint8Array { + if ( + value.length % 4 !== 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value) + ) { + error(422, "invalid_artifact", "The artifact encoding is invalid.") + } + const bytes = Buffer.from(value, "base64") + if (bytes.length > maximumArtifactBytes || bytes.toString("base64") !== value) { + error(413, "artifact_too_large", "The artifact exceeds the maximum size.") + } + return bytes +} + +function sha256(bytes: Uint8Array): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}` +} + +async function lockPrototype( + database: Prisma.TransactionClient, + prototypeId: string, +): Promise { + await database.$queryRaw`SELECT "id" FROM "prototypes" WHERE "id" = ${prototypeId}::uuid FOR UPDATE` +} + +function versionResponse(version: { + id: string + prototypeId: string + sequence: number + label: string | null + artifactFormat: string | null + artifactVersion: string | null + contentDigest: string | null + sizeBytes: number | null + builtAt: Date | null + provenanceProvider: string | null + provenanceSourceUrl: string | null + provenanceRevision: string | null + provenanceRef: string | null + provenancePipeline: string | null + provenanceRunId: string | null + createdAt: Date +}) { + return { + id: version.id, + prototypeId: version.prototypeId, + sequence: version.sequence, + label: version.label, + artifact: { + format: version.artifactFormat, + version: version.artifactVersion, + digest: version.contentDigest, + sizeBytes: version.sizeBytes, + builtAt: version.builtAt, + }, + provenance: { + provider: version.provenanceProvider, + sourceUrl: version.provenanceSourceUrl, + revision: version.provenanceRevision, + ref: version.provenanceRef, + pipeline: version.provenancePipeline, + runId: version.provenanceRunId, + }, + createdAt: version.createdAt, + } +} + +export function createArtifactPublishingService({ + database, + storage, +}: { + database: PrismaClient + storage: ArtifactStorage +}): ArtifactPublishingService { + async function authenticate(authorization: string, prototypeId: string, action: PublisherAction) { + parsePrototypeId(prototypeId) + const match = authorizationSchema.exec(authorization) + if (!match?.[1] || !match[2]) { + error(401, "publisher_authentication_required", "Publisher authentication failed.") + } + const [, token, credentialId] = match + const credential = await database.workloadCredential.findUnique({ + where: { id: credentialId }, + include: { workloadPublisher: true }, + }) + if ( + !credential || + credential.revokedAt || + (credential.expiresAt && credential.expiresAt <= new Date()) || + !credential.workloadPublisher.active || + credential.workloadPublisher.prototypeId !== prototypeId || + !credential.workloadPublisher.actions.includes(action) || + !secretsMatch(credential.tokenDigest, digestSecret(token)) + ) { + error(403, "publisher_access_denied", "Publisher access is denied.") + } + return credential.workloadPublisher + } + + return { + async publish(authorization, prototypeId, idempotencyKey, input) { + const publisher = await authenticate(authorization, prototypeId, "publish") + const parsed = publishSchema.safeParse(input) + if (!parsed.success || parsed.data.manifest.prototypeId !== prototypeId) { + error(422, "invalid_artifact_manifest", "The artifact manifest is invalid.") + } + const keyDigest = digestSecret(parseIdempotencyKey(idempotencyKey)) + const bytes = parseArtifactBase64(parsed.data.artifactBase64) + if (bytes.length !== parsed.data.manifest.artifact.sizeBytes) { + error(422, "artifact_size_mismatch", "The artifact size does not match the manifest.") + } + if (sha256(bytes) !== parsed.data.manifest.artifact.digest) { + error(422, "artifact_digest_mismatch", "The artifact digest does not match the manifest.") + } + const requestDigest = digestSecret( + JSON.stringify({ + manifest: parsed.data.manifest, + promoteChannel: parsed.data.promoteChannel ?? null, + }), + ) + const stored = await storage.put(parsed.data.manifest.artifact.digest, bytes) + + return database.$transaction(async (transaction) => { + await lockPrototype(transaction, prototypeId) + const prototype = await transaction.prototype.findUnique({ where: { id: prototypeId } }) + if (!prototype) error(404, "prototype_not_found", "The prototype was not found.") + if (prototype.archivedAt) { + error(409, "prototype_archived", "Archived prototypes cannot receive versions.") + } + const existing = await transaction.artifactPublication.findUnique({ + where: { + workloadPublisherId_idempotencyKeyDigest: { + workloadPublisherId: publisher.id, + idempotencyKeyDigest: keyDigest, + }, + }, + include: { prototypeVersion: true }, + }) + if (existing) { + if (existing.requestDigest !== requestDigest) { + error( + 409, + "idempotency_conflict", + "The idempotency key was already used for a different publication.", + ) + } + return { + version: versionResponse(existing.prototypeVersion), + replayed: true, + promotedChannel: parsed.data.promoteChannel ?? null, + } + } + const duplicate = await transaction.prototypeVersion.findFirst({ + where: { + prototypeId, + contentDigest: parsed.data.manifest.artifact.digest, + }, + }) + if (duplicate) { + error(409, "artifact_already_published", "This artifact was already published.") + } + const latest = await transaction.prototypeVersion.findFirst({ + where: { prototypeId }, + orderBy: { sequence: "desc" }, + }) + const manifest = parsed.data.manifest + const version = await transaction.prototypeVersion.create({ + data: { + prototypeId, + workloadPublisherId: publisher.id, + sequence: (latest?.sequence ?? 0) + 1, + artifactFormat: manifest.artifact.format, + artifactVersion: manifest.artifact.version, + contentDigest: manifest.artifact.digest, + sizeBytes: manifest.artifact.sizeBytes, + builtAt: new Date(manifest.artifact.builtAt), + provenanceProvider: manifest.provenance.provider, + provenanceSourceUrl: manifest.provenance.sourceUrl ?? null, + provenanceRevision: manifest.provenance.revision ?? null, + provenanceRef: manifest.provenance.ref ?? null, + provenancePipeline: manifest.provenance.pipeline ?? null, + provenanceRunId: manifest.provenance.runId ?? null, + storageKey: stored.key, + }, + }) + await transaction.artifactPublication.create({ + data: { + prototypeId, + workloadPublisherId: publisher.id, + prototypeVersionId: version.id, + idempotencyKeyDigest: keyDigest, + requestDigest, + }, + }) + if (parsed.data.promoteChannel) { + await transaction.prototypeChannel.upsert({ + where: { + prototypeId_name: { prototypeId, name: parsed.data.promoteChannel }, + }, + create: { + prototypeId, + prototypeVersionId: version.id, + name: parsed.data.promoteChannel, + }, + update: { prototypeVersionId: version.id }, + }) + } + await saveSecurityAudit(transaction, { + event: "prototype.version.publish", + actorUserId: null, + sessionId: null, + authenticationProviderRevisionId: null, + workspaceId: prototype.workspaceId, + prototypeId, + prototypeVersionId: version.id, + workloadPublisherId: publisher.id, + context: { + source: "workload_publisher", + ...(parsed.data.promoteChannel + ? { reason: `channel_${parsed.data.promoteChannel}` } + : {}), + }, + }) + return { + version: versionResponse(version), + replayed: false, + promotedChannel: parsed.data.promoteChannel ?? null, + } + }) + }, + + async getStatus(authorization, prototypeId, idempotencyKey) { + const publisher = await authenticate(authorization, prototypeId, "read-status") + const publication = await database.artifactPublication.findUnique({ + where: { + workloadPublisherId_idempotencyKeyDigest: { + workloadPublisherId: publisher.id, + idempotencyKeyDigest: digestSecret(parseIdempotencyKey(idempotencyKey)), + }, + }, + include: { prototypeVersion: true }, + }) + if (!publication || publication.prototypeId !== prototypeId) { + error(404, "publication_not_found", "The publication was not found.") + } + return { + publicationId: publication.id, + version: versionResponse(publication.prototypeVersion), + } + }, + + async promote(authorization, prototypeId, channel, input) { + const publisher = await authenticate(authorization, prototypeId, "publish") + const parsedChannel = channelSchema.safeParse(channel) + const parsed = promotionSchema.safeParse(input) + if (!parsedChannel.success || !parsed.success) { + error(422, "invalid_promotion", "The promotion is invalid.") + } + return database.$transaction(async (transaction) => { + await lockPrototype(transaction, prototypeId) + const prototype = await transaction.prototype.findUnique({ where: { id: prototypeId } }) + if (!prototype) error(404, "prototype_not_found", "The prototype was not found.") + const version = await transaction.prototypeVersion.findFirst({ + where: { id: parsed.data.versionId, prototypeId }, + }) + if (!version) error(404, "version_not_found", "The version was not found.") + const pointer = await transaction.prototypeChannel.upsert({ + where: { + prototypeId_name: { + prototypeId, + name: parsedChannel.data satisfies PrototypeChannelName, + }, + }, + create: { + prototypeId, + prototypeVersionId: version.id, + name: parsedChannel.data, + }, + update: { prototypeVersionId: version.id }, + }) + await saveSecurityAudit(transaction, { + event: "prototype.channel.promote", + actorUserId: null, + sessionId: null, + authenticationProviderRevisionId: null, + workspaceId: prototype.workspaceId, + prototypeId, + prototypeVersionId: version.id, + workloadPublisherId: publisher.id, + context: { + source: "workload_publisher", + reason: `channel_${parsedChannel.data}`, + }, + }) + return pointer + }) + }, + } +} diff --git a/hub/server/domains/artifact-publishing/create-service/types.ts b/hub/server/domains/artifact-publishing/create-service/types.ts new file mode 100644 index 00000000..54a12a56 --- /dev/null +++ b/hub/server/domains/artifact-publishing/create-service/types.ts @@ -0,0 +1,38 @@ +export type PublisherAction = "publish" | "read-status" +export type PrototypeChannelName = "current" | "preview" + +export interface ArtifactManifest { + schemaVersion: 1 + prototypeId: string + artifact: { + format: string + version: string + digest: string + sizeBytes: number + builtAt: string + } + provenance: { + provider: string + sourceUrl?: string + revision?: string + ref?: string + pipeline?: string + runId?: string + } +} + +export interface ArtifactPublishingService { + publish( + authorization: string, + prototypeId: string, + idempotencyKey: string, + input: unknown, + ): Promise<{ version: unknown; replayed: boolean; promotedChannel: string | null }> + getStatus(authorization: string, prototypeId: string, idempotencyKey: string): Promise + promote( + authorization: string, + prototypeId: string, + channel: string, + input: unknown, + ): Promise +} diff --git a/hub/server/domains/authentication/create-runtime/index.ts b/hub/server/domains/authentication/create-runtime/index.ts index f6c062d2..293a03f7 100644 --- a/hub/server/domains/authentication/create-runtime/index.ts +++ b/hub/server/domains/authentication/create-runtime/index.ts @@ -1,4 +1,7 @@ +import path from "node:path" + import { createAdmissionPolicy } from "#server/domains/authentication/create-admission-policy" +import { createArtifactPublishingService } from "#server/domains/artifact-publishing/create-service" import { createAdministrationService } from "#server/domains/administration/create-service" import { createAuthenticationService } from "#server/domains/authentication/create-service" import type { AuthenticationSettings } from "#server/domains/authentication/create-service/types" @@ -7,6 +10,7 @@ import { createIdentityStore } from "#server/domains/identity/create-store" import { createPrototypeAuthorizationService } from "#server/domains/prototype-authorization/create-service" import { createGitHubAuthorizationService } from "#server/domains/github-authorization/create-service" import { createGitHubUserAuthClient } from "#server/integrations/github/create-user-auth-client" +import { createLocalArtifactStorage } from "#server/integrations/artifact-storage/create-local-storage" import type { GitHubUserAuthConfiguration } from "#server/integrations/github/create-user-auth-client/types" import { readTrustedProxyHops } from "#server/integrations/express/read-trusted-proxy-hops" import { prisma } from "#server/integrations/prisma" @@ -89,8 +93,15 @@ export async function createAuthenticationRuntime(environment: NodeJS.ProcessEnv admissionPolicy: admissionPolicy.name, trustedProxyHops, } + const artifactStoragePath = path.resolve( + environment.HUB_ARTIFACT_STORAGE_PATH ?? path.join(process.cwd(), ".data", "artifacts"), + ) return { administration: createAdministrationService({ store }), + artifactPublishing: createArtifactPublishingService({ + database: prisma, + storage: createLocalArtifactStorage(artifactStoragePath), + }), githubAuthorization, prototypeAuthorization: createPrototypeAuthorizationService({ baseline: { diff --git a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts index fdf09ebe..14fac68b 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts @@ -1369,59 +1369,6 @@ describe("createPrototypeAuthorizationService() database behavior", () => { ).toEqual([{ event: "prototype.archive" }]) }) - it("does not commit a version queued behind archival", async () => { - const { prisma, service, store } = await database() - const actor = await createActor(store, { - subject: "archive-version-race", - token: "archive-version-race-token", - }) - const workspace = await prisma.workspace.create({ - data: { - name: "Workspace", - createdByUserId: actor.user.id, - grants: { - create: { - userId: actor.user.id, - effect: "allow", - role: "editor", - }, - }, - }, - }) - const prototype = await prisma.prototype.create({ - data: { - name: "Prototype", - workspaceId: workspace.id, - createdByUserId: actor.user.id, - }, - }) - const lock = await holdPrototypeLock(prisma, prototype.id) - - const archive = service.archivePrototype("archive-version-race-token", prototype.id) - await waitForBlockedRowLock(prisma, "prototypes") - const version = service.createVersion("archive-version-race-token", prototype.id, { - label: "Must not commit", - }) - lock.release() - await lock.holder - - const archived = (await archive) as { id: string; archivedAt: Date | null } - expect(archived.id).toBe(prototype.id) - expect(archived.archivedAt).toBeInstanceOf(Date) - await expect(version).rejects.toMatchObject({ - status: 409, - code: "prototype_archived", - }) - expect(await prisma.prototypeVersion.count({ where: { prototypeId: prototype.id } })).toBe(0) - expect( - await prisma.securityAudit.findMany({ - where: { prototypeId: prototype.id }, - orderBy: { createdAt: "asc" }, - select: { event: true }, - }), - ).toEqual([{ event: "prototype.archive" }]) - }) - it("rolls back an access change when its audit cannot be saved", async () => { const { prisma, service, store } = await database() await createActor(store, { @@ -2423,7 +2370,6 @@ describe("createPrototypeAuthorizationService() database behavior", () => { expect(second.nextCursor).toBeNull() expect(new Set([...first.items, ...second.items].map(({ id }) => id)).size).toBe(4) }) - it("rejects a prototype cursor from another workspace", async () => { const { prisma, service, store } = await database() const actor = await createActor(store, { @@ -2484,7 +2430,6 @@ describe("createPrototypeAuthorizationService() database behavior", () => { )) as { items: { id: string }[] } expect(page.items.map(({ id }) => id)).toEqual(firstWorkspacePrototypeIds) }) - it("accepts a scoped prototype cursor with an uppercase workspace ID", async () => { const { prisma, service, store } = await database() const actor = await createActor(store, { @@ -2564,7 +2509,6 @@ describe("createPrototypeAuthorizationService() database behavior", () => { }) } }) - it("lists immutable versions by sequence and paginates in that order", async () => { const { prisma, service, store } = await database() const actor = await createActor(store, { @@ -2640,7 +2584,6 @@ describe("createPrototypeAuthorizationService() database behavior", () => { ]) expect(second.nextCursor).toBeNull() }) - it("rejects a version cursor from another prototype", async () => { const { prisma, service, store } = await database() const actor = await createActor(store, { @@ -2768,65 +2711,4 @@ describe("createPrototypeAuthorizationService() database behavior", () => { }) }) - it("creates sequential immutable versions and rejects additions after archival", async () => { - const { prisma, service, store } = await database() - await createActor(store, { - subject: "versions", - token: "versions-token", - role: "administrator", - }) - const workspace = (await service.createWorkspace("versions-token", { - name: "Workspace", - })) as { id: string } - const prototype = (await service.createPrototype("versions-token", workspace.id, { - name: "Prototype", - })) as { id: string } - - const created = await Promise.all([ - service.createVersion("versions-token", prototype.id, { label: "First request" }), - service.createVersion("versions-token", prototype.id, { label: "Second request" }), - ]) - - expect( - created - .map((version) => (version as { sequence: number }).sequence) - .sort((left, right) => left - right), - ).toEqual([1, 2]) - expect( - await prisma.prototypeVersion.findMany({ - where: { prototypeId: prototype.id }, - orderBy: { sequence: "asc" }, - select: { sequence: true }, - }), - ).toEqual([{ sequence: 1 }, { sequence: 2 }]) - - await service.archivePrototype("versions-token", prototype.id) - await expect( - service.createVersion("versions-token", prototype.id, { label: "Too late" }), - ).rejects.toMatchObject({ - status: 409, - code: "prototype_archived", - }) - expect(await prisma.prototypeVersion.count({ where: { prototypeId: prototype.id } })).toBe(2) - const audits = await prisma.securityAudit.findMany({ - where: { prototypeId: prototype.id }, - orderBy: { createdAt: "asc" }, - select: { - event: true, - context: true, - prototypeVersionId: true, - }, - }) - expect(audits.map(({ event }) => event)).toEqual([ - "prototype.create", - "prototype.version.create", - "prototype.version.create", - "prototype.archive", - ]) - expect(audits.every(({ context }) => JSON.stringify(context) === "{}")).toBe(true) - expect(audits[0]?.prototypeVersionId).toBeNull() - expect(audits[1]?.prototypeVersionId).toEqual(expect.any(String)) - expect(audits[2]?.prototypeVersionId).toEqual(expect.any(String)) - expect(audits[3]?.prototypeVersionId).toBeNull() - }) }) diff --git a/hub/server/domains/prototype-authorization/create-service/index.ts b/hub/server/domains/prototype-authorization/create-service/index.ts index 8f116720..498af4d4 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.ts @@ -1,3 +1,5 @@ +import { randomBytes, randomUUID } from "node:crypto" + import { z } from "zod" import { createIdentityRecords } from "#server/domains/identity/create-records" @@ -15,11 +17,22 @@ import type { import { getCurrentSession } from "#server/domains/authentication/get-current-session" import { saveSecurityAudit } from "#server/repositories/security-audits/save-audit" import { AuthenticationError } from "#server/utils/authentication-error" +import { digestSecret } from "#server/utils/digest-secret" import type { Prisma, PrismaClient } from "../../../generated/prisma/client.ts" const nameSchema = z.object({ name: z.string().trim().min(1).max(120) }) -const versionSchema = z.object({ label: z.string().trim().min(1).max(120).nullable().optional() }) +const publisherSchema = z + .object({ + name: z.string().trim().min(1).max(120), + actions: z + .array(z.enum(["publish", "read-status"])) + .min(1) + .max(2) + .refine((actions) => new Set(actions).size === actions.length), + expiresAt: z.iso.datetime({ offset: true }).nullable().optional(), + }) + .strict() const pageSchema = z.object({ cursor: z.uuid().optional(), limit: z.coerce.number().int().min(1).max(100).default(20), @@ -601,6 +614,52 @@ async function hasAnyPrototypeAccess({ ) } +function versionResult(version: { + id: string + prototypeId: string + sequence: number + label: string | null + artifactFormat: string | null + artifactVersion: string | null + contentDigest: string | null + sizeBytes: number | null + builtAt: Date | null + provenanceProvider: string | null + provenanceSourceUrl: string | null + provenanceRevision: string | null + provenanceRef: string | null + provenancePipeline: string | null + provenanceRunId: string | null + createdAt: Date +}) { + return { + id: version.id, + prototypeId: version.prototypeId, + sequence: version.sequence, + label: version.label, + artifact: version.contentDigest + ? { + format: version.artifactFormat, + version: version.artifactVersion, + digest: version.contentDigest, + sizeBytes: version.sizeBytes, + builtAt: version.builtAt, + } + : null, + provenance: version.provenanceProvider + ? { + provider: version.provenanceProvider, + sourceUrl: version.provenanceSourceUrl, + revision: version.provenanceRevision, + ref: version.provenanceRef, + pipeline: version.provenancePipeline, + runId: version.provenanceRunId, + } + : null, + createdAt: version.createdAt, + } +} + async function audit( database: Prisma.TransactionClient, actor: Awaited>, @@ -876,6 +935,69 @@ export function createPrototypeAuthorizationService({ } }, + async createPublisherCredential(sessionToken, prototypeId, input) { + const parsed = publisherSchema.safeParse(input) + if ( + !parsed.success || + (parsed.data.expiresAt !== null && + parsed.data.expiresAt !== undefined && + new Date(parsed.data.expiresAt) <= new Date()) + ) { + throw new AuthenticationError( + 422, + "invalid_workload_publisher", + "The workload publisher is invalid.", + ) + } + return store.lock(async (_records, transaction) => { + const actor = await currentSession(store, sessionToken, transaction) + requireFreshAuthorization(actor) + requireIdentifier(prototypeId, "prototype_not_found", "The prototype was not found.") + await lockPrototype(transaction, prototypeId) + const { prototype, decision } = await prototypeDecision( + transaction, + actor.userId, + prototypeId, + baseline, + ) + requireDecision(decision, "administrator") + const credentialId = randomUUID() + const token = `rpl_pub_${credentialId}.${randomBytes(32).toString("base64url")}` + const publisher = await transaction.workloadPublisher.create({ + data: { + prototypeId, + name: parsed.data.name, + actions: parsed.data.actions, + credentials: { + create: { + id: credentialId, + tokenDigest: digestSecret(token), + expiresAt: parsed.data.expiresAt ? new Date(parsed.data.expiresAt) : null, + }, + }, + }, + }) + await audit(transaction, actor, { + event: "prototype.publisher.create", + workspaceId: prototype.workspaceId, + prototypeId, + workloadPublisherId: publisher.id, + context: { source: "prototype_administrator" }, + }) + return { + publisher: { + id: publisher.id, + prototypeId: publisher.prototypeId, + name: publisher.name, + actions: publisher.actions, + active: publisher.active, + createdAt: publisher.createdAt, + }, + credential: { id: credentialId, token }, + } + }) + }, + async archivePrototype(sessionToken, prototypeId) { return store.lock(async (_records, transaction) => { const actor = await currentSession(store, sessionToken, transaction) @@ -1073,52 +1195,8 @@ export function createPrototypeAuthorizationService({ ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), take: limit + 1, }) - return pageResult(items, limit) - }, - - async createVersion(sessionToken, prototypeId, input) { - const parsed = versionSchema.safeParse(input) - if (!parsed.success) { - throw new AuthenticationError(422, "invalid_version", "The version is invalid.") - } - return store.lock(async (_records, transaction) => { - const actor = await currentSession(store, sessionToken, transaction) - requireIdentifier(prototypeId, "prototype_not_found", "The prototype was not found.") - await lockPrototype(transaction, prototypeId) - const { prototype, decision } = await prototypeDecision( - baseline, - transaction, - actor.userId, - prototypeId, - ) - requireDecision(decision, "editor") - if (prototype.archivedAt) { - throw new AuthenticationError( - 409, - "prototype_archived", - "Archived prototypes cannot receive versions.", - ) - } - const latest = await transaction.prototypeVersion.findFirst({ - where: { prototypeId }, - orderBy: { sequence: "desc" }, - }) - const version = await transaction.prototypeVersion.create({ - data: { - prototypeId, - createdByUserId: actor.userId, - sequence: (latest?.sequence ?? 0) + 1, - label: parsed.data.label ?? null, - }, - }) - await audit(transaction, actor, { - event: "prototype.version.create", - workspaceId: prototype.workspaceId, - prototypeId, - prototypeVersionId: version.id, - }) - return version - }) + const page = pageResult(items, limit) + return { ...page, items: page.items.map(versionResult) } }, async getVersion(sessionToken, prototypeId, versionId) { @@ -1132,7 +1210,7 @@ export function createPrototypeAuthorizationService({ if (!version) { throw new AuthenticationError(404, "version_not_found", "The version was not found.") } - return version + return versionResult(version) }, } } diff --git a/hub/server/domains/prototype-authorization/create-service/types.ts b/hub/server/domains/prototype-authorization/create-service/types.ts index ba9ad592..eec58cb6 100644 --- a/hub/server/domains/prototype-authorization/create-service/types.ts +++ b/hub/server/domains/prototype-authorization/create-service/types.ts @@ -24,6 +24,11 @@ export interface PrototypeAuthorizationService { listPrototypes(sessionToken: string, workspaceId: string, query: unknown): Promise> createPrototype(sessionToken: string, workspaceId: string, input: unknown): Promise getPrototype(sessionToken: string, prototypeId: string): Promise + createPublisherCredential( + sessionToken: string, + prototypeId: string, + input: unknown, + ): Promise archivePrototype(sessionToken: string, prototypeId: string): Promise setPrototypeAuthorization( sessionToken: string, @@ -43,7 +48,6 @@ export interface PrototypeAuthorizationService { targetUserId: string, ): Promise listVersions(sessionToken: string, prototypeId: string, query: unknown): Promise> - createVersion(sessionToken: string, prototypeId: string, input: unknown): Promise getVersion(sessionToken: string, prototypeId: string, versionId: string): Promise } diff --git a/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts b/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts new file mode 100644 index 00000000..ba016379 --- /dev/null +++ b/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts @@ -0,0 +1,67 @@ +import { createHash, randomUUID } from "node:crypto" +import { rm } from "node:fs/promises" +import path from "node:path" + +import { describe, expect, it, onTestFinished } from "vitest" + +import { createLocalArtifactStorage } from "#server/integrations/artifact-storage/create-local-storage" + +function storageRoot() { + const root = path.join(process.cwd(), `.artifact-storage-test-${randomUUID()}`) + onTestFinished(() => rm(root, { recursive: true, force: true })) + return root +} + +function digest(bytes: Uint8Array) { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}` +} + +describe("createLocalArtifactStorage()", () => { + it("requires an absolute storage root", () => { + expect(() => createLocalArtifactStorage("relative/artifacts")).toThrow( + "HUB_ARTIFACT_STORAGE_PATH must be an absolute path.", + ) + }) + + it("stores and reads artifacts by a deterministic content-addressed key", async () => { + const storage = createLocalArtifactStorage(storageRoot()) + const bytes = Buffer.from("content-addressed artifact") + const contentDigest = digest(bytes) + + const stored = await storage.put(contentDigest, bytes) + + expect(stored).toEqual({ + key: `sha256/${contentDigest.slice(7, 9)}/${contentDigest.slice(7)}`, + created: true, + }) + expect(Buffer.from(await storage.read(stored.key))).toEqual(bytes) + }) + + it("rejects mismatched bytes without overwriting existing content", async () => { + const storage = createLocalArtifactStorage(storageRoot()) + const original = Buffer.from("immutable artifact") + const contentDigest = digest(original) + + const first = await storage.put(contentDigest, original) + await expect(storage.put(contentDigest, Buffer.from("different bytes"))).rejects.toThrow( + "Artifact bytes do not match the content digest.", + ) + + expect(Buffer.from(await storage.read(first.key))).toEqual(original) + }) + + it("converges concurrent writes for one digest on a single stored object", async () => { + const storage = createLocalArtifactStorage(storageRoot()) + const bytes = Buffer.from("concurrent artifact") + const contentDigest = digest(bytes) + + const results = await Promise.all([ + storage.put(contentDigest, bytes), + storage.put(contentDigest, bytes), + ]) + + expect(new Set(results.map(({ key }) => key))).toEqual(new Set([results[0].key])) + expect(results.filter(({ created }) => created)).toHaveLength(1) + expect(Buffer.from(await storage.read(results[0].key))).toEqual(bytes) + }) +}) diff --git a/hub/server/integrations/artifact-storage/create-local-storage/index.ts b/hub/server/integrations/artifact-storage/create-local-storage/index.ts new file mode 100644 index 00000000..544a2fee --- /dev/null +++ b/hub/server/integrations/artifact-storage/create-local-storage/index.ts @@ -0,0 +1,96 @@ +import { createHash, randomUUID, timingSafeEqual } from "node:crypto" +import { constants } from "node:fs" +import { access, link, mkdir, open, readFile, rm } from "node:fs/promises" +import path from "node:path" + +import type { ArtifactStorage, StoredArtifact } from "#server/integrations/artifact-storage/types" + +function artifactKey(contentDigest: string): string { + if (!/^sha256:[a-f0-9]{64}$/.test(contentDigest)) { + throw new Error("Artifact content digest is invalid.") + } + const digest = contentDigest.replace(/^sha256:/, "") + return path.posix.join("sha256", digest.slice(0, 2), digest) +} + +function storagePath(root: string, key: string): string { + const candidate = path.resolve(root, key) + const resolvedRoot = path.resolve(root) + if (!candidate.startsWith(`${resolvedRoot}${path.sep}`)) { + throw new Error("Artifact storage key is invalid.") + } + return candidate +} + +function calculateContentDigest(bytes: Uint8Array): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}` +} + +async function verifyExisting(destination: string, contentDigest: string): Promise { + const expected = Buffer.from(contentDigest.replace(/^sha256:/, ""), "hex") + const actual = createHash("sha256") + .update(await readFile(destination)) + .digest() + if (!timingSafeEqual(expected, actual)) { + throw new Error("Stored artifact content does not match its key.") + } +} + +function isFileSystemError(error: unknown, code: string): boolean { + return ( + error instanceof Error && + "code" in error && + typeof error.code === "string" && + error.code === code + ) +} + +export function createLocalArtifactStorage(root: string): ArtifactStorage { + if (!path.isAbsolute(root)) { + throw new Error("HUB_ARTIFACT_STORAGE_PATH must be an absolute path.") + } + return { + async put(contentDigest: string, bytes: Uint8Array): Promise { + const key = artifactKey(contentDigest) + if (calculateContentDigest(bytes) !== contentDigest) { + throw new Error("Artifact bytes do not match the content digest.") + } + const destination = storagePath(root, key) + await mkdir(path.dirname(destination), { recursive: true }) + try { + await access(destination, constants.F_OK) + await verifyExisting(destination, contentDigest) + return { key, created: false } + } catch (error) { + if (!isFileSystemError(error, "ENOENT")) throw error + } + + const temporary = `${destination}.${randomUUID()}.tmp` + const file = await open(temporary, "wx", 0o600) + try { + await file.writeFile(bytes) + await file.sync() + } finally { + await file.close() + } + try { + await link(temporary, destination) + return { key, created: true } + } catch (error) { + if (!isFileSystemError(error, "EEXIST")) throw error + await verifyExisting(destination, contentDigest) + return { key, created: false } + } finally { + await rm(temporary, { force: true }) + } + }, + + async read(key: string): Promise { + return readFile(storagePath(root, key)) + }, + + async remove(key: string): Promise { + await rm(storagePath(root, key), { force: true }) + }, + } +} diff --git a/hub/server/integrations/artifact-storage/types.ts b/hub/server/integrations/artifact-storage/types.ts new file mode 100644 index 00000000..43f08e07 --- /dev/null +++ b/hub/server/integrations/artifact-storage/types.ts @@ -0,0 +1,10 @@ +export interface StoredArtifact { + key: string + created: boolean +} + +export interface ArtifactStorage { + put(contentDigest: string, bytes: Uint8Array): Promise + read(key: string): Promise + remove(key: string): Promise +} diff --git a/hub/server/repositories/security-audits/save-audit/index.ts b/hub/server/repositories/security-audits/save-audit/index.ts index 12c3a6dd..d139afde 100644 --- a/hub/server/repositories/security-audits/save-audit/index.ts +++ b/hub/server/repositories/security-audits/save-audit/index.ts @@ -27,6 +27,7 @@ export async function saveSecurityAudit(database: Prisma.TransactionClient, data workspaceId: data.workspaceId ?? null, prototypeId: data.prototypeId ?? null, prototypeVersionId: data.prototypeVersionId ?? null, + workloadPublisherId: data.workloadPublisherId ?? null, targetUserId: data.targetUserId ?? null, githubRepositorySourceId: data.githubRepositorySourceId ?? null, context, diff --git a/hub/server/repositories/security-audits/types.ts b/hub/server/repositories/security-audits/types.ts index ef4c2ef7..05be2a06 100644 --- a/hub/server/repositories/security-audits/types.ts +++ b/hub/server/repositories/security-audits/types.ts @@ -9,6 +9,7 @@ export interface AuditInput { workspaceId?: string | null prototypeId?: string | null prototypeVersionId?: string | null + workloadPublisherId?: string | null targetUserId?: string | null githubRepositorySourceId?: string | null context?: Record @@ -26,6 +27,7 @@ export interface SecurityAuditRecord { workspaceId: string | null prototypeId: string | null prototypeVersionId: string | null + workloadPublisherId: string | null targetUserId: string | null githubRepositorySourceId: string | null context: unknown diff --git a/hub/server/routes/index.ts b/hub/server/routes/index.ts index 1720c2cf..c648ece9 100644 --- a/hub/server/routes/index.ts +++ b/hub/server/routes/index.ts @@ -19,7 +19,7 @@ import { post as handleApiSessionVerifyPost } from "#server/routes/api/session/v import { archivePrototype, createPrototype, - createVersion, + createPublisherCredential, createWorkspace, explainPrototypeAccess, getPrototype, @@ -42,6 +42,7 @@ import { unlinkRepository, updateRepository, } from "#server/routes/github-authorization" +import { getPublicationStatus, promoteVersion, publishArtifact } from "#server/routes/publishing" import type { AuthenticationRoutes } from "#server/routes/types" /** Keep endpoint registration separate from the app's shared middleware. */ @@ -153,6 +154,11 @@ export function routes(app: Express, authentication?: AuthenticationRoutes): voi productMutationLimit, setPrototypeAuthorization.bind(null, authentication), ) + app.post( + "/api/prototypes/:prototypeId/publishers", + productMutationLimit, + createPublisherCredential.bind(null, authentication), + ) app.get( "/api/prototypes/:prototypeId/access/explanation", explainPrototypeAccess.bind(null, authentication), @@ -168,12 +174,30 @@ export function routes(app: Express, authentication?: AuthenticationRoutes): voi removePrototypeAccess.bind(null, authentication), ) app.get("/api/prototypes/:prototypeId/versions", listVersions.bind(null, authentication)) - app.post( - "/api/prototypes/:prototypeId/versions", - productMutationLimit, - createVersion.bind(null, authentication), - ) app.get("/api/prototypes/:prototypeId/versions/:versionId", getVersion.bind(null, authentication)) app.use(["/api/workspaces", "/api/prototypes"], respondAdministrationError) + if (authentication.artifactPublishing) { + const publishingLimit = createRateLimit({ + limit: 60, + windowMilliseconds: 5 * 60_000, + message: "Too many publishing requests.", + }) + app.post( + "/api/publishing/prototypes/:prototypeId/versions", + publishingLimit, + publishArtifact.bind(null, authentication.artifactPublishing), + ) + app.get( + "/api/publishing/prototypes/:prototypeId/status", + publishingLimit, + getPublicationStatus.bind(null, authentication.artifactPublishing), + ) + app.put( + "/api/publishing/prototypes/:prototypeId/channels/:channel", + publishingLimit, + promoteVersion.bind(null, authentication.artifactPublishing), + ) + app.use("/api/publishing", respondAdministrationError) + } app.use(respondAuthenticationError) } diff --git a/hub/server/routes/product/index.test.ts b/hub/server/routes/product/index.test.ts index 40893e1a..ef393101 100644 --- a/hub/server/routes/product/index.test.ts +++ b/hub/server/routes/product/index.test.ts @@ -39,11 +39,11 @@ function runtime( getPrototype: vi.fn(), archivePrototype: vi.fn(), setPrototypeAuthorization: vi.fn(), + createPublisherCredential: vi.fn(), explainPrototypeAccess: vi.fn(), setPrototypeAccess: vi.fn(), removePrototypeAccess: vi.fn(), listVersions: vi.fn(), - createVersion: vi.fn(), getVersion: vi.fn(), }, overrides, @@ -228,6 +228,12 @@ describe("product routes", () => { body: { name: "P" }, }, { method: "GET", path: "/api/prototypes/prototype", call: "getPrototype" }, + { + method: "POST", + path: "/api/prototypes/prototype/publishers", + call: "createPublisherCredential", + body: { name: "Local publisher", actions: ["publish"] }, + }, { method: "POST", path: "/api/prototypes/prototype/archive", @@ -263,12 +269,6 @@ describe("product routes", () => { path: "/api/prototypes/prototype/versions", call: "listVersions", }, - { - method: "POST", - path: "/api/prototypes/prototype/versions", - call: "createVersion", - body: { label: "Version" }, - }, { method: "GET", path: "/api/prototypes/prototype/versions/version", diff --git a/hub/server/routes/product/index.ts b/hub/server/routes/product/index.ts index 00d0ea2a..bebe917d 100644 --- a/hub/server/routes/product/index.ts +++ b/hub/server/routes/product/index.ts @@ -152,6 +152,19 @@ export async function setPrototypeAuthorization( response.json({ prototype }) } +export async function createPublisherCredential( + authentication: AuthenticationRoutes, + request: Request, + response: Response, +): Promise { + const result = await authentication.prototypeAuthorization.createPublisherCredential( + mutationSessionToken(authentication, request), + parameter(request, "prototypeId"), + request.body, + ) + response.status(201).json(result) +} + export async function explainPrototypeAccess( authentication: AuthenticationRoutes, request: Request, @@ -205,19 +218,6 @@ export async function listVersions( ) } -export async function createVersion( - authentication: AuthenticationRoutes, - request: Request, - response: Response, -): Promise { - const version = await authentication.prototypeAuthorization.createVersion( - mutationSessionToken(authentication, request), - parameter(request, "prototypeId"), - request.body, - ) - response.status(201).json({ version }) -} - export async function getVersion( authentication: AuthenticationRoutes, request: Request, diff --git a/hub/server/routes/publishing.test.ts b/hub/server/routes/publishing.test.ts new file mode 100644 index 00000000..c22888f5 --- /dev/null +++ b/hub/server/routes/publishing.test.ts @@ -0,0 +1,235 @@ +import { once } from "node:events" +import { createServer } from "node:http" +import type { AddressInfo } from "node:net" + +import { describe, expect, it, onTestFinished, vi } from "vitest" +import type { Mocked } from "vitest" + +import { maximumPublishingJsonBytes } from "#server/domains/artifact-publishing/create-service/constants" +import type { ArtifactPublishingService } from "#server/domains/artifact-publishing/create-service/types" +import type { AdministrationService } from "#server/domains/administration/create-service/types" +import type { AuthenticationService } from "#server/domains/authentication/create-service/types" +import type { PrototypeAuthorizationService } from "#server/domains/prototype-authorization/create-service/types" +import { createLogger } from "#server/integrations/pino/create-logger" +import type { AuthenticationRoutes } from "#server/routes/types" + +import { createApp } from "../create-app/index.ts" + +function runtime( + overrides: Partial> = {}, +): AuthenticationRoutes & { artifactPublishing: Mocked } { + const artifactPublishing = Object.assign( + { + publish: vi.fn(), + getStatus: vi.fn(), + promote: vi.fn(), + }, + overrides, + ) as Mocked + return { + service: {} as Mocked, + administration: {} as Mocked, + prototypeAuthorization: {} as Mocked, + artifactPublishing, + settings: { + origin: "https://hub.example", + secureCookies: true, + configured: true, + admissionPolicy: "hub-managed", + trustedProxyHops: 0, + }, + } +} + +async function serve(authentication: AuthenticationRoutes, logger = createLogger()) { + const server = createServer(createApp({ authentication, logger })) + server.listen(0, "127.0.0.1") + await once(server, "listening") + onTestFinished( + () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())) + }), + ) + const { port } = server.address() as AddressInfo + return `http://127.0.0.1:${port}` +} + +describe("publishing routes", () => { + it("passes workload authorization and idempotency to publication without browser state", async () => { + const publish = vi.fn().mockResolvedValue({ + version: { id: "version-id" }, + replayed: false, + promotedChannel: null, + }) + const authentication = runtime({ + publish, + }) + const baseUrl = await serve(authentication) + const body = { + manifest: { schemaVersion: 1 }, + artifactBase64: "cHJpdmF0ZS1hcnRpZmFjdA==", + } + + const response = await fetch(`${baseUrl}/api/publishing/prototypes/prototype-id/versions`, { + method: "POST", + headers: { + authorization: "Bearer workload-token", + "content-type": "application/json", + "idempotency-key": "request-key", + }, + body: JSON.stringify(body), + }) + + expect(response.status).toBe(201) + expect(await response.json()).toEqual({ + version: { id: "version-id" }, + replayed: false, + promotedChannel: null, + }) + expect(publish).toHaveBeenCalledWith( + "Bearer workload-token", + "prototype-id", + "request-key", + body, + ) + }) + + it("returns 200 for an idempotent publication replay", async () => { + const authentication = runtime({ + publish: vi.fn().mockResolvedValue({ + version: { id: "same-version" }, + replayed: true, + promotedChannel: "preview", + }), + }) + const baseUrl = await serve(authentication) + + const response = await fetch(`${baseUrl}/api/publishing/prototypes/prototype-id/versions`, { + method: "POST", + headers: { + authorization: "Bearer workload-token", + "content-type": "application/json", + "idempotency-key": "retry-key", + }, + body: JSON.stringify({ manifest: {}, artifactBase64: "" }), + }) + + expect(response.status).toBe(200) + expect(await response.json()).toMatchObject({ + version: { id: "same-version" }, + replayed: true, + }) + }) + + it("routes status reads and channel promotion through workload credentials", async () => { + const getStatus = vi.fn().mockResolvedValue({ publicationId: "publication-id" }) + const promote = vi.fn().mockResolvedValue({ + name: "current", + prototypeVersionId: "version-id", + }) + const authentication = runtime({ + getStatus, + promote, + }) + const baseUrl = await serve(authentication) + const headers = { + authorization: "Bearer workload-token", + "idempotency-key": "request-key", + } + + const status = await fetch(`${baseUrl}/api/publishing/prototypes/prototype-id/status`, { + headers, + }) + const promotion = await fetch( + `${baseUrl}/api/publishing/prototypes/prototype-id/channels/current`, + { + method: "PUT", + headers: { ...headers, "content-type": "application/json" }, + body: JSON.stringify({ versionId: "version-id" }), + }, + ) + + expect(status.status).toBe(200) + expect(await status.json()).toEqual({ publicationId: "publication-id" }) + expect(getStatus).toHaveBeenCalledWith("Bearer workload-token", "prototype-id", "request-key") + expect(promotion.status).toBe(200) + expect(await promotion.json()).toEqual({ + channel: { name: "current", prototypeVersionId: "version-id" }, + }) + expect(promote).toHaveBeenCalledWith("Bearer workload-token", "prototype-id", "current", { + versionId: "version-id", + }) + }) + + it("omits workload authorization, query values, and artifact content from request logs", async () => { + const records: Record[] = [] + const logger = createLogger({ + destination: { + write: (line) => records.push(JSON.parse(line) as Record), + }, + }) + const authentication = runtime({ + publish: vi.fn().mockResolvedValue({ + version: { id: "version-id" }, + replayed: false, + promotedChannel: null, + }), + }) + const baseUrl = await serve(authentication, logger) + + const response = await fetch( + `${baseUrl}/api/publishing/prototypes/prototype-id/versions?note=private-query`, + { + method: "POST", + headers: { + authorization: "Bearer private-workload-token", + "content-type": "application/json", + "idempotency-key": "request-key", + }, + body: JSON.stringify({ + manifest: { provenance: { revision: "private-revision" } }, + artifactBase64: "cHJpdmF0ZS1hcnRpZmFjdC1jb250ZW50", + }), + }, + ) + + expect(response.status).toBe(201) + await vi.waitFor(() => expect(records).toHaveLength(1)) + expect(records[0]).not.toHaveProperty("query") + expect(records[0]).not.toHaveProperty("body") + expect(JSON.stringify(records)).not.toMatch( + /private-workload-token|private-query|private-revision|cHJpdmF0ZS1hcnRpZmFjdC1jb250ZW50/, + ) + }) + + it("rejects a publishing body above its dedicated limit before the service runs", async () => { + const records: Record[] = [] + const logger = createLogger({ + destination: { + write: (line) => records.push(JSON.parse(line) as Record), + }, + }) + const publish = vi.fn() + const authentication = runtime({ publish }) + const baseUrl = await serve(authentication, logger) + const privateContent = "x".repeat(maximumPublishingJsonBytes) + + const response = await fetch(`${baseUrl}/api/publishing/prototypes/prototype-id/versions`, { + method: "POST", + headers: { + authorization: "Bearer private-workload-token", + "content-type": "application/json", + "idempotency-key": "oversized-request", + }, + body: JSON.stringify({ artifactBase64: privateContent }), + }) + + expect(response.status).toBe(413) + expect(await response.json()).toEqual({ error: "Invalid request" }) + expect(publish).not.toHaveBeenCalled() + await vi.waitFor(() => expect(records).toHaveLength(1)) + expect(records[0]).not.toHaveProperty("body") + expect(JSON.stringify(records)).not.toContain(privateContent.slice(0, 100)) + }) +}) diff --git a/hub/server/routes/publishing/index.ts b/hub/server/routes/publishing/index.ts new file mode 100644 index 00000000..e87057e4 --- /dev/null +++ b/hub/server/routes/publishing/index.ts @@ -0,0 +1,58 @@ +import type { Request, Response } from "express" + +import type { ArtifactPublishingService } from "#server/domains/artifact-publishing/create-service/types" + +function parameter(request: Request, name: string): string { + const value = request.params[name] + return typeof value === "string" ? value : "" +} + +function authorization(request: Request): string { + return request.get("authorization") ?? "" +} + +function idempotencyKey(request: Request): string { + return request.get("idempotency-key") ?? "" +} + +export async function publishArtifact( + artifactPublishing: ArtifactPublishingService, + request: Request, + response: Response, +): Promise { + const result = await artifactPublishing.publish( + authorization(request), + parameter(request, "prototypeId"), + idempotencyKey(request), + request.body, + ) + response.status(result.replayed ? 200 : 201).json(result) +} + +export async function getPublicationStatus( + artifactPublishing: ArtifactPublishingService, + request: Request, + response: Response, +): Promise { + response.json( + await artifactPublishing.getStatus( + authorization(request), + parameter(request, "prototypeId"), + idempotencyKey(request), + ), + ) +} + +export async function promoteVersion( + artifactPublishing: ArtifactPublishingService, + request: Request, + response: Response, +): Promise { + const channel = await artifactPublishing.promote( + authorization(request), + parameter(request, "prototypeId"), + parameter(request, "channel"), + request.body, + ) + response.json({ channel }) +} diff --git a/hub/server/routes/types.ts b/hub/server/routes/types.ts index 530c61ce..d4a86df7 100644 --- a/hub/server/routes/types.ts +++ b/hub/server/routes/types.ts @@ -5,11 +5,13 @@ import type { import type { AdministrationService } from "#server/domains/administration/create-service/types" import type { PrototypeAuthorizationService } from "#server/domains/prototype-authorization/create-service/types" import type { GitHubAuthorizationService } from "#server/domains/github-authorization/create-service/types" +import type { ArtifactPublishingService } from "#server/domains/artifact-publishing/create-service/types" export interface AuthenticationRoutes { service: AuthenticationService administration: AdministrationService prototypeAuthorization: PrototypeAuthorizationService githubAuthorization?: GitHubAuthorizationService + artifactPublishing?: ArtifactPublishingService settings: AuthenticationSettings } From 6d3f4a2c9412e2d2b99f373675b18ee63163e053 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Tue, 27 Oct 2026 16:42:31 -0700 Subject: [PATCH 02/13] fix(hub): harden artifact publishing boundaries Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- hub/docs/artifact-ingestion.md | 8 +- hub/docs/database.md | 4 +- hub/docs/prototype-authorization.md | 2 + .../migration.sql | 5 + hub/schema/security-audit.prisma | 2 + .../create-service/index.db.test.ts | 195 ++++++++++++ .../create-service/index.ts | 24 +- .../create-service/index.db.test.ts | 282 ++++++++++++++++++ .../create-service/index.ts | 120 ++++++++ .../create-service/types.ts | 12 + .../create-local-storage/index.test.ts | 139 ++++++++- .../create-local-storage/index.ts | 159 ++++++++-- .../security-audits/save-audit/index.ts | 1 + .../repositories/security-audits/types.ts | 2 + hub/server/routes/index.ts | 12 + hub/server/routes/product/index.test.ts | 13 + hub/server/routes/product/index.ts | 28 ++ 17 files changed, 973 insertions(+), 35 deletions(-) create mode 100644 hub/schema/migrations/20261027231000_publisher_lifecycle/migration.sql diff --git a/hub/docs/artifact-ingestion.md b/hub/docs/artifact-ingestion.md index b2c5bdd1..8f79582c 100644 --- a/hub/docs/artifact-ingestion.md +++ b/hub/docs/artifact-ingestion.md @@ -13,7 +13,9 @@ A prototype administrator creates a workload publisher with one or both actions: Hub returns a bearer credential once and stores only its SHA-256 digest. The credential is scoped to one prototype through its workload publisher. It cannot create a browser session, view a prototype through user APIs, administer access, or publish to another prototype. A producer's source URL, revision, ref, pipeline, and run ID describe where a build came from; they never grant access. -Browser users provision credentials through `POST /api/prototypes/:prototypeId/publishers`. This requires a fresh prototype-administrator session and returns the raw token only in that response. +Browser users provision credentials through `POST /api/prototypes/:prototypeId/publishers`. This requires a fresh prototype-administrator session and returns the raw token only in that response. The same administrator may temporarily disable or re-enable a publisher with `PATCH /api/prototypes/:prototypeId/publishers/:publisherId`, or permanently revoke one credential with `DELETE /api/prototypes/:prototypeId/publishers/:publisherId/credentials/:credentialId`. Hub audits each state change. Re-enabling a publisher does not restore a revoked credential. + +Archiving a prototype blocks new publishers, publication, publisher re-enablement, and `current` or `preview` promotion. Administrators may still disable publishers and revoke credentials after archival. Existing versions and publication status remain readable through their authorized interfaces. ## Manifest contract @@ -45,13 +47,13 @@ Browser users provision credentials through `POST /api/prototypes/:prototypeId/p } ``` -The manifest is closed to unknown fields. Hub bounds every string, accepts only HTTP or HTTPS source URLs, limits decoded artifacts to 8 MiB in this reference layer, verifies the decoded size, and recalculates the SHA-256 digest before storing bytes. The manifest prototype ID must match the route. +The manifest is closed to unknown fields. Hub bounds every string, accepts only HTTP or HTTPS source URLs without embedded usernames or passwords, limits decoded artifacts to 8 MiB in this reference layer, verifies the decoded size, and recalculates the SHA-256 digest before storing bytes. The manifest prototype ID must match the route. ## Storage boundary `ArtifactStorage` separates ingestion from byte storage. The reference adapter writes content-addressed files under `HUB_ARTIFACT_STORAGE_PATH`, defaulting to `hub/.data/artifacts` for local use. Database rows store only the storage key and validated metadata, never the artifact bytes. -The local adapter is suitable for one Hub process with a durable mounted filesystem. It does not provide replication, remote durability, encryption-key management, or multi-region consistency. A production object-store adapter must preserve the same content-addressed, immutable `put`, `read`, and `remove` boundary. +The local adapter is suitable for one Hub process with a durable mounted filesystem. It rejects symbolic-link roots, shards, and artifact paths, resolves directories under the configured root, and opens artifact files without following a final symbolic link. Operators must keep the storage directory writable only by the Hub workload; the local adapter does not provide replication, remote durability, encryption-key management, hostile concurrent filesystem isolation, or multi-region consistency. A production object-store adapter must preserve the same content-addressed, immutable `put`, `read`, and `remove` boundary. Storage happens before the database transaction. A failed transaction can therefore leave an unreferenced content-addressed file. Hub never deletes referenced bytes automatically. Operators may remove only files that have no matching `PrototypeVersion.storageKey`; retain unreferenced files long enough to exclude in-flight transactions and backups. Version metadata has no deletion API and must not be rewritten silently. diff --git a/hub/docs/database.md b/hub/docs/database.md index faf0ed19..42ff40c9 100644 --- a/hub/docs/database.md +++ b/hub/docs/database.md @@ -57,10 +57,10 @@ GitHub synchronization uses separate records: - `GitHubSynchronizedAccess` stores one user's raw permission, mapped prototype role, admission eligibility, source, status, and last confirmation for one repository source. - `GitHubAuthorizationPolicy` stores the bounded last-known-good duration. The initial policy is one day and may not exceed seven days. - `GitHubWebhookDelivery` deduplicates verified deliveries by GitHub delivery ID. -`Prototype.authorizationMode` defaults to `hub-managed`. `sourceRepositoryId` is nullable and applies only when the mode is explicitly `github-synchronized`. Synchronized records never share rows with workspace or prototype grants, so reconciliation cannot overwrite manual Hub decisions. + `Prototype.authorizationMode` defaults to `hub-managed`. `sourceRepositoryId` is nullable and applies only when the mode is explicitly `github-synchronized`. Synchronized records never share rows with workspace or prototype grants, so reconciliation cannot overwrite manual Hub decisions. -A database trigger rejects prototype version updates and deletions so history cannot be rewritten or silently removed. Workload credentials store token digests only. Publication rows bind a publisher-scoped idempotency-key digest to one version, and channel rows keep mutable pointers separate from immutable version metadata. Artifact bytes stay outside relational columns behind the storage boundary. +A database trigger rejects prototype version updates and deletions so history cannot be rewritten or silently removed. Workload credentials store token digests plus optional expiry and revocation timestamps; publishers have an independent active flag for temporary suspension. Publication rows bind a publisher-scoped idempotency-key digest to one version, and channel rows keep mutable pointers separate from immutable version metadata. Artifact bytes stay outside relational columns behind the storage boundary. Installation membership `source` distinguishes linked-source synchronization from the `legacy-synchronized` compatibility policy. Any administrator lifecycle change or assignment of Hub administration converts a synchronized membership to `manual`, so reconciliation cannot undo the Hub override or orphan administration. diff --git a/hub/docs/prototype-authorization.md b/hub/docs/prototype-authorization.md index 1bf32fe2..56fabb6b 100644 --- a/hub/docs/prototype-authorization.md +++ b/hub/docs/prototype-authorization.md @@ -77,6 +77,8 @@ The product API uses: - `GET /api/prototypes/:prototypeId/versions` - `GET /api/prototypes/:prototypeId/versions/:versionId` - `POST /api/prototypes/:prototypeId/publishers` +- `PATCH /api/prototypes/:prototypeId/publishers/:publisherId` +- `DELETE /api/prototypes/:prototypeId/publishers/:publisherId/credentials/:credentialId` List routes use an opaque UUID cursor and a limit from 1 through 100. Prototype and version cursors belong to their owning workspace or prototype; a cursor from another scope is invalid. Invalid pagination returns `422 invalid_pagination`, while malformed or missing path resources return their resource-specific `404` error. State-changing requests require the configured origin and matching CSRF cookie and header. Product responses are private and non-cacheable, request payloads are omitted from logs, and mutation routes use a bounded rate limit. diff --git a/hub/schema/migrations/20261027231000_publisher_lifecycle/migration.sql b/hub/schema/migrations/20261027231000_publisher_lifecycle/migration.sql new file mode 100644 index 00000000..bbac9505 --- /dev/null +++ b/hub/schema/migrations/20261027231000_publisher_lifecycle/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "security_audits" ADD COLUMN "workload_credential_id" UUID; + +-- CreateIndex +CREATE INDEX "security_audits_workload_credential_id_idx" ON "security_audits"("workload_credential_id"); diff --git a/hub/schema/security-audit.prisma b/hub/schema/security-audit.prisma index cbc1f36a..7075b111 100644 --- a/hub/schema/security-audit.prisma +++ b/hub/schema/security-audit.prisma @@ -10,6 +10,7 @@ model SecurityAudit { prototypeId String? @map("prototype_id") @db.Uuid prototypeVersionId String? @map("prototype_version_id") @db.Uuid workloadPublisherId String? @map("workload_publisher_id") @db.Uuid + workloadCredentialId String? @map("workload_credential_id") @db.Uuid targetUserId String? @map("target_user_id") @db.Uuid githubRepositorySourceId String? @map("github_repository_source_id") @db.Uuid @@ -34,6 +35,7 @@ model SecurityAudit { @@index([prototypeId]) @@index([prototypeVersionId]) @@index([workloadPublisherId]) + @@index([workloadCredentialId]) @@index([targetUserId]) @@index([githubRepositorySourceId]) @@map("security_audits") diff --git a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts index 0c6103b9..a25e1418 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts @@ -141,6 +141,44 @@ async function createAdministratorSession(prisma: PrismaClient, token: string) { return { actor, store } } +async function provisionPublisher(prisma: PrismaClient, sessionToken: string) { + const { actor, store } = await createAdministratorSession(prisma, sessionToken) + const workspace = await prisma.workspace.create({ + data: { + name: "Publisher workspace", + createdByUserId: actor.id, + grants: { + create: { + userId: actor.id, + effect: "allow", + role: "administrator", + }, + }, + }, + }) + const prototype = await prisma.prototype.create({ + data: { + name: "Publisher prototype", + workspaceId: workspace.id, + createdByUserId: actor.id, + }, + }) + const authorization = createPrototypeAuthorizationService({ database: prisma, store }) + const provisioned = (await authorization.createPublisherCredential(sessionToken, prototype.id, { + name: "CI publisher", + actions: ["publish", "read-status"], + })) as { + publisher: { id: string } + credential: { id: string; token: string } + } + return { + authorization, + prototype, + provisioned, + workloadAuthorization: `Bearer ${provisioned.credential.token}`, + } +} + async function createCredential( prisma: PrismaClient, prototypeId: string, @@ -219,6 +257,32 @@ describe("createArtifactPublishingService() database behavior", () => { }, }), }, + { + name: "provenance URL username", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { + ...valid.manifest, + provenance: { + ...valid.manifest.provenance, + sourceUrl: "https://publisher@ci.example/runs/123", + }, + }, + }), + }, + { + name: "provenance URL password", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { + ...valid.manifest, + provenance: { + ...valid.manifest.provenance, + sourceUrl: "https://publisher:secret@ci.example/runs/123", + }, + }, + }), + }, { name: "different prototype identifier", change: (valid: ReturnType) => ({ @@ -335,6 +399,97 @@ describe("createArtifactPublishingService() database behavior", () => { expect(JSON.stringify(saved)).not.toContain(provisioned.credential.token) }) + it("accepts and persists an ordinary HTTPS provenance source URL", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const sourceUrl = "https://ci.example/runs/ordinary" + + const result = await context.service.publish( + credential.authorization, + prototype.id, + "ordinary-source-url", + publicationInput(prototype.id, artifact("ordinary-source-url"), { + manifest: manifest(prototype.id, artifact("ordinary-source-url"), { + provenance: { + provider: "build-system", + sourceUrl, + revision: "abc123", + ref: "refs/heads/main", + pipeline: "publish", + runId: "123", + }, + }), + }), + ) + + expect( + await context.prisma.prototypeVersion.findUniqueOrThrow({ + where: { id: publishedVersion(result).id }, + select: { provenanceSourceUrl: true }, + }), + ).toEqual({ provenanceSourceUrl: sourceUrl }) + }) + + it("keeps a revoked credential unusable after its publisher is disabled and re-enabled", async () => { + const context = await database() + const sessionToken = "publisher-lifecycle-session" + const { authorization, prototype, provisioned, workloadAuthorization } = + await provisionPublisher(context.prisma, sessionToken) + + await context.service.publish( + workloadAuthorization, + prototype.id, + "before-disable", + publicationInput(prototype.id, artifact("before-disable")), + ) + await authorization.setPublisherActive(sessionToken, prototype.id, provisioned.publisher.id, { + active: false, + }) + await expect( + context.service.publish( + workloadAuthorization, + prototype.id, + "while-disabled", + publicationInput(prototype.id, artifact("while-disabled")), + ), + ).rejects.toMatchObject({ status: 403, code: "publisher_access_denied" }) + + await authorization.setPublisherActive(sessionToken, prototype.id, provisioned.publisher.id, { + active: true, + }) + await context.service.publish( + workloadAuthorization, + prototype.id, + "after-enable", + publicationInput(prototype.id, artifact("after-enable")), + ) + await authorization.revokePublisherCredential( + sessionToken, + prototype.id, + provisioned.publisher.id, + provisioned.credential.id, + ) + await authorization.setPublisherActive(sessionToken, prototype.id, provisioned.publisher.id, { + active: false, + }) + await authorization.setPublisherActive(sessionToken, prototype.id, provisioned.publisher.id, { + active: true, + }) + + await expect( + context.service.publish( + workloadAuthorization, + prototype.id, + "after-revocation", + publicationInput(prototype.id, artifact("after-revocation")), + ), + ).rejects.toMatchObject({ status: 403, code: "publisher_access_denied" }) + expect( + await context.prisma.prototypeVersion.count({ where: { prototypeId: prototype.id } }), + ).toBe(2) + }) + it("enforces publish and read-status action scopes independently", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) @@ -544,6 +699,46 @@ describe("createArtifactPublishingService() database behavior", () => { ]) }) + it("blocks publication and current or preview promotion after a prototype is archived", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const published = await context.service.publish( + credential.authorization, + prototype.id, + "before-archive", + publicationInput(prototype.id, artifact("before-archive")), + ) + const version = publishedVersion(published) + await context.prisma.prototype.update({ + where: { id: prototype.id }, + data: { archivedAt: new Date() }, + }) + + await expect( + context.service.publish( + credential.authorization, + prototype.id, + "after-archive", + publicationInput(prototype.id, artifact("after-archive")), + ), + ).rejects.toMatchObject({ status: 409, code: "prototype_archived" }) + for (const channel of ["current", "preview"] as const) { + await expect( + context.service.promote(credential.authorization, prototype.id, channel, { + versionId: version.id, + }), + ).rejects.toMatchObject({ status: 409, code: "prototype_archived" }) + } + + expect( + await context.prisma.prototypeVersion.count({ where: { prototypeId: prototype.id } }), + ).toBe(1) + expect( + await context.prisma.prototypeChannel.count({ where: { prototypeId: prototype.id } }), + ).toBe(0) + }) + it("rolls back a published version when its audit cannot be saved", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) diff --git a/hub/server/domains/artifact-publishing/create-service/index.ts b/hub/server/domains/artifact-publishing/create-service/index.ts index e6316bb3..39ad72d0 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.ts @@ -30,7 +30,14 @@ const manifestSchema = z sourceUrl: z .url() .max(2_048) - .refine((value) => ["http:", "https:"].includes(new URL(value).protocol)) + .refine((value) => { + const url = new URL(value) + return ( + ["http:", "https:"].includes(url.protocol) && + url.username.length === 0 && + url.password.length === 0 + ) + }) .optional(), revision: boundedIdentifier.optional(), ref: boundedIdentifier.optional(), @@ -148,6 +155,17 @@ export function createArtifactPublishingService({ database: PrismaClient storage: ArtifactStorage }): ArtifactPublishingService { + async function requireUnarchivedPrototype(prototypeId: string) { + const prototype = await database.prototype.findUnique({ + where: { id: prototypeId }, + select: { archivedAt: true }, + }) + if (!prototype) error(404, "prototype_not_found", "The prototype was not found.") + if (prototype.archivedAt) { + error(409, "prototype_archived", "Archived prototypes cannot be published.") + } + } + async function authenticate(authorization: string, prototypeId: string, action: PublisherAction) { parsePrototypeId(prototypeId) const match = authorizationSchema.exec(authorization) @@ -176,6 +194,7 @@ export function createArtifactPublishingService({ return { async publish(authorization, prototypeId, idempotencyKey, input) { const publisher = await authenticate(authorization, prototypeId, "publish") + await requireUnarchivedPrototype(prototypeId) const parsed = publishSchema.safeParse(input) if (!parsed.success || parsed.data.manifest.prototypeId !== prototypeId) { error(422, "invalid_artifact_manifest", "The artifact manifest is invalid.") @@ -336,6 +355,9 @@ export function createArtifactPublishingService({ await lockPrototype(transaction, prototypeId) const prototype = await transaction.prototype.findUnique({ where: { id: prototypeId } }) if (!prototype) error(404, "prototype_not_found", "The prototype was not found.") + if (prototype.archivedAt) { + error(409, "prototype_archived", "Archived prototypes cannot be promoted.") + } const version = await transaction.prototypeVersion.findFirst({ where: { id: parsed.data.versionId, prototypeId }, }) diff --git a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts index 14fac68b..250d9cde 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts @@ -137,6 +137,58 @@ async function holdPrototypeLock(database: PrismaClient, prototypeId: string) { return { holder, release: released.resolve } } +async function createAdministeredPrototype({ + service, + store, + subject, + token, +}: { + service: ReturnType + store: IdentityStore + subject: string + token: string +}) { + const actor = await createActor(store, { subject, token, role: "administrator" }) + const workspace = (await service.createWorkspace(token, { + name: `${subject} workspace`, + })) as { id: string } + const prototype = (await service.createPrototype(token, workspace.id, { + name: `${subject} prototype`, + })) as { id: string } + return { actor, prototype, workspace } +} + +async function rejectAudit( + database: PrismaClient, + event: + | "prototype.publisher.create" + | "prototype.publisher.disable" + | "prototype.publisher.credential.revoke", + action: () => Promise, +) { + await database.$executeRawUnsafe(` + CREATE FUNCTION reject_test_publisher_audit() RETURNS trigger AS $$ + BEGIN + IF NEW.event = '${event}' THEN + RAISE EXCEPTION 'audit unavailable'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + `) + await database.$executeRawUnsafe(` + CREATE TRIGGER reject_test_publisher_audit + BEFORE INSERT ON security_audits + FOR EACH ROW EXECUTE FUNCTION reject_test_publisher_audit(); + `) + try { + await expect(action()).rejects.toThrow("audit unavailable") + } finally { + await database.$executeRawUnsafe("DROP TRIGGER reject_test_publisher_audit ON security_audits") + await database.$executeRawUnsafe("DROP FUNCTION reject_test_publisher_audit()") + } +} + describe("createPrototypeAuthorizationService() database behavior", () => { it("separates installation authority from prototype access and explicitly grants a creator", async () => { const { prisma, service, store } = await database() @@ -795,6 +847,236 @@ describe("createPrototypeAuthorizationService() database behavior", () => { }) }) + it("audits publisher disable, re-enable, and credential revocation with their state changes", async () => { + const { prisma, service, store } = await database() + const token = "publisher-lifecycle-token" + const { actor, prototype } = await createAdministeredPrototype({ + service, + store, + subject: "publisher-lifecycle", + token, + }) + + await rejectAudit(prisma, "prototype.publisher.create", () => + service.createPublisherCredential(token, prototype.id, { + name: "Rejected publisher", + actions: ["publish"], + }), + ) + expect(await prisma.workloadPublisher.count({ where: { prototypeId: prototype.id } })).toBe(0) + + const provisioned = (await service.createPublisherCredential(token, prototype.id, { + name: "CI publisher", + actions: ["publish", "read-status"], + })) as { + publisher: { id: string } + credential: { id: string } + } + + await rejectAudit(prisma, "prototype.publisher.disable", () => + service.setPublisherActive(token, prototype.id, provisioned.publisher.id, { + active: false, + }), + ) + expect( + await prisma.workloadPublisher.findUniqueOrThrow({ + where: { id: provisioned.publisher.id }, + select: { active: true }, + }), + ).toEqual({ active: true }) + + await service.setPublisherActive(token, prototype.id, provisioned.publisher.id, { + active: false, + }) + await service.setPublisherActive(token, prototype.id, provisioned.publisher.id, { + active: true, + }) + await rejectAudit(prisma, "prototype.publisher.credential.revoke", () => + service.revokePublisherCredential( + token, + prototype.id, + provisioned.publisher.id, + provisioned.credential.id, + ), + ) + expect( + await prisma.workloadCredential.findUniqueOrThrow({ + where: { id: provisioned.credential.id }, + select: { revokedAt: true }, + }), + ).toEqual({ revokedAt: null }) + + await service.revokePublisherCredential( + token, + prototype.id, + provisioned.publisher.id, + provisioned.credential.id, + ) + + const revokedCredential = await prisma.workloadCredential.findUniqueOrThrow({ + where: { id: provisioned.credential.id }, + select: { revokedAt: true }, + }) + expect(revokedCredential.revokedAt).toBeInstanceOf(Date) + expect( + await prisma.securityAudit.findMany({ + where: { prototypeId: prototype.id, workloadPublisherId: provisioned.publisher.id }, + select: { + actorUserId: true, + event: true, + workloadCredentialId: true, + workloadPublisherId: true, + }, + }), + ).toEqual( + expect.arrayContaining([ + { + actorUserId: actor.user.id, + event: "prototype.publisher.create", + workloadCredentialId: null, + workloadPublisherId: provisioned.publisher.id, + }, + { + actorUserId: actor.user.id, + event: "prototype.publisher.disable", + workloadCredentialId: null, + workloadPublisherId: provisioned.publisher.id, + }, + { + actorUserId: actor.user.id, + event: "prototype.publisher.enable", + workloadCredentialId: null, + workloadPublisherId: provisioned.publisher.id, + }, + { + actorUserId: actor.user.id, + event: "prototype.publisher.credential.revoke", + workloadCredentialId: provisioned.credential.id, + workloadPublisherId: provisioned.publisher.id, + }, + ]), + ) + expect( + await prisma.securityAudit.count({ + where: { prototypeId: prototype.id, workloadPublisherId: provisioned.publisher.id }, + }), + ).toBe(4) + }) + + it("does not apply publisher or credential lifecycle operations across prototypes", async () => { + const { prisma, service, store } = await database() + const token = "publisher-scope-token" + const { prototype, workspace } = await createAdministeredPrototype({ + service, + store, + subject: "publisher-scope", + token, + }) + const otherPrototype = (await service.createPrototype(token, workspace.id, { + name: "Other prototype", + })) as { id: string } + const provisioned = (await service.createPublisherCredential(token, prototype.id, { + name: "Scoped publisher", + actions: ["publish"], + })) as { + publisher: { id: string } + credential: { id: string } + } + + await expect( + service.setPublisherActive(token, otherPrototype.id, provisioned.publisher.id, { + active: false, + }), + ).rejects.toMatchObject({ status: 404, code: "workload_publisher_not_found" }) + await expect( + service.revokePublisherCredential( + token, + otherPrototype.id, + provisioned.publisher.id, + provisioned.credential.id, + ), + ).rejects.toMatchObject({ status: 404, code: "workload_credential_not_found" }) + + expect( + await prisma.workloadPublisher.findUniqueOrThrow({ + where: { id: provisioned.publisher.id }, + select: { active: true, credentials: { select: { revokedAt: true } } }, + }), + ).toEqual({ active: true, credentials: [{ revokedAt: null }] }) + expect( + await prisma.securityAudit.count({ + where: { + prototypeId: otherPrototype.id, + event: { startsWith: "prototype.publisher." }, + }, + }), + ).toBe(0) + }) + + it("allows disable and credential revocation after archive but blocks creation and re-enable", async () => { + const { prisma, service, store } = await database() + const token = "archived-publisher-token" + const { prototype } = await createAdministeredPrototype({ + service, + store, + subject: "archived-publisher", + token, + }) + const provisioned = (await service.createPublisherCredential(token, prototype.id, { + name: "Archived publisher", + actions: ["publish"], + })) as { + publisher: { id: string } + credential: { id: string } + } + await service.archivePrototype(token, prototype.id) + + await expect( + service.createPublisherCredential(token, prototype.id, { + name: "New archived publisher", + actions: ["publish"], + }), + ).rejects.toMatchObject({ status: 409, code: "prototype_archived" }) + await service.setPublisherActive(token, prototype.id, provisioned.publisher.id, { + active: false, + }) + await expect( + service.setPublisherActive(token, prototype.id, provisioned.publisher.id, { + active: true, + }), + ).rejects.toMatchObject({ status: 409, code: "prototype_archived" }) + await service.revokePublisherCredential( + token, + prototype.id, + provisioned.publisher.id, + provisioned.credential.id, + ) + + const publisher = await prisma.workloadPublisher.findUniqueOrThrow({ + where: { id: provisioned.publisher.id }, + select: { active: true, credentials: { select: { revokedAt: true } } }, + }) + expect(publisher.active).toBe(false) + expect(publisher.credentials).toHaveLength(1) + expect(publisher.credentials[0]?.revokedAt).toBeInstanceOf(Date) + expect( + ( + await prisma.securityAudit.findMany({ + where: { + prototypeId: prototype.id, + event: { + in: ["prototype.publisher.disable", "prototype.publisher.credential.revoke"], + }, + }, + select: { event: true }, + }) + ).sort((left, right) => left.event.localeCompare(right.event)), + ).toEqual([ + { event: "prototype.publisher.credential.revoke" }, + { event: "prototype.publisher.disable" }, + ]) + }) + it("rechecks membership in the mutation transaction instead of trusting a stale session", async () => { const { prisma, service, store } = await database() const actor = await createActor(store, { diff --git a/hub/server/domains/prototype-authorization/create-service/index.ts b/hub/server/domains/prototype-authorization/create-service/index.ts index 498af4d4..c517cf7d 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.ts @@ -33,6 +33,7 @@ const publisherSchema = z expiresAt: z.iso.datetime({ offset: true }).nullable().optional(), }) .strict() +const publisherStateSchema = z.object({ active: z.boolean() }).strict() const pageSchema = z.object({ cursor: z.uuid().optional(), limit: z.coerce.number().int().min(1).max(100).default(20), @@ -961,6 +962,13 @@ export function createPrototypeAuthorizationService({ baseline, ) requireDecision(decision, "administrator") + if (prototype.archivedAt) { + throw new AuthenticationError( + 409, + "prototype_archived", + "Archived prototypes cannot receive publishers.", + ) + } const credentialId = randomUUID() const token = `rpl_pub_${credentialId}.${randomBytes(32).toString("base64url")}` const publisher = await transaction.workloadPublisher.create({ @@ -998,6 +1006,118 @@ export function createPrototypeAuthorizationService({ }) }, + async setPublisherActive(sessionToken, prototypeId, publisherId, input) { + const parsed = publisherStateSchema.safeParse(input) + if (!parsed.success) { + throw new AuthenticationError( + 422, + "invalid_workload_publisher", + "The workload publisher is invalid.", + ) + } + return store.lock(async (_records, transaction) => { + const actor = await currentSession(store, sessionToken, transaction) + requireFreshAuthorization(actor) + requireIdentifier(prototypeId, "prototype_not_found", "The prototype was not found.") + requireIdentifier( + publisherId, + "workload_publisher_not_found", + "The workload publisher was not found.", + ) + await lockPrototype(transaction, prototypeId) + const { prototype, decision } = await prototypeDecision( + transaction, + actor.userId, + prototypeId, + baseline, + ) + requireDecision(decision, "administrator") + if (prototype.archivedAt && parsed.data.active) { + throw new AuthenticationError( + 409, + "prototype_archived", + "Archived prototypes cannot enable publishers.", + ) + } + const publisher = await transaction.workloadPublisher.findFirst({ + where: { id: publisherId, prototypeId }, + }) + if (!publisher) { + throw new AuthenticationError( + 404, + "workload_publisher_not_found", + "The workload publisher was not found.", + ) + } + if (publisher.active === parsed.data.active) return publisher + const updated = await transaction.workloadPublisher.update({ + where: { id: publisher.id }, + data: { active: parsed.data.active }, + }) + await audit(transaction, actor, { + event: parsed.data.active ? "prototype.publisher.enable" : "prototype.publisher.disable", + workspaceId: prototype.workspaceId, + prototypeId, + workloadPublisherId: publisher.id, + context: { source: "prototype_administrator" }, + }) + return updated + }) + }, + + async revokePublisherCredential(sessionToken, prototypeId, publisherId, credentialId) { + await store.lock(async (_records, transaction) => { + const actor = await currentSession(store, sessionToken, transaction) + requireFreshAuthorization(actor) + requireIdentifier(prototypeId, "prototype_not_found", "The prototype was not found.") + requireIdentifier( + publisherId, + "workload_publisher_not_found", + "The workload publisher was not found.", + ) + requireIdentifier( + credentialId, + "workload_credential_not_found", + "The workload credential was not found.", + ) + await lockPrototype(transaction, prototypeId) + const { prototype, decision } = await prototypeDecision( + transaction, + actor.userId, + prototypeId, + baseline, + ) + requireDecision(decision, "administrator") + const credential = await transaction.workloadCredential.findFirst({ + where: { + id: credentialId, + workloadPublisherId: publisherId, + workloadPublisher: { prototypeId }, + }, + }) + if (!credential) { + throw new AuthenticationError( + 404, + "workload_credential_not_found", + "The workload credential was not found.", + ) + } + if (credential.revokedAt) return + await transaction.workloadCredential.update({ + where: { id: credential.id }, + data: { revokedAt: new Date() }, + }) + await audit(transaction, actor, { + event: "prototype.publisher.credential.revoke", + workspaceId: prototype.workspaceId, + prototypeId, + workloadPublisherId: publisherId, + workloadCredentialId: credential.id, + context: { source: "prototype_administrator" }, + }) + }) + }, + async archivePrototype(sessionToken, prototypeId) { return store.lock(async (_records, transaction) => { const actor = await currentSession(store, sessionToken, transaction) diff --git a/hub/server/domains/prototype-authorization/create-service/types.ts b/hub/server/domains/prototype-authorization/create-service/types.ts index eec58cb6..21cfb109 100644 --- a/hub/server/domains/prototype-authorization/create-service/types.ts +++ b/hub/server/domains/prototype-authorization/create-service/types.ts @@ -29,6 +29,18 @@ export interface PrototypeAuthorizationService { prototypeId: string, input: unknown, ): Promise + setPublisherActive( + sessionToken: string, + prototypeId: string, + publisherId: string, + input: unknown, + ): Promise + revokePublisherCredential( + sessionToken: string, + prototypeId: string, + publisherId: string, + credentialId: string, + ): Promise archivePrototype(sessionToken: string, prototypeId: string): Promise setPrototypeAuthorization( sessionToken: string, diff --git a/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts b/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts index ba016379..e72e9f55 100644 --- a/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts +++ b/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto" -import { rm } from "node:fs/promises" +import { mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises" import path from "node:path" import { describe, expect, it, onTestFinished } from "vitest" @@ -16,6 +16,58 @@ function digest(bytes: Uint8Array) { return `sha256:${createHash("sha256").update(bytes).digest("hex")}` } +function artifactPath(root: string, contentDigest: string) { + const hash = contentDigest.slice("sha256:".length) + return { + algorithm: path.join(root, "sha256"), + shard: path.join(root, "sha256", hash.slice(0, 2)), + artifact: path.join(root, "sha256", hash.slice(0, 2), hash), + key: `sha256/${hash.slice(0, 2)}/${hash}`, + } +} + +function bytesInShard(shard: string) { + for (let index = 0; ; index += 1) { + const bytes = Buffer.from(`outside write probe ${index}`) + if (digest(bytes).slice("sha256:".length, "sha256:".length + 2) === shard) return bytes + } +} + +async function expectSymlinkRejected({ + root, + key, + contentDigest, + bytes, + sentinel, + writeProbe, +}: { + root: string + key: string + contentDigest: string + bytes: Uint8Array + sentinel: string + writeProbe?: { bytes: Uint8Array; outsidePath: string } +}) { + const storage = createLocalArtifactStorage(root) + + await expect(storage.read(key)).rejects.toThrow( + "Artifact storage paths cannot contain symbolic links.", + ) + await expect(storage.put(contentDigest, bytes)).rejects.toThrow( + "Artifact storage paths cannot contain symbolic links.", + ) + if (writeProbe) { + await expect(storage.put(digest(writeProbe.bytes), writeProbe.bytes)).rejects.toThrow( + "Artifact storage paths cannot contain symbolic links.", + ) + await expect(readFile(writeProbe.outsidePath)).rejects.toMatchObject({ code: "ENOENT" }) + } + await expect(storage.remove(key)).rejects.toThrow( + "Artifact storage paths cannot contain symbolic links.", + ) + expect(await readFile(sentinel)).toEqual(bytes) +} + describe("createLocalArtifactStorage()", () => { it("requires an absolute storage root", () => { expect(() => createLocalArtifactStorage("relative/artifacts")).toThrow( @@ -64,4 +116,89 @@ describe("createLocalArtifactStorage()", () => { expect(results.filter(({ created }) => created)).toHaveLength(1) expect(Buffer.from(await storage.read(results[0].key))).toEqual(bytes) }) + + it("rejects a symbolic-link storage root without reading, replaying, or removing outside content", async () => { + const root = storageRoot() + const outside = storageRoot() + const bytes = Buffer.from("outside root sentinel") + const contentDigest = digest(bytes) + const location = artifactPath(outside, contentDigest) + await mkdir(location.shard, { recursive: true }) + await writeFile(location.artifact, bytes) + await symlink(outside, root, "dir") + const writeBytes = Buffer.from("outside write probe") + + await expectSymlinkRejected({ + root, + key: location.key, + contentDigest, + bytes, + sentinel: location.artifact, + writeProbe: { + bytes: writeBytes, + outsidePath: artifactPath(outside, digest(writeBytes)).artifact, + }, + }) + }) + + it.each(["sha256", "shard"] as const)( + "rejects a symbolic-link %s directory without reading, replaying, or removing outside content", + async (level) => { + const root = storageRoot() + const outside = storageRoot() + const bytes = Buffer.from(`outside ${level} sentinel`) + const contentDigest = digest(bytes) + const inside = artifactPath(root, contentDigest) + const outsideLocation = artifactPath(outside, contentDigest) + const writeBytes = + level === "shard" + ? bytesInShard(contentDigest.slice("sha256:".length, "sha256:".length + 2)) + : Buffer.from("outside write probe") + + if (level === "sha256") { + await mkdir(outsideLocation.shard, { recursive: true }) + await writeFile(outsideLocation.artifact, bytes) + await mkdir(root, { recursive: true }) + await symlink(outsideLocation.algorithm, inside.algorithm, "dir") + } else { + await mkdir(inside.algorithm, { recursive: true }) + await mkdir(outsideLocation.shard, { recursive: true }) + await writeFile(outsideLocation.artifact, bytes) + await symlink(outsideLocation.shard, inside.shard, "dir") + } + + await expectSymlinkRejected({ + root, + key: inside.key, + contentDigest, + bytes, + sentinel: outsideLocation.artifact, + writeProbe: { + bytes: writeBytes, + outsidePath: artifactPath(outside, digest(writeBytes)).artifact, + }, + }) + }, + ) + + it("rejects a symbolic-link artifact without reading, replaying, or removing its outside target", async () => { + const root = storageRoot() + const outside = storageRoot() + const bytes = Buffer.from("outside artifact sentinel") + const contentDigest = digest(bytes) + const location = artifactPath(root, contentDigest) + const sentinel = path.join(outside, "sentinel") + await mkdir(location.shard, { recursive: true }) + await mkdir(outside, { recursive: true }) + await writeFile(sentinel, bytes) + await symlink(sentinel, location.artifact) + + await expectSymlinkRejected({ + root, + key: location.key, + contentDigest, + bytes, + sentinel, + }) + }) }) diff --git a/hub/server/integrations/artifact-storage/create-local-storage/index.ts b/hub/server/integrations/artifact-storage/create-local-storage/index.ts index 544a2fee..d6fa5eb7 100644 --- a/hub/server/integrations/artifact-storage/create-local-storage/index.ts +++ b/hub/server/integrations/artifact-storage/create-local-storage/index.ts @@ -1,48 +1,133 @@ import { createHash, randomUUID, timingSafeEqual } from "node:crypto" import { constants } from "node:fs" -import { access, link, mkdir, open, readFile, rm } from "node:fs/promises" +import { link, lstat, mkdir, open, realpath, rm, unlink } from "node:fs/promises" import path from "node:path" import type { ArtifactStorage, StoredArtifact } from "#server/integrations/artifact-storage/types" +function isFileSystemError(error: unknown, code: string): boolean { + return ( + error instanceof Error && + "code" in error && + typeof error.code === "string" && + error.code === code + ) +} + +function rejectSymbolicLink(): never { + throw new Error("Artifact storage paths cannot contain symbolic links.") +} + function artifactKey(contentDigest: string): string { if (!/^sha256:[a-f0-9]{64}$/.test(contentDigest)) { throw new Error("Artifact content digest is invalid.") } - const digest = contentDigest.replace(/^sha256:/, "") + const digest = contentDigest.slice("sha256:".length) return path.posix.join("sha256", digest.slice(0, 2), digest) } -function storagePath(root: string, key: string): string { - const candidate = path.resolve(root, key) - const resolvedRoot = path.resolve(root) - if (!candidate.startsWith(`${resolvedRoot}${path.sep}`)) { +function parseArtifactKey(key: string) { + const match = /^sha256\/([a-f0-9]{2})\/([a-f0-9]{64})$/.exec(key) + if (!match?.[1] || !match[2] || match[2].slice(0, 2) !== match[1]) { throw new Error("Artifact storage key is invalid.") } - return candidate + return { shard: match[1], digest: match[2] } } function calculateContentDigest(bytes: Uint8Array): string { return `sha256:${createHash("sha256").update(bytes).digest("hex")}` } -async function verifyExisting(destination: string, contentDigest: string): Promise { - const expected = Buffer.from(contentDigest.replace(/^sha256:/, ""), "hex") - const actual = createHash("sha256") - .update(await readFile(destination)) - .digest() - if (!timingSafeEqual(expected, actual)) { - throw new Error("Stored artifact content does not match its key.") +function isWithinRoot(root: string, candidate: string): boolean { + return candidate === root || candidate.startsWith(`${root}${path.sep}`) +} + +async function secureRoot(root: string): Promise { + await mkdir(root, { recursive: true, mode: 0o700 }) + const status = await lstat(root) + if (status.isSymbolicLink()) rejectSymbolicLink() + if (!status.isDirectory()) throw new Error("Artifact storage root must be a directory.") + return realpath(root) +} + +async function secureDirectory( + root: string, + parent: string, + name: string, + create: boolean, +): Promise { + const candidate = path.join(parent, name) + if (create) { + try { + await mkdir(candidate, { mode: 0o700 }) + } catch (error) { + if (!isFileSystemError(error, "EEXIST")) throw error + } } + const status = await lstat(candidate) + if (status.isSymbolicLink()) rejectSymbolicLink() + if (!status.isDirectory()) throw new Error("Artifact storage path is not a directory.") + const resolved = await realpath(candidate) + if (!isWithinRoot(root, resolved)) { + throw new Error("Artifact storage path escapes its root.") + } + return resolved } -function isFileSystemError(error: unknown, code: string): boolean { - return ( - error instanceof Error && - "code" in error && - typeof error.code === "string" && - error.code === code +async function resolveArtifactPath(root: string, key: string, createDirectories: boolean) { + const { shard, digest } = parseArtifactKey(key) + const resolvedRoot = await secureRoot(root) + const algorithmDirectory = await secureDirectory( + resolvedRoot, + resolvedRoot, + "sha256", + createDirectories, + ) + const shardDirectory = await secureDirectory( + resolvedRoot, + algorithmDirectory, + shard, + createDirectories, ) + const destination = path.join(shardDirectory, digest) + if (!isWithinRoot(resolvedRoot, destination)) { + throw new Error("Artifact storage path escapes its root.") + } + return { destination, resolvedRoot } +} + +async function validateArtifactPath(resolvedRoot: string, destination: string) { + const status = await lstat(destination) + if (status.isSymbolicLink()) rejectSymbolicLink() + if (!status.isFile()) throw new Error("Artifact storage path is not a file.") + const resolved = await realpath(destination) + if (!isWithinRoot(resolvedRoot, resolved)) { + throw new Error("Artifact storage path escapes its root.") + } +} + +async function openArtifact(resolvedRoot: string, destination: string) { + await validateArtifactPath(resolvedRoot, destination) + return open(destination, constants.O_RDONLY | constants.O_NOFOLLOW) +} + +async function verifyExisting( + resolvedRoot: string, + destination: string, + contentDigest: string, +): Promise { + const file = await openArtifact(resolvedRoot, destination) + try { + const expected = Buffer.from(contentDigest.slice("sha256:".length), "hex") + const actual = createHash("sha256") + .update(await file.readFile()) + .digest() + if (!timingSafeEqual(expected, actual)) { + throw new Error("Stored artifact content does not match its key.") + } + } finally { + await file.close() + } } export function createLocalArtifactStorage(root: string): ArtifactStorage { @@ -55,18 +140,20 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { if (calculateContentDigest(bytes) !== contentDigest) { throw new Error("Artifact bytes do not match the content digest.") } - const destination = storagePath(root, key) - await mkdir(path.dirname(destination), { recursive: true }) + const { destination, resolvedRoot } = await resolveArtifactPath(root, key, true) try { - await access(destination, constants.F_OK) - await verifyExisting(destination, contentDigest) + await verifyExisting(resolvedRoot, destination, contentDigest) return { key, created: false } } catch (error) { if (!isFileSystemError(error, "ENOENT")) throw error } const temporary = `${destination}.${randomUUID()}.tmp` - const file = await open(temporary, "wx", 0o600) + const file = await open( + temporary, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + 0o600, + ) try { await file.writeFile(bytes) await file.sync() @@ -74,11 +161,15 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { await file.close() } try { + const resolved = await resolveArtifactPath(root, key, false) + if (resolved.destination !== destination) { + throw new Error("Artifact storage path changed during the write.") + } await link(temporary, destination) return { key, created: true } } catch (error) { if (!isFileSystemError(error, "EEXIST")) throw error - await verifyExisting(destination, contentDigest) + await verifyExisting(resolvedRoot, destination, contentDigest) return { key, created: false } } finally { await rm(temporary, { force: true }) @@ -86,11 +177,23 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { }, async read(key: string): Promise { - return readFile(storagePath(root, key)) + const { destination, resolvedRoot } = await resolveArtifactPath(root, key, false) + const file = await openArtifact(resolvedRoot, destination) + try { + return file.readFile() + } finally { + await file.close() + } }, async remove(key: string): Promise { - await rm(storagePath(root, key), { force: true }) + try { + const { destination, resolvedRoot } = await resolveArtifactPath(root, key, false) + await validateArtifactPath(resolvedRoot, destination) + await unlink(destination) + } catch (error) { + if (!isFileSystemError(error, "ENOENT")) throw error + } }, } } diff --git a/hub/server/repositories/security-audits/save-audit/index.ts b/hub/server/repositories/security-audits/save-audit/index.ts index d139afde..4ed58250 100644 --- a/hub/server/repositories/security-audits/save-audit/index.ts +++ b/hub/server/repositories/security-audits/save-audit/index.ts @@ -28,6 +28,7 @@ export async function saveSecurityAudit(database: Prisma.TransactionClient, data prototypeId: data.prototypeId ?? null, prototypeVersionId: data.prototypeVersionId ?? null, workloadPublisherId: data.workloadPublisherId ?? null, + workloadCredentialId: data.workloadCredentialId ?? null, targetUserId: data.targetUserId ?? null, githubRepositorySourceId: data.githubRepositorySourceId ?? null, context, diff --git a/hub/server/repositories/security-audits/types.ts b/hub/server/repositories/security-audits/types.ts index 05be2a06..55bbcb22 100644 --- a/hub/server/repositories/security-audits/types.ts +++ b/hub/server/repositories/security-audits/types.ts @@ -10,6 +10,7 @@ export interface AuditInput { prototypeId?: string | null prototypeVersionId?: string | null workloadPublisherId?: string | null + workloadCredentialId?: string | null targetUserId?: string | null githubRepositorySourceId?: string | null context?: Record @@ -28,6 +29,7 @@ export interface SecurityAuditRecord { prototypeId: string | null prototypeVersionId: string | null workloadPublisherId: string | null + workloadCredentialId: string | null targetUserId: string | null githubRepositorySourceId: string | null context: unknown diff --git a/hub/server/routes/index.ts b/hub/server/routes/index.ts index c648ece9..8fbe4ce9 100644 --- a/hub/server/routes/index.ts +++ b/hub/server/routes/index.ts @@ -30,6 +30,8 @@ import { listWorkspaces, removePrototypeAccess, removeWorkspaceAccess, + revokePublisherCredential, + setPublisherActive, setPrototypeAccess, setPrototypeAuthorization, setWorkspaceAccess, @@ -159,6 +161,16 @@ export function routes(app: Express, authentication?: AuthenticationRoutes): voi productMutationLimit, createPublisherCredential.bind(null, authentication), ) + app.patch( + "/api/prototypes/:prototypeId/publishers/:publisherId", + productMutationLimit, + setPublisherActive.bind(null, authentication), + ) + app.delete( + "/api/prototypes/:prototypeId/publishers/:publisherId/credentials/:credentialId", + productMutationLimit, + revokePublisherCredential.bind(null, authentication), + ) app.get( "/api/prototypes/:prototypeId/access/explanation", explainPrototypeAccess.bind(null, authentication), diff --git a/hub/server/routes/product/index.test.ts b/hub/server/routes/product/index.test.ts index ef393101..546668f3 100644 --- a/hub/server/routes/product/index.test.ts +++ b/hub/server/routes/product/index.test.ts @@ -40,6 +40,8 @@ function runtime( archivePrototype: vi.fn(), setPrototypeAuthorization: vi.fn(), createPublisherCredential: vi.fn(), + setPublisherActive: vi.fn(), + revokePublisherCredential: vi.fn(), explainPrototypeAccess: vi.fn(), setPrototypeAccess: vi.fn(), removePrototypeAccess: vi.fn(), @@ -234,6 +236,17 @@ describe("product routes", () => { call: "createPublisherCredential", body: { name: "Local publisher", actions: ["publish"] }, }, + { + method: "PATCH", + path: "/api/prototypes/prototype/publishers/publisher", + call: "setPublisherActive", + body: { active: false }, + }, + { + method: "DELETE", + path: "/api/prototypes/prototype/publishers/publisher/credentials/credential", + call: "revokePublisherCredential", + }, { method: "POST", path: "/api/prototypes/prototype/archive", diff --git a/hub/server/routes/product/index.ts b/hub/server/routes/product/index.ts index bebe917d..2126c361 100644 --- a/hub/server/routes/product/index.ts +++ b/hub/server/routes/product/index.ts @@ -165,6 +165,34 @@ export async function createPublisherCredential( response.status(201).json(result) } +export async function setPublisherActive( + authentication: AuthenticationRoutes, + request: Request, + response: Response, +): Promise { + const publisher = await authentication.prototypeAuthorization.setPublisherActive( + mutationSessionToken(authentication, request), + parameter(request, "prototypeId"), + parameter(request, "publisherId"), + request.body, + ) + response.json({ publisher }) +} + +export async function revokePublisherCredential( + authentication: AuthenticationRoutes, + request: Request, + response: Response, +): Promise { + await authentication.prototypeAuthorization.revokePublisherCredential( + mutationSessionToken(authentication, request), + parameter(request, "prototypeId"), + parameter(request, "publisherId"), + parameter(request, "credentialId"), + ) + response.status(204).end() +} + export async function explainPrototypeAccess( authentication: AuthenticationRoutes, request: Request, From 099781fd840c72e1f04880e7edb2179fe0a54ea3 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Tue, 27 Oct 2026 16:54:48 -0700 Subject: [PATCH 03/13] fix(hub): recheck publisher authorization under lock Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- hub/docs/artifact-ingestion.md | 2 + .../create-service/index.db.test.ts | 159 ++++++++++++++++++ .../create-service/index.ts | 58 ++++++- 3 files changed, 214 insertions(+), 5 deletions(-) diff --git a/hub/docs/artifact-ingestion.md b/hub/docs/artifact-ingestion.md index 8f79582c..13250fad 100644 --- a/hub/docs/artifact-ingestion.md +++ b/hub/docs/artifact-ingestion.md @@ -15,6 +15,8 @@ Hub returns a bearer credential once and stores only its SHA-256 digest. The cre Browser users provision credentials through `POST /api/prototypes/:prototypeId/publishers`. This requires a fresh prototype-administrator session and returns the raw token only in that response. The same administrator may temporarily disable or re-enable a publisher with `PATCH /api/prototypes/:prototypeId/publishers/:publisherId`, or permanently revoke one credential with `DELETE /api/prototypes/:prototypeId/publishers/:publisherId/credentials/:credentialId`. Hub audits each state change. Re-enabling a publisher does not restore a revoked credential. +Publication, promotion, publisher state changes, and credential revocation use the same prototype lock. Publication and promotion re-read credential expiry, revocation, scope, action, token digest, and publisher state after acquiring that lock. A revocation or disable that locks first prevents an already-started workload request from committing; a workload operation that locks first completes before the administrator mutation. + Archiving a prototype blocks new publishers, publication, publisher re-enablement, and `current` or `preview` promotion. Administrators may still disable publishers and revoke credentials after archival. Existing versions and publication status remain readable through their authorized interfaces. ## Manifest contract diff --git a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts index a25e1418..f725e003 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts @@ -79,6 +79,56 @@ async function database() { } } +function deferred() { + let resolvePromise!: () => void + const promise = new Promise((resolve) => { + resolvePromise = resolve + }) + return { promise, resolve: resolvePromise } +} + +async function waitForBlockedDatabaseQuery(database: PrismaClient, queryPattern: string) { + await vi.waitFor( + async () => { + const [result] = await database.$queryRaw<{ blocked: boolean }[]>` + SELECT EXISTS ( + SELECT 1 + FROM pg_stat_activity + WHERE datname = current_database() + AND pid <> pg_backend_pid() + AND wait_event_type = 'Lock' + AND query LIKE ${queryPattern} + ) AS blocked + ` + expect(result?.blocked).toBe(true) + }, + { interval: 20, timeout: 5_000 }, + ) +} + +async function holdPublisherAuthorizationRow( + database: PrismaClient, + target: { type: "credential" | "publisher"; id: string }, +) { + const released = deferred() + const locked = deferred() + const holder = database.$transaction(async (transaction) => { + if (target.type === "credential") { + await transaction.$queryRaw` + SELECT "id" FROM "workload_credentials" WHERE "id" = ${target.id}::uuid FOR UPDATE + ` + } else { + await transaction.$queryRaw` + SELECT "id" FROM "workload_publishers" WHERE "id" = ${target.id}::uuid FOR UPDATE + ` + } + locked.resolve() + await released.promise + }) + await locked.promise + return { holder, release: released.resolve } +} + async function createPrototype(prisma: PrismaClient) { const user = await prisma.user.create({ data: { displayName: "Publisher owner" } }) const workspace = await prisma.workspace.create({ @@ -490,6 +540,115 @@ describe("createArtifactPublishingService() database behavior", () => { ).toBe(2) }) + it.each([ + { mutation: "revoke", operation: "publish" }, + { mutation: "disable", operation: "publish" }, + { mutation: "revoke", operation: "promote" }, + { mutation: "disable", operation: "promote" }, + ] as const)( + "rejects an already-started $operation after an administrator wins the $mutation race", + async ({ mutation, operation }) => { + const context = await database() + const sessionToken = `${mutation}-${operation}-session` + const { authorization, prototype, provisioned, workloadAuthorization } = + await provisionPublisher(context.prisma, sessionToken) + const published = + operation === "promote" + ? await context.service.publish( + workloadAuthorization, + prototype.id, + `${mutation}-promotion-source`, + publicationInput(prototype.id, artifact(`${mutation}-promotion-source`)), + ) + : null + const version = published ? publishedVersion(published) : null + const authorizationRow = + mutation === "revoke" + ? { type: "credential" as const, id: provisioned.credential.id } + : { type: "publisher" as const, id: provisioned.publisher.id } + const authorizationTable = + mutation === "revoke" ? "workload_credentials" : "workload_publishers" + const before = { + publications: await context.prisma.artifactPublication.count({ + where: { prototypeId: prototype.id }, + }), + versions: await context.prisma.prototypeVersion.count({ + where: { prototypeId: prototype.id }, + }), + channels: await context.prisma.prototypeChannel.findMany({ + where: { prototypeId: prototype.id }, + select: { name: true, prototypeVersionId: true }, + orderBy: { name: "asc" }, + }), + workloadAudits: await context.prisma.securityAudit.count({ + where: { + prototypeId: prototype.id, + event: { in: ["prototype.version.publish", "prototype.channel.promote"] }, + }, + }), + } + const rowLock = await holdPublisherAuthorizationRow(context.prisma, authorizationRow) + + const administratorMutation = + mutation === "revoke" + ? authorization.revokePublisherCredential( + sessionToken, + prototype.id, + provisioned.publisher.id, + provisioned.credential.id, + ) + : authorization.setPublisherActive(sessionToken, prototype.id, provisioned.publisher.id, { + active: false, + }) + await waitForBlockedDatabaseQuery(context.prisma, `%${authorizationTable}%`) + + const workloadResult = ( + operation === "publish" + ? context.service.publish( + workloadAuthorization, + prototype.id, + `${mutation}-concurrent-publish`, + publicationInput(prototype.id, artifact(`${mutation}-concurrent-publish`)), + ) + : context.service.promote(workloadAuthorization, prototype.id, "current", { + versionId: version!.id, + }) + ).then( + () => ({ error: null }), + (error: unknown) => ({ error }), + ) + await waitForBlockedDatabaseQuery(context.prisma, "%prototypes%FOR UPDATE%") + + rowLock.release() + await rowLock.holder + await administratorMutation + + expect((await workloadResult).error).toMatchObject({ + status: 403, + code: "publisher_access_denied", + }) + expect({ + publications: await context.prisma.artifactPublication.count({ + where: { prototypeId: prototype.id }, + }), + versions: await context.prisma.prototypeVersion.count({ + where: { prototypeId: prototype.id }, + }), + channels: await context.prisma.prototypeChannel.findMany({ + where: { prototypeId: prototype.id }, + select: { name: true, prototypeVersionId: true }, + orderBy: { name: "asc" }, + }), + workloadAudits: await context.prisma.securityAudit.count({ + where: { + prototypeId: prototype.id, + event: { in: ["prototype.version.publish", "prototype.channel.promote"] }, + }, + }), + }).toEqual(before) + }, + ) + it("enforces publish and read-status action scopes independently", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) diff --git a/hub/server/domains/artifact-publishing/create-service/index.ts b/hub/server/domains/artifact-publishing/create-service/index.ts index 39ad72d0..5b48569b 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.ts @@ -155,6 +155,12 @@ export function createArtifactPublishingService({ database: PrismaClient storage: ArtifactStorage }): ArtifactPublishingService { + type PublisherAuthentication = { + credentialId: string + publisherId: string + tokenDigest: string + } + async function requireUnarchivedPrototype(prototypeId: string) { const prototype = await database.prototype.findUnique({ where: { id: prototypeId }, @@ -173,6 +179,7 @@ export function createArtifactPublishingService({ error(401, "publisher_authentication_required", "Publisher authentication failed.") } const [, token, credentialId] = match + const tokenDigest = digestSecret(token) const credential = await database.workloadCredential.findUnique({ where: { id: credentialId }, include: { workloadPublisher: true }, @@ -184,7 +191,36 @@ export function createArtifactPublishingService({ !credential.workloadPublisher.active || credential.workloadPublisher.prototypeId !== prototypeId || !credential.workloadPublisher.actions.includes(action) || - !secretsMatch(credential.tokenDigest, digestSecret(token)) + !secretsMatch(credential.tokenDigest, tokenDigest) + ) { + error(403, "publisher_access_denied", "Publisher access is denied.") + } + return { + credentialId: credential.id, + publisherId: credential.workloadPublisher.id, + tokenDigest, + } + } + + async function requireCurrentPublisherAuthorization( + transaction: Prisma.TransactionClient, + authentication: PublisherAuthentication, + prototypeId: string, + action: PublisherAction, + ) { + const credential = await transaction.workloadCredential.findUnique({ + where: { id: authentication.credentialId }, + include: { workloadPublisher: true }, + }) + if ( + !credential || + credential.revokedAt || + (credential.expiresAt && credential.expiresAt <= new Date()) || + !credential.workloadPublisher.active || + credential.workloadPublisher.id !== authentication.publisherId || + credential.workloadPublisher.prototypeId !== prototypeId || + !credential.workloadPublisher.actions.includes(action) || + !secretsMatch(credential.tokenDigest, authentication.tokenDigest) ) { error(403, "publisher_access_denied", "Publisher access is denied.") } @@ -193,7 +229,7 @@ export function createArtifactPublishingService({ return { async publish(authorization, prototypeId, idempotencyKey, input) { - const publisher = await authenticate(authorization, prototypeId, "publish") + const authentication = await authenticate(authorization, prototypeId, "publish") await requireUnarchivedPrototype(prototypeId) const parsed = publishSchema.safeParse(input) if (!parsed.success || parsed.data.manifest.prototypeId !== prototypeId) { @@ -217,6 +253,12 @@ export function createArtifactPublishingService({ return database.$transaction(async (transaction) => { await lockPrototype(transaction, prototypeId) + const publisher = await requireCurrentPublisherAuthorization( + transaction, + authentication, + prototypeId, + "publish", + ) const prototype = await transaction.prototype.findUnique({ where: { id: prototypeId } }) if (!prototype) error(404, "prototype_not_found", "The prototype was not found.") if (prototype.archivedAt) { @@ -325,11 +367,11 @@ export function createArtifactPublishingService({ }, async getStatus(authorization, prototypeId, idempotencyKey) { - const publisher = await authenticate(authorization, prototypeId, "read-status") + const authentication = await authenticate(authorization, prototypeId, "read-status") const publication = await database.artifactPublication.findUnique({ where: { workloadPublisherId_idempotencyKeyDigest: { - workloadPublisherId: publisher.id, + workloadPublisherId: authentication.publisherId, idempotencyKeyDigest: digestSecret(parseIdempotencyKey(idempotencyKey)), }, }, @@ -345,7 +387,7 @@ export function createArtifactPublishingService({ }, async promote(authorization, prototypeId, channel, input) { - const publisher = await authenticate(authorization, prototypeId, "publish") + const authentication = await authenticate(authorization, prototypeId, "publish") const parsedChannel = channelSchema.safeParse(channel) const parsed = promotionSchema.safeParse(input) if (!parsedChannel.success || !parsed.success) { @@ -353,6 +395,12 @@ export function createArtifactPublishingService({ } return database.$transaction(async (transaction) => { await lockPrototype(transaction, prototypeId) + const publisher = await requireCurrentPublisherAuthorization( + transaction, + authentication, + prototypeId, + "publish", + ) const prototype = await transaction.prototype.findUnique({ where: { id: prototypeId } }) if (!prototype) error(404, "prototype_not_found", "The prototype was not found.") if (prototype.archivedAt) { From d4727c37161df02bcfbc6984f2688b21fee508c9 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Wed, 28 Oct 2026 14:01:45 -0700 Subject: [PATCH 04/13] fix(hub): address artifact ingestion review Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- hub/.env.example | 3 +- hub/docs/artifact-ingestion.md | 12 +- hub/docs/logging.md | 2 +- hub/docs/prototype-authorization.md | 1 + hub/server/create-app/index.ts | 16 +- .../create-service/index.db.test.ts | 188 ++++++++++++++++-- .../create-service/index.ts | 73 ++++--- .../create-runtime/index.test.ts | 36 ++++ .../authentication/create-runtime/index.ts | 3 +- .../create-service/index.db.test.ts | 66 ++++++ .../create-service/index.ts | 43 +++- .../create-service/types.ts | 1 + .../create-local-storage/index.test.ts | 60 +++++- .../create-local-storage/index.ts | 34 ++-- hub/server/routes/index.ts | 10 +- hub/server/routes/product/index.test.ts | 28 +++ hub/server/routes/product/index.ts | 13 ++ hub/server/routes/publishing.test.ts | 70 +++++++ 18 files changed, 571 insertions(+), 88 deletions(-) create mode 100644 hub/server/domains/authentication/create-runtime/index.test.ts diff --git a/hub/.env.example b/hub/.env.example index b5e26553..7e5afe18 100644 --- a/hub/.env.example +++ b/hub/.env.example @@ -20,7 +20,8 @@ DATABASE_URL=postgresql://replica:replica@127.0.0.1:5432/replica # Only database-enabled tests require this URL and CREATEDB permission. TEST_DATABASE_URL=postgresql://replica:replica@127.0.0.1:5432/postgres -# Local reference artifact storage. Production deployments should mount durable storage here. +# Local reference artifact storage. The default is ~/.replica/artifacts, outside the checkout. +# Production deployments should mount durable storage here. # HUB_ARTIFACT_STORAGE_PATH=/var/lib/replica/artifacts # GitHub user authorization is optional until both values are configured. diff --git a/hub/docs/artifact-ingestion.md b/hub/docs/artifact-ingestion.md index 13250fad..03a663b1 100644 --- a/hub/docs/artifact-ingestion.md +++ b/hub/docs/artifact-ingestion.md @@ -4,16 +4,16 @@ Hub accepts one provider-neutral artifact contract from workload producers. GitH ## Identity and access -A prototype administrator creates a workload publisher with one or both actions: +A prototype administrator creates a workload publisher with `publish` and may also grant `read-status`: | Action | Authority | | ------------- | ---------------------------------------------------------------------------- | | `publish` | Publish an artifact and move the prototype's `current` or `preview` channel. | -| `read-status` | Read the result associated with that publisher's idempotency key. | +| `read-status` | Read the result associated with that publisher's idempotency key. This action supplements `publish`; a status-only publisher would have no publication to read. | Hub returns a bearer credential once and stores only its SHA-256 digest. The credential is scoped to one prototype through its workload publisher. It cannot create a browser session, view a prototype through user APIs, administer access, or publish to another prototype. A producer's source URL, revision, ref, pipeline, and run ID describe where a build came from; they never grant access. -Browser users provision credentials through `POST /api/prototypes/:prototypeId/publishers`. This requires a fresh prototype-administrator session and returns the raw token only in that response. The same administrator may temporarily disable or re-enable a publisher with `PATCH /api/prototypes/:prototypeId/publishers/:publisherId`, or permanently revoke one credential with `DELETE /api/prototypes/:prototypeId/publishers/:publisherId/credentials/:credentialId`. Hub audits each state change. Re-enabling a publisher does not restore a revoked credential. +Browser users provision credentials through `POST /api/prototypes/:prototypeId/publishers`. This requires a fresh prototype-administrator session and returns the raw token only in that response. `GET /api/prototypes/:prototypeId/publishers` returns administrator-only publisher and credential metadata without token digests or raw tokens, so another administrator can discover and revoke existing access. An administrator may temporarily disable or re-enable a publisher with `PATCH /api/prototypes/:prototypeId/publishers/:publisherId`, or permanently revoke one credential with `DELETE /api/prototypes/:prototypeId/publishers/:publisherId/credentials/:credentialId`. Hub audits each state change. Re-enabling a publisher does not restore a revoked credential. Publication, promotion, publisher state changes, and credential revocation use the same prototype lock. Publication and promotion re-read credential expiry, revocation, scope, action, token digest, and publisher state after acquiring that lock. A revocation or disable that locks first prevents an already-started workload request from committing; a workload operation that locks first completes before the administrator mutation. @@ -49,11 +49,13 @@ Archiving a prototype blocks new publishers, publication, publisher re-enablemen } ``` -The manifest is closed to unknown fields. Hub bounds every string, accepts only HTTP or HTTPS source URLs without embedded usernames or passwords, limits decoded artifacts to 8 MiB in this reference layer, verifies the decoded size, and recalculates the SHA-256 digest before storing bytes. The manifest prototype ID must match the route. +The manifest and its nested objects are closed to unknown fields. Hub bounds every string, accepts only HTTP or HTTPS source URLs without embedded usernames, passwords, query strings, or fragments, limits decoded artifacts to 8 MiB in this reference layer, verifies canonical base64, checks the decoded size, and recalculates the SHA-256 digest before storing bytes. The manifest prototype ID must match the route. + +Publishing requests are rate-limited before JSON parsing. The version-upload route alone accepts the bounded upload-sized JSON body; status and promotion retain the API's default JSON limit. ## Storage boundary -`ArtifactStorage` separates ingestion from byte storage. The reference adapter writes content-addressed files under `HUB_ARTIFACT_STORAGE_PATH`, defaulting to `hub/.data/artifacts` for local use. Database rows store only the storage key and validated metadata, never the artifact bytes. +`ArtifactStorage` separates ingestion from byte storage. The reference adapter writes content-addressed files under `HUB_ARTIFACT_STORAGE_PATH`, defaulting to `~/.replica/artifacts` outside the source checkout and Vite development root. Database rows store only the storage key and validated metadata, never the artifact bytes. The local adapter is suitable for one Hub process with a durable mounted filesystem. It rejects symbolic-link roots, shards, and artifact paths, resolves directories under the configured root, and opens artifact files without following a final symbolic link. Operators must keep the storage directory writable only by the Hub workload; the local adapter does not provide replication, remote durability, encryption-key management, hostile concurrent filesystem isolation, or multi-region consistency. A production object-store adapter must preserve the same content-addressed, immutable `put`, `read`, and `remove` boundary. diff --git a/hub/docs/logging.md b/hub/docs/logging.md index e30872e6..78b39aea 100644 --- a/hub/docs/logging.md +++ b/hub/docs/logging.md @@ -30,7 +30,7 @@ Each request produces one final record with `reqId`, `method`, `route`, `statusC Automatic records exclude raw URLs, headers, cookies, uploads, response bodies, and error details. JSON parsing uses Express's default 100 KiB limit; rejected bodies are not logged. The Express error handler returns generic JSON errors, or closes the connection if headers were sent, without exposing raw errors. -`createApp()` installs `omitRequestLogPayload` for authentication, session, administration, product, and publishing paths after request logging but before JSON parsing. This ordering covers parser failures as well as later origin checks, rate limits, and handlers: +`createApp()` installs `omitRequestLogPayload` for authentication, session, administration, product, and publishing paths after request logging but before JSON parsing. Publishing rate limits also run before parsing, and only the artifact-upload route receives the larger bounded parser. This ordering covers parser failures as well as later origin checks and handlers: ```ts import { omitRequestLogPayload } from "../integrations/pino/omit-request-log-payload/index.ts" diff --git a/hub/docs/prototype-authorization.md b/hub/docs/prototype-authorization.md index 56fabb6b..31dec895 100644 --- a/hub/docs/prototype-authorization.md +++ b/hub/docs/prototype-authorization.md @@ -77,6 +77,7 @@ The product API uses: - `GET /api/prototypes/:prototypeId/versions` - `GET /api/prototypes/:prototypeId/versions/:versionId` - `POST /api/prototypes/:prototypeId/publishers` +- `GET /api/prototypes/:prototypeId/publishers` - `PATCH /api/prototypes/:prototypeId/publishers/:publisherId` - `DELETE /api/prototypes/:prototypeId/publishers/:publisherId/credentials/:credentialId` diff --git a/hub/server/create-app/index.ts b/hub/server/create-app/index.ts index 947ade16..f58b96b0 100644 --- a/hub/server/create-app/index.ts +++ b/hub/server/create-app/index.ts @@ -1,13 +1,14 @@ import express from "express" import type { Logger } from "pino" +import { maximumPublishingJsonBytes } from "../domains/artifact-publishing/create-service/constants.ts" +import { createRateLimit } from "../integrations/express/create-rate-limit/index.ts" import { handleError } from "../integrations/express/handle-error/index.ts" import { respondNotFound } from "../integrations/express/respond-not-found/index.ts" import { createRequestLogger } from "../integrations/pino/create-request-logger/index.ts" import { omitRequestLogPayload } from "../integrations/pino/omit-request-log-payload/index.ts" import { routes } from "../routes/index.ts" import type { AuthenticationRoutes } from "../routes/types.ts" -import { maximumPublishingJsonBytes } from "../domains/artifact-publishing/create-service/constants.ts" /** Give every endpoint the same request logging and safe error responses. */ export function createApp({ @@ -31,7 +32,18 @@ export function createApp({ omitRequestLogPayload, ) app.use("/api/github/webhooks", express.raw({ type: "application/json", limit: "1mb" })) - app.use("/api/publishing", express.json({ limit: maximumPublishingJsonBytes })) + app.use( + "/api/publishing", + createRateLimit({ + limit: 60, + windowMilliseconds: 5 * 60_000, + message: "Too many publishing requests.", + }), + ) + app.use( + "/api/publishing/prototypes/:prototypeId/versions", + express.json({ limit: maximumPublishingJsonBytes }), + ) app.use(express.json()) routes(app, authentication) diff --git a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts index f725e003..79bb7303 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts @@ -1,8 +1,10 @@ import { createHash, randomUUID } from "node:crypto" +import { readFile } from "node:fs/promises" import { describe, expect, it, onTestFinished, vi } from "vitest" import { createArtifactPublishingService } from "#server/domains/artifact-publishing/create-service" +import { maximumArtifactBytes } from "#server/domains/artifact-publishing/create-service/constants" import type { ArtifactManifest } from "#server/domains/artifact-publishing/create-service/types" import { createIdentityStore } from "#server/domains/identity/create-store" import { createPrototypeAuthorizationService } from "#server/domains/prototype-authorization/create-service" @@ -12,7 +14,10 @@ import { digestSecret } from "#server/utils/digest-secret" import type { PrismaClient } from "../../../generated/prisma/client.ts" function artifact(content: string) { - const bytes = Buffer.from(content) + return artifactBytes(Buffer.from(content)) +} + +function artifactBytes(bytes: Buffer) { return { bytes, base64: bytes.toString("base64"), @@ -47,13 +52,13 @@ function manifest( } } -async function database() { +async function database(storageOverride?: ArtifactStorage) { vi.stubGlobal("prisma", undefined) const { prisma } = await import("../../../integrations/prisma") onTestFinished(() => prisma.$disconnect()) const stored = new Map() - const storage: ArtifactStorage = { + const storage: ArtifactStorage = storageOverride ?? { put(digest, bytes) { const key = digest.replace(":", "/") const created = !stored.has(key) @@ -290,6 +295,26 @@ describe("createArtifactPublishingService() database behavior", () => { manifest: { ...valid.manifest, unrecognized: true }, }), }, + { + name: "unknown artifact field", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { + ...valid.manifest, + artifact: { ...valid.manifest.artifact, unrecognized: true }, + }, + }), + }, + { + name: "unknown provenance field", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { + ...valid.manifest, + provenance: { ...valid.manifest.provenance, unrecognized: true }, + }, + }), + }, { name: "unsupported schema version", change: (valid: ReturnType) => ({ @@ -307,6 +332,16 @@ describe("createArtifactPublishingService() database behavior", () => { }, }), }, + { + name: "malformed provenance URL", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { + ...valid.manifest, + provenance: { ...valid.manifest.provenance, sourceUrl: "https://[invalid" }, + }, + }), + }, { name: "provenance URL username", change: (valid: ReturnType) => ({ @@ -333,6 +368,32 @@ describe("createArtifactPublishingService() database behavior", () => { }, }), }, + { + name: "provenance URL query", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { + ...valid.manifest, + provenance: { + ...valid.manifest.provenance, + sourceUrl: "https://ci.example/runs/123?token=private", + }, + }, + }), + }, + { + name: "provenance URL fragment", + change: (valid: ReturnType) => ({ + ...valid, + manifest: { + ...valid.manifest, + provenance: { + ...valid.manifest.provenance, + sourceUrl: "https://ci.example/runs/123#private", + }, + }, + }), + }, { name: "different prototype identifier", change: (valid: ReturnType) => ({ @@ -360,6 +421,50 @@ describe("createArtifactPublishingService() database behavior", () => { expect(await context.prisma.prototypeVersion.count()).toBe(0) }) + it("publishes a valid artifact at the decoded size limit", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const publishedArtifact = artifactBytes(Buffer.alloc(maximumArtifactBytes, 0x61)) + + await expect( + context.service.publish( + credential.authorization, + prototype.id, + "maximum-artifact", + publicationInput(prototype.id, publishedArtifact), + ), + ).resolves.toMatchObject({ replayed: false }) + }) + + it("rejects noncanonical base64 as an invalid artifact instead of an oversized artifact", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const input = publicationInput(prototype.id, artifact("base64")) + input.artifactBase64 = "AB==" + + await expect( + context.service.publish(credential.authorization, prototype.id, "invalid-base64", input), + ).rejects.toMatchObject({ status: 422, code: "invalid_artifact" }) + }) + + it("rejects malformed workload credential identifiers before querying the database", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const findCredential = vi.spyOn(context.prisma.workloadCredential, "findUnique") + + await expect( + context.service.publish( + `Bearer rpl_pub_00000000-0000-0000-0000-00000000000-.${"t".repeat(43)}`, + prototype.id, + "malformed-credential", + publicationInput(prototype.id, artifact("malformed-credential")), + ), + ).rejects.toMatchObject({ status: 401, code: "publisher_authentication_required" }) + expect(findCredential).not.toHaveBeenCalled() + }) + it("rejects an artifact whose decoded size differs from the manifest", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) @@ -772,18 +877,19 @@ describe("createArtifactPublishingService() database behavior", () => { expect(results.map(({ replayed }) => replayed)).toEqual([false, false]) expect(versions.map(({ sequence }) => sequence)).toEqual([1, 2]) - await context.prisma.$executeRawUnsafe(` - CREATE FUNCTION reject_test_prototype_version_mutation() RETURNS trigger AS $$ - BEGIN - RAISE EXCEPTION 'prototype versions are immutable'; - END; - $$ LANGUAGE plpgsql; - `) - await context.prisma.$executeRawUnsafe(` - CREATE TRIGGER test_prototype_versions_immutable - BEFORE UPDATE OR DELETE ON prototype_versions - FOR EACH ROW EXECUTE FUNCTION reject_test_prototype_version_mutation(); - `) + const migration = await readFile( + new URL( + "../../../../schema/migrations/20261027230000_artifact_ingestion/migration.sql", + import.meta.url, + ), + "utf8", + ) + const immutableVersionStart = migration.indexOf( + "CREATE FUNCTION reject_prototype_version_mutation()", + ) + expect(immutableVersionStart).toBeGreaterThanOrEqual(0) + const immutableVersionSql = migration.slice(immutableVersionStart) + await context.prisma.$executeRawUnsafe(immutableVersionSql) try { await expect( context.prisma.prototypeVersion.update({ @@ -796,12 +902,56 @@ describe("createArtifactPublishingService() database behavior", () => { ).rejects.toThrow(/prototype versions are immutable/i) } finally { await context.prisma.$executeRawUnsafe( - "DROP TRIGGER test_prototype_versions_immutable ON prototype_versions", - ) - await context.prisma.$executeRawUnsafe( - "DROP FUNCTION reject_test_prototype_version_mutation()", + "DROP TRIGGER prototype_versions_immutable ON prototype_versions", ) + await context.prisma.$executeRawUnsafe("DROP FUNCTION reject_prototype_version_mutation()") + } + }) + + it("rejects an already-started publication when archive acquires the prototype lock first", async () => { + const putStarted = deferred() + const releasePut = deferred() + const storage: ArtifactStorage = { + async put(digest) { + putStarted.resolve() + await releasePut.promise + return { key: digest.replace(":", "/"), created: true } + }, + read: vi.fn(), + remove: vi.fn(), } + const context = await database(storage) + const sessionToken = "archive-race-session" + const { authorization, prototype, workloadAuthorization } = await provisionPublisher( + context.prisma, + sessionToken, + ) + const publication = context.service.publish( + workloadAuthorization, + prototype.id, + "archive-race", + publicationInput(prototype.id, artifact("archive-race")), + ) + await putStarted.promise + + await authorization.archivePrototype(sessionToken, prototype.id) + releasePut.resolve() + + await expect(publication).rejects.toMatchObject({ + status: 409, + code: "prototype_archived", + }) + expect( + await context.prisma.prototypeVersion.count({ where: { prototypeId: prototype.id } }), + ).toBe(0) + expect( + await context.prisma.artifactPublication.count({ where: { prototypeId: prototype.id } }), + ).toBe(0) + expect( + await context.prisma.securityAudit.count({ + where: { prototypeId: prototype.id, event: "prototype.version.publish" }, + }), + ).toBe(0) }) it("promotes and rolls back a channel by moving one atomic pointer", async () => { diff --git a/hub/server/domains/artifact-publishing/create-service/index.ts b/hub/server/domains/artifact-publishing/create-service/index.ts index 5b48569b..c0ae530c 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.ts @@ -14,36 +14,44 @@ import type { ArtifactPublishingService, PrototypeChannelName, PublisherAction } import type { Prisma, PrismaClient } from "../../../generated/prisma/client.ts" const boundedIdentifier = z.string().trim().min(1).max(255) +function isSafeProvenanceUrl(value: string): boolean { + try { + const url = new URL(value) + return ( + ["http:", "https:"].includes(url.protocol) && + url.username.length === 0 && + url.password.length === 0 && + url.search.length === 0 && + url.hash.length === 0 + ) + } catch { + return false + } +} + const manifestSchema = z .object({ schemaVersion: z.literal(1), prototypeId: z.uuid(), - artifact: z.object({ - format: z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/), - version: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,31}$/), - digest: z.string().regex(/^sha256:[a-f0-9]{64}$/), - sizeBytes: z.number().int().min(0).max(maximumArtifactBytes), - builtAt: z.iso.datetime({ offset: true }), - }), - provenance: z.object({ - provider: z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/), - sourceUrl: z - .url() - .max(2_048) - .refine((value) => { - const url = new URL(value) - return ( - ["http:", "https:"].includes(url.protocol) && - url.username.length === 0 && - url.password.length === 0 - ) - }) - .optional(), - revision: boundedIdentifier.optional(), - ref: boundedIdentifier.optional(), - pipeline: boundedIdentifier.optional(), - runId: boundedIdentifier.optional(), - }), + artifact: z + .object({ + format: z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/), + version: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,31}$/), + digest: z.string().regex(/^sha256:[a-f0-9]{64}$/), + sizeBytes: z.number().int().min(0).max(maximumArtifactBytes), + builtAt: z.iso.datetime({ offset: true }), + }) + .strict(), + provenance: z + .object({ + provider: z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/), + sourceUrl: z.string().max(2_048).refine(isSafeProvenanceUrl).optional(), + revision: boundedIdentifier.optional(), + ref: boundedIdentifier.optional(), + pipeline: boundedIdentifier.optional(), + runId: boundedIdentifier.optional(), + }) + .strict(), }) .strict() const publishSchema = z @@ -61,7 +69,8 @@ const idempotencyKeySchema = z .min(1) .max(200) .regex(/^[\x21-\x7e]+$/) -const authorizationSchema = /^Bearer (rpl_pub_([0-9a-f-]{36})\.([A-Za-z0-9_-]{43}))$/ +const authorizationSchema = + /^Bearer (rpl_pub_([0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\.([A-Za-z0-9_-]{43}))$/i function error(status: number, code: string, message: string): never { throw new AuthenticationError(status, code, message) @@ -82,14 +91,14 @@ function parseIdempotencyKey(value: string): string { } function parseArtifactBase64(value: string): Uint8Array { - if ( - value.length % 4 !== 0 || - !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value) - ) { + if (value.length % 4 !== 0) { error(422, "invalid_artifact", "The artifact encoding is invalid.") } const bytes = Buffer.from(value, "base64") - if (bytes.length > maximumArtifactBytes || bytes.toString("base64") !== value) { + if (bytes.toString("base64") !== value) { + error(422, "invalid_artifact", "The artifact encoding is invalid.") + } + if (bytes.length > maximumArtifactBytes) { error(413, "artifact_too_large", "The artifact exceeds the maximum size.") } return bytes diff --git a/hub/server/domains/authentication/create-runtime/index.test.ts b/hub/server/domains/authentication/create-runtime/index.test.ts new file mode 100644 index 00000000..1fdf2b99 --- /dev/null +++ b/hub/server/domains/authentication/create-runtime/index.test.ts @@ -0,0 +1,36 @@ +import os from "node:os" +import path from "node:path" + +import { describe, expect, it, vi } from "vitest" + +const { createLocalArtifactStorage } = vi.hoisted(() => ({ + createLocalArtifactStorage: vi.fn((root: string) => { + void root + return { + put: vi.fn(), + read: vi.fn(), + remove: vi.fn(), + } + }), +})) + +vi.mock("#server/integrations/artifact-storage/create-local-storage", () => ({ + createLocalArtifactStorage, +})) +vi.mock("#server/integrations/prisma", () => ({ prisma: {} })) + +import { createAuthenticationRuntime } from "#server/domains/authentication/create-runtime" + +describe("createAuthenticationRuntime()", () => { + it("keeps default artifact storage outside the checkout", async () => { + await createAuthenticationRuntime({}) + + expect(createLocalArtifactStorage).toHaveBeenCalledWith( + path.join(os.homedir(), ".replica", "artifacts"), + ) + const storagePath = createLocalArtifactStorage.mock.calls[0]?.[0] + expect(storagePath).toBeTypeOf("string") + if (!storagePath) throw new Error("Expected an artifact storage path") + expect(path.relative(process.cwd(), storagePath)).toMatch(/^\.\./) + }) +}) diff --git a/hub/server/domains/authentication/create-runtime/index.ts b/hub/server/domains/authentication/create-runtime/index.ts index 293a03f7..3a61da70 100644 --- a/hub/server/domains/authentication/create-runtime/index.ts +++ b/hub/server/domains/authentication/create-runtime/index.ts @@ -1,4 +1,5 @@ import path from "node:path" +import os from "node:os" import { createAdmissionPolicy } from "#server/domains/authentication/create-admission-policy" import { createArtifactPublishingService } from "#server/domains/artifact-publishing/create-service" @@ -94,7 +95,7 @@ export async function createAuthenticationRuntime(environment: NodeJS.ProcessEnv trustedProxyHops, } const artifactStoragePath = path.resolve( - environment.HUB_ARTIFACT_STORAGE_PATH ?? path.join(process.cwd(), ".data", "artifacts"), + environment.HUB_ARTIFACT_STORAGE_PATH ?? path.join(os.homedir(), ".replica", "artifacts"), ) return { administration: createAdministrationService({ store }), diff --git a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts index 250d9cde..0a051b54 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts @@ -963,6 +963,72 @@ describe("createPrototypeAuthorizationService() database behavior", () => { ).toBe(4) }) + it("requires publish permission when provisioning a publisher", async () => { + const { prisma, service, store } = await database() + const token = "publisher-action-token" + const { prototype } = await createAdministeredPrototype({ + service, + store, + subject: "publisher-action", + token, + }) + + await expect( + service.createPublisherCredential(token, prototype.id, { + name: "Status-only publisher", + actions: ["read-status"], + }), + ).rejects.toMatchObject({ status: 422, code: "invalid_workload_publisher" }) + expect(await prisma.workloadPublisher.count({ where: { prototypeId: prototype.id } })).toBe(0) + }) + + it("lists bounded publisher and credential metadata without reusable secrets", async () => { + const { prisma, service, store } = await database() + const token = "publisher-list-token" + const { prototype } = await createAdministeredPrototype({ + service, + store, + subject: "publisher-list", + token, + }) + const first = (await service.createPublisherCredential(token, prototype.id, { + name: "First publisher", + actions: ["publish"], + })) as { credential: { token: string } } + const second = (await service.createPublisherCredential(token, prototype.id, { + name: "Second publisher", + actions: ["publish", "read-status"], + })) as { credential: { token: string } } + + const firstPage = (await service.listPublishers(token, prototype.id, { + limit: 1, + })) as { + items: { id: string; credentials: { id: string }[] }[] + nextCursor: string | null + } + const secondPage = (await service.listPublishers(token, prototype.id, { + limit: 1, + cursor: firstPage.nextCursor, + })) as { + items: { id: string; credentials: { id: string }[] }[] + nextCursor: string | null + } + const serialized = JSON.stringify([firstPage, secondPage]) + const tokenDigests = await prisma.workloadCredential.findMany({ + select: { tokenDigest: true }, + }) + + expect(firstPage.items).toHaveLength(1) + expect(firstPage.nextCursor).toBe(firstPage.items[0]?.id) + expect(secondPage.items).toHaveLength(1) + expect(secondPage.nextCursor).toBeNull() + expect(new Set([...firstPage.items, ...secondPage.items].map(({ id }) => id)).size).toBe(2) + expect(serialized).not.toContain(first.credential.token) + expect(serialized).not.toContain(second.credential.token) + for (const { tokenDigest } of tokenDigests) expect(serialized).not.toContain(tokenDigest) + expect(serialized).not.toContain("tokenDigest") + }) + it("does not apply publisher or credential lifecycle operations across prototypes", async () => { const { prisma, service, store } = await database() const token = "publisher-scope-token" diff --git a/hub/server/domains/prototype-authorization/create-service/index.ts b/hub/server/domains/prototype-authorization/create-service/index.ts index c517cf7d..6376dd35 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.ts @@ -29,7 +29,7 @@ const publisherSchema = z .array(z.enum(["publish", "read-status"])) .min(1) .max(2) - .refine((actions) => new Set(actions).size === actions.length), + .refine((actions) => new Set(actions).size === actions.length && actions.includes("publish")), expiresAt: z.iso.datetime({ offset: true }).nullable().optional(), }) .strict() @@ -1006,6 +1006,47 @@ export function createPrototypeAuthorizationService({ }) }, + async listPublishers(sessionToken, prototypeId, query) { + const { cursor, limit } = parsePage(query) + return store.lock(async (_records, transaction) => { + const actor = await currentSession(store, sessionToken, transaction) + requireFreshAuthorization(actor) + requireIdentifier(prototypeId, "prototype_not_found", "The prototype was not found.") + await lockPrototype(transaction, prototypeId) + const { decision } = await prototypeDecision( + transaction, + actor.userId, + prototypeId, + baseline, + ) + requireDecision(decision, "administrator") + const items = await transaction.workloadPublisher.findMany({ + where: { prototypeId }, + orderBy: { id: "asc" }, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + take: limit + 1, + select: { + id: true, + name: true, + actions: true, + active: true, + createdAt: true, + updatedAt: true, + credentials: { + orderBy: { createdAt: "asc" }, + select: { + id: true, + expiresAt: true, + revokedAt: true, + createdAt: true, + }, + }, + }, + }) + return pageResult(items, limit) + }) + }, + async setPublisherActive(sessionToken, prototypeId, publisherId, input) { const parsed = publisherStateSchema.safeParse(input) if (!parsed.success) { diff --git a/hub/server/domains/prototype-authorization/create-service/types.ts b/hub/server/domains/prototype-authorization/create-service/types.ts index 21cfb109..e578434d 100644 --- a/hub/server/domains/prototype-authorization/create-service/types.ts +++ b/hub/server/domains/prototype-authorization/create-service/types.ts @@ -29,6 +29,7 @@ export interface PrototypeAuthorizationService { prototypeId: string, input: unknown, ): Promise + listPublishers(sessionToken: string, prototypeId: string, query: unknown): Promise> setPublisherActive( sessionToken: string, prototypeId: string, diff --git a/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts b/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts index e72e9f55..d0273e2a 100644 --- a/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts +++ b/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts @@ -1,8 +1,8 @@ import { createHash, randomUUID } from "node:crypto" -import { mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises" +import { mkdir, open, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises" import path from "node:path" -import { describe, expect, it, onTestFinished } from "vitest" +import { describe, expect, it, onTestFinished, vi } from "vitest" import { createLocalArtifactStorage } from "#server/integrations/artifact-storage/create-local-storage" @@ -117,6 +117,62 @@ describe("createLocalArtifactStorage()", () => { expect(Buffer.from(await storage.read(results[0].key))).toEqual(bytes) }) + it.each(["writeFile", "sync"] as const)( + "removes its temporary file when %s fails", + async (operation) => { + const root = storageRoot() + const bytes = Buffer.from(`failed artifact ${operation}`) + const contentDigest = digest(bytes) + const location = artifactPath(root, contentDigest) + await mkdir(location.shard, { recursive: true }) + const probe = await open(path.join(root, "probe"), "w") + await probe.close() + const failure = vi + .spyOn(Object.getPrototypeOf(probe) as typeof probe, operation) + .mockRejectedValue(new Error(`${operation} failed`)) + + await expect(createLocalArtifactStorage(root).put(contentDigest, bytes)).rejects.toThrow( + `${operation} failed`, + ) + + expect((await readdir(location.shard)).filter((name) => name.endsWith(".tmp"))).toEqual([]) + failure.mockRestore() + }, + ) + + it("waits for a failed read before closing the artifact file", async () => { + const root = storageRoot() + const storage = createLocalArtifactStorage(root) + const bytes = Buffer.from("failed artifact read") + const stored = await storage.put(digest(bytes), bytes) + const probe = await open(path.join(root, "probe"), "w") + const fileHandlePrototype = Object.getPrototypeOf(probe) as typeof probe + await probe.close() + let rejectRead!: (error: Error) => void + let artifactFile: typeof probe | undefined + function captureArtifactFile(file: typeof probe) { + artifactFile = file + } + const readFailure = new Promise((_resolve, reject) => { + rejectRead = reject + }) + const readFile = vi.spyOn(fileHandlePrototype, "readFile").mockImplementation(function ( + this: typeof probe, + ) { + captureArtifactFile(this) + return readFailure + }) + + const read = storage.read(stored.key) + await vi.waitFor(() => expect(artifactFile).toBeDefined()) + await expect(artifactFile!.stat()).resolves.toBeDefined() + rejectRead(new Error("read failed")) + + await expect(read).rejects.toThrow("read failed") + await expect(artifactFile!.stat()).rejects.toMatchObject({ code: "EBADF" }) + readFile.mockRestore() + }) + it("rejects a symbolic-link storage root without reading, replaying, or removing outside content", async () => { const root = storageRoot() const outside = storageRoot() diff --git a/hub/server/integrations/artifact-storage/create-local-storage/index.ts b/hub/server/integrations/artifact-storage/create-local-storage/index.ts index d6fa5eb7..d2827ca6 100644 --- a/hub/server/integrations/artifact-storage/create-local-storage/index.ts +++ b/hub/server/integrations/artifact-storage/create-local-storage/index.ts @@ -155,22 +155,24 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { 0o600, ) try { - await file.writeFile(bytes) - await file.sync() - } finally { - await file.close() - } - try { - const resolved = await resolveArtifactPath(root, key, false) - if (resolved.destination !== destination) { - throw new Error("Artifact storage path changed during the write.") + try { + await file.writeFile(bytes) + await file.sync() + } finally { + await file.close() + } + try { + const resolved = await resolveArtifactPath(root, key, false) + if (resolved.destination !== destination) { + throw new Error("Artifact storage path changed during the write.") + } + await link(temporary, destination) + return { key, created: true } + } catch (error) { + if (!isFileSystemError(error, "EEXIST")) throw error + await verifyExisting(resolvedRoot, destination, contentDigest) + return { key, created: false } } - await link(temporary, destination) - return { key, created: true } - } catch (error) { - if (!isFileSystemError(error, "EEXIST")) throw error - await verifyExisting(resolvedRoot, destination, contentDigest) - return { key, created: false } } finally { await rm(temporary, { force: true }) } @@ -180,7 +182,7 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { const { destination, resolvedRoot } = await resolveArtifactPath(root, key, false) const file = await openArtifact(resolvedRoot, destination) try { - return file.readFile() + return await file.readFile() } finally { await file.close() } diff --git a/hub/server/routes/index.ts b/hub/server/routes/index.ts index 8fbe4ce9..42f27f8f 100644 --- a/hub/server/routes/index.ts +++ b/hub/server/routes/index.ts @@ -25,6 +25,7 @@ import { getPrototype, getVersion, getWorkspace, + listPublishers, listPrototypes, listVersions, listWorkspaces, @@ -161,6 +162,7 @@ export function routes(app: Express, authentication?: AuthenticationRoutes): voi productMutationLimit, createPublisherCredential.bind(null, authentication), ) + app.get("/api/prototypes/:prototypeId/publishers", listPublishers.bind(null, authentication)) app.patch( "/api/prototypes/:prototypeId/publishers/:publisherId", productMutationLimit, @@ -189,24 +191,16 @@ export function routes(app: Express, authentication?: AuthenticationRoutes): voi app.get("/api/prototypes/:prototypeId/versions/:versionId", getVersion.bind(null, authentication)) app.use(["/api/workspaces", "/api/prototypes"], respondAdministrationError) if (authentication.artifactPublishing) { - const publishingLimit = createRateLimit({ - limit: 60, - windowMilliseconds: 5 * 60_000, - message: "Too many publishing requests.", - }) app.post( "/api/publishing/prototypes/:prototypeId/versions", - publishingLimit, publishArtifact.bind(null, authentication.artifactPublishing), ) app.get( "/api/publishing/prototypes/:prototypeId/status", - publishingLimit, getPublicationStatus.bind(null, authentication.artifactPublishing), ) app.put( "/api/publishing/prototypes/:prototypeId/channels/:channel", - publishingLimit, promoteVersion.bind(null, authentication.artifactPublishing), ) app.use("/api/publishing", respondAdministrationError) diff --git a/hub/server/routes/product/index.test.ts b/hub/server/routes/product/index.test.ts index 546668f3..31f92a63 100644 --- a/hub/server/routes/product/index.test.ts +++ b/hub/server/routes/product/index.test.ts @@ -40,6 +40,7 @@ function runtime( archivePrototype: vi.fn(), setPrototypeAuthorization: vi.fn(), createPublisherCredential: vi.fn(), + listPublishers: vi.fn(), setPublisherActive: vi.fn(), revokePublisherCredential: vi.fn(), explainPrototypeAccess: vi.fn(), @@ -236,6 +237,11 @@ describe("product routes", () => { call: "createPublisherCredential", body: { name: "Local publisher", actions: ["publish"] }, }, + { + method: "GET", + path: "/api/prototypes/prototype/publishers", + call: "listPublishers", + }, { method: "PATCH", path: "/api/prototypes/prototype/publishers/publisher", @@ -399,6 +405,28 @@ describe("product routes", () => { expect(JSON.stringify(records)).not.toMatch(/private-query-value|private-body-value/) }) + it("forwards publisher pagination and returns the bounded metadata page", async () => { + const page = { + items: [{ id: "publisher-id", credentials: [{ id: "credential-id" }] }], + nextCursor: "publisher-id", + } + const listPublishers = vi.fn().mockResolvedValue(page) + const authentication = runtime({ listPublishers }) + const baseUrl = await serve(authentication) + + const response = await fetch( + `${baseUrl}/api/prototypes/prototype/publishers?limit=1&cursor=cursor-id`, + { headers: { cookie: `__Host-replica-session=${sessionToken}` } }, + ) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual(page) + expect(listPublishers).toHaveBeenCalledWith(sessionToken, "prototype", { + limit: "1", + cursor: "cursor-id", + }) + }) + it("allows 120 product mutations per boundary and rejects the next one", async () => { const authentication = runtime({ createWorkspace: vi.fn().mockResolvedValue({ id: "workspace" }), diff --git a/hub/server/routes/product/index.ts b/hub/server/routes/product/index.ts index 2126c361..f946f38a 100644 --- a/hub/server/routes/product/index.ts +++ b/hub/server/routes/product/index.ts @@ -165,6 +165,19 @@ export async function createPublisherCredential( response.status(201).json(result) } +export async function listPublishers( + authentication: AuthenticationRoutes, + request: Request, + response: Response, +): Promise { + const publishers = await authentication.prototypeAuthorization.listPublishers( + sessionToken(authentication, request), + parameter(request, "prototypeId"), + request.query, + ) + response.json(publishers) +} + export async function setPublisherActive( authentication: AuthenticationRoutes, request: Request, diff --git a/hub/server/routes/publishing.test.ts b/hub/server/routes/publishing.test.ts index c22888f5..58fd5437 100644 --- a/hub/server/routes/publishing.test.ts +++ b/hub/server/routes/publishing.test.ts @@ -232,4 +232,74 @@ describe("publishing routes", () => { expect(records[0]).not.toHaveProperty("body") expect(JSON.stringify(records)).not.toContain(privateContent.slice(0, 100)) }) + + it("applies publishing rate limits before parsing malformed JSON", async () => { + const getStatus = vi.fn().mockResolvedValue({ publicationId: null }) + const publish = vi.fn() + const baseUrl = await serve( + runtime({ getStatus, publish }), + createLogger({ destination: { write: () => undefined } }), + ) + + for (let request = 0; request < 60; request++) { + const response = await fetch(`${baseUrl}/api/publishing/prototypes/prototype-id/status`, { + headers: { authorization: "******" }, + }) + expect(response.status).toBe(200) + } + const rejected = await fetch(`${baseUrl}/api/publishing/prototypes/prototype-id/versions`, { + method: "POST", + headers: { + authorization: "******", + "content-type": "application/json", + "idempotency-key": "rate-limited", + }, + body: "{", + }) + + expect(rejected.status).toBe(429) + expect(await rejected.json()).toEqual({ + error: "rate_limited", + message: "Too many publishing requests.", + }) + expect(getStatus).toHaveBeenCalledTimes(60) + expect(publish).not.toHaveBeenCalled() + }) + + it("uses the larger JSON limit only for version uploads", async () => { + const publish = vi.fn().mockResolvedValue({ + version: { id: "version-id" }, + replayed: false, + promotedChannel: null, + }) + const promote = vi.fn() + const baseUrl = await serve(runtime({ publish, promote })) + const padding = "x".repeat(128 * 1024) + + const upload = await fetch(`${baseUrl}/api/publishing/prototypes/prototype-id/versions`, { + method: "POST", + headers: { + authorization: "******", + "content-type": "application/json", + "idempotency-key": "larger-parser", + }, + body: JSON.stringify({ padding }), + }) + const promotion = await fetch( + `${baseUrl}/api/publishing/prototypes/prototype-id/channels/current`, + { + method: "PUT", + headers: { + authorization: "******", + "content-type": "application/json", + }, + body: JSON.stringify({ padding }), + }, + ) + + expect(upload.status).toBe(201) + expect(publish).toHaveBeenCalledOnce() + expect(promotion.status).toBe(413) + expect(promote).not.toHaveBeenCalled() + }) }) From 5229f40297a76a82fa9ec890f9df2b91aa3460b2 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Wed, 28 Oct 2026 14:11:41 -0700 Subject: [PATCH 05/13] test(hub): preserve baseline transaction coverage Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../create-service/index.db.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts index 0a051b54..90131044 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts @@ -2403,7 +2403,7 @@ describe("createPrototypeAuthorizationService() database behavior", () => { it("passes the mutation transaction to a synchronized prototype baseline", async () => { const baselineAccess = vi.fn().mockResolvedValue({ - role: "editor", + role: "administrator", repositorySourceId: "repository-source", githubRepositoryId: "100", rawPermission: "write", @@ -2427,10 +2427,11 @@ describe("createPrototypeAuthorizationService() database behavior", () => { }) await expect( - service.createVersion("transaction-baseline-token", prototype.id, { - label: "Baseline version", + service.createPublisherCredential("transaction-baseline-token", prototype.id, { + name: "Baseline publisher", + actions: ["publish"], }), - ).resolves.toMatchObject({ prototypeId: prototype.id, sequence: 1 }) + ).resolves.toMatchObject({ publisher: { prototypeId: prototype.id } }) expect(baselineAccess).toHaveBeenCalledOnce() expect(baselineAccess.mock.calls[0]?.slice(1)).toEqual([actor.user.id, prototype.id]) From b450f55151276f54cee9793816be06c98eea1ed3 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Wed, 28 Oct 2026 14:16:23 -0700 Subject: [PATCH 06/13] fix(hub): integrate final authorization parent Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../create-service/index.db.test.ts | 7 ++++--- .../prototype-authorization/create-service/index.ts | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts index 90131044..9489304c 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts @@ -2098,10 +2098,11 @@ describe("createPrototypeAuthorizationService() database behavior", () => { service.getVersion("baseline-single-prototype-token", prototype.id, firstVersion.id), ).resolves.toMatchObject({ id: firstVersion.id }) await expect( - service.createVersion("baseline-single-prototype-token", prototype.id, { - label: "Second", + service.createPublisherCredential("baseline-single-prototype-token", prototype.id, { + name: "Baseline publisher", + actions: ["publish"], }), - ).resolves.toMatchObject({ sequence: 2 }) + ).resolves.toMatchObject({ publisher: { prototypeId: prototype.id } }) await expect( service.setPrototypeAccess("baseline-single-prototype-token", prototype.id, target.user.id, { effect: "allow", diff --git a/hub/server/domains/prototype-authorization/create-service/index.ts b/hub/server/domains/prototype-authorization/create-service/index.ts index 6376dd35..e9b26dae 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.ts @@ -956,10 +956,10 @@ export function createPrototypeAuthorizationService({ requireIdentifier(prototypeId, "prototype_not_found", "The prototype was not found.") await lockPrototype(transaction, prototypeId) const { prototype, decision } = await prototypeDecision( + baseline, transaction, actor.userId, prototypeId, - baseline, ) requireDecision(decision, "administrator") if (prototype.archivedAt) { @@ -1014,10 +1014,10 @@ export function createPrototypeAuthorizationService({ requireIdentifier(prototypeId, "prototype_not_found", "The prototype was not found.") await lockPrototype(transaction, prototypeId) const { decision } = await prototypeDecision( + baseline, transaction, actor.userId, prototypeId, - baseline, ) requireDecision(decision, "administrator") const items = await transaction.workloadPublisher.findMany({ @@ -1067,10 +1067,10 @@ export function createPrototypeAuthorizationService({ ) await lockPrototype(transaction, prototypeId) const { prototype, decision } = await prototypeDecision( + baseline, transaction, actor.userId, prototypeId, - baseline, ) requireDecision(decision, "administrator") if (prototype.archivedAt && parsed.data.active) { @@ -1123,10 +1123,10 @@ export function createPrototypeAuthorizationService({ ) await lockPrototype(transaction, prototypeId) const { prototype, decision } = await prototypeDecision( + baseline, transaction, actor.userId, prototypeId, - baseline, ) requireDecision(decision, "administrator") const credential = await transaction.workloadCredential.findFirst({ From ee5b677e721e52ba55aad405f838ab82f459ed1e Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Wed, 28 Oct 2026 16:04:32 -0700 Subject: [PATCH 07/13] fix(hub): harden artifact durability and recovery Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- hub/docs/artifact-ingestion.md | 4 +- hub/docs/logging.md | 2 +- hub/docs/testing-and-operations.md | 2 +- hub/server/create-app/index.ts | 2 + .../create-service/index.db.test.ts | 130 +++++++++++++++++- .../create-runtime/index.test.ts | 37 +++-- .../authentication/create-runtime/index.ts | 6 +- .../create-service/index.db.test.ts | 129 ++++++++++++++++- .../create-service/index.ts | 60 ++++---- .../create-local-storage/index.test.ts | 76 ++++++++++ .../create-local-storage/index.ts | 18 +++ .../prepare-publishing-request/index.ts | 11 ++ hub/server/routes/publishing.test.ts | 8 ++ 13 files changed, 429 insertions(+), 56 deletions(-) create mode 100644 hub/server/integrations/express/prepare-publishing-request/index.ts diff --git a/hub/docs/artifact-ingestion.md b/hub/docs/artifact-ingestion.md index 03a663b1..97b3826c 100644 --- a/hub/docs/artifact-ingestion.md +++ b/hub/docs/artifact-ingestion.md @@ -57,7 +57,7 @@ Publishing requests are rate-limited before JSON parsing. The version-upload rou `ArtifactStorage` separates ingestion from byte storage. The reference adapter writes content-addressed files under `HUB_ARTIFACT_STORAGE_PATH`, defaulting to `~/.replica/artifacts` outside the source checkout and Vite development root. Database rows store only the storage key and validated metadata, never the artifact bytes. -The local adapter is suitable for one Hub process with a durable mounted filesystem. It rejects symbolic-link roots, shards, and artifact paths, resolves directories under the configured root, and opens artifact files without following a final symbolic link. Operators must keep the storage directory writable only by the Hub workload; the local adapter does not provide replication, remote durability, encryption-key management, hostile concurrent filesystem isolation, or multi-region consistency. A production object-store adapter must preserve the same content-addressed, immutable `put`, `read`, and `remove` boundary. +The local adapter is suitable for one Hub process with a durable mounted filesystem. It rejects relative configured paths and symbolic-link roots, shards, and artifact paths, resolves directories under the configured root, opens artifact files without following a final symbolic link, and syncs file and directory entries before reporting a successful write. Operators must keep the storage directory writable only by the Hub workload; the local adapter does not provide replication, remote durability, encryption-key management, hostile concurrent filesystem isolation, or multi-region consistency. A production object-store adapter must preserve the same content-addressed, immutable `put`, `read`, and `remove` boundary. Storage happens before the database transaction. A failed transaction can therefore leave an unreferenced content-addressed file. Hub never deletes referenced bytes automatically. Operators may remove only files that have no matching `PrototypeVersion.storageKey`; retain unreferenced files long enough to exclude in-flight transactions and backups. Version metadata has no deletion API and must not be rewritten silently. @@ -73,7 +73,7 @@ Publishing locks the prototype while it assigns the next sequence. Concurrent re ## Status, audits, and failures -`GET /api/publishing/prototypes/:prototypeId/status` reads the publication associated with the caller's `Idempotency-Key`. It requires `read-status`. +`GET /api/publishing/prototypes/:prototypeId/status` reads the publication associated with the caller's `Idempotency-Key`. It requires `read-status`. Publishing responses are private and non-cacheable because the idempotency key is carried in a request header rather than the URL. Version creation, idempotency ownership, optional promotion, and the `prototype.version.publish` security audit commit in one transaction. Promotion writes `prototype.channel.promote` in its transaction. Audit failure rolls back database state. Responses expose validated version metadata and stable error codes, not credentials, storage paths, or artifact contents. Publishing routes omit headers and request payloads from structured request logs. diff --git a/hub/docs/logging.md b/hub/docs/logging.md index 78b39aea..0a8bd4ec 100644 --- a/hub/docs/logging.md +++ b/hub/docs/logging.md @@ -30,7 +30,7 @@ Each request produces one final record with `reqId`, `method`, `route`, `statusC Automatic records exclude raw URLs, headers, cookies, uploads, response bodies, and error details. JSON parsing uses Express's default 100 KiB limit; rejected bodies are not logged. The Express error handler returns generic JSON errors, or closes the connection if headers were sent, without exposing raw errors. -`createApp()` installs `omitRequestLogPayload` for authentication, session, administration, product, and publishing paths after request logging but before JSON parsing. Publishing rate limits also run before parsing, and only the artifact-upload route receives the larger bounded parser. This ordering covers parser failures as well as later origin checks and handlers: +`createApp()` installs `omitRequestLogPayload` for authentication, session, administration, product, and publishing paths after request logging but before JSON parsing. Publishing responses receive private, non-cacheable headers before rate limits and parsing; publishing rate limits also run before parsing, and only the artifact-upload route receives the larger bounded parser. This ordering covers parser failures as well as later origin checks and handlers: ```ts import { omitRequestLogPayload } from "../integrations/pino/omit-request-log-payload/index.ts" diff --git a/hub/docs/testing-and-operations.md b/hub/docs/testing-and-operations.md index 900be503..2e780e97 100644 --- a/hub/docs/testing-and-operations.md +++ b/hub/docs/testing-and-operations.md @@ -57,7 +57,7 @@ The database command uses `--passWithNoTests` so an empty selection exits succes A dedicated migration case applies the foundation migration, seeds retained lifecycle and security records, then applies the next migration. This catches upgrade behavior that an empty final schema cannot prove, including audit-outcome backfills and lifecycle constraints. -Prototype authorization cases cover direct-rule precedence, explicit deny, no-access defaults, GitHub baseline mapping and staleness, prototype-specific restrictions, cross-prototype and cross-repository isolation, installation membership versus product authority, stale sessions, deterministic overlapping administrator changes, membership lifecycle conflicts, audit rollback, route enforcement, scalar list responses, deny-aware and terminal pagination, version ownership, workload credential revocation, publisher suspension, archive lifecycle, and explainable decisions. Artifact publishing cases cover manifest bounds, credential-free provenance URLs, digest and size verification, scoped workload actions, token non-storage, idempotent replay and conflicts, concurrent sequence assignment, immutable rows, atomic promotion and rollback, revocation and disable races, cross-prototype isolation, audit rollback, body limits, request-log omission, and the local non-GitHub producer contract. Local storage cases reject symbolic-link roots, shards, and artifact paths. +Prototype authorization cases cover direct-rule precedence, explicit deny, no-access defaults, GitHub baseline mapping and staleness, prototype-specific restrictions, cross-prototype and cross-repository isolation, installation membership versus product authority, stale sessions, deterministic overlapping administrator changes, membership lifecycle conflicts, audit rollback, route enforcement, scalar list responses, deny-aware and terminal pagination, version ownership, workload credential revocation, publisher suspension, provisioning authority, lock-free metadata reads, archive lifecycle, and explainable decisions. Artifact publishing cases cover manifest bounds, credential-free provenance URLs, digest and size verification, scoped workload actions, token non-storage, idempotent replay and conflicts, status isolation, concurrent sequence assignment, immutable rows, atomic publish-and-promotion, promotion rollback and replay, revocation and disable races, cross-prototype isolation, audit rollback, body limits, request-log omission, and the local non-GitHub producer contract. Local storage cases reject relative and symbolic-link paths and cover durable directory synchronization. GitHub synchronization cases use mocked GitHub API responses and real database transactions. They cover transactional cache revalidation, invitation precedence during source outage, immutable-ID rename and 404 invalidation, token-404 installation removal, credential-independent webhook revocation, retryable source-generation changes, per-result confirmation ordering, active-identity filtering, bounded provider requests and apply batches, partial scans, dirty-event retry coalescing, first-enrollment and admission-failure auditing, repository and installation lifecycle, provider-response versus apply failures, scheduled failure logging, stale per-user and overlapping reconciliation or sign-in, manual suspension and audited administration preservation, compatibility membership isolation, synchronized session freshness, and rollback. diff --git a/hub/server/create-app/index.ts b/hub/server/create-app/index.ts index f58b96b0..14ed4740 100644 --- a/hub/server/create-app/index.ts +++ b/hub/server/create-app/index.ts @@ -4,6 +4,7 @@ import type { Logger } from "pino" import { maximumPublishingJsonBytes } from "../domains/artifact-publishing/create-service/constants.ts" import { createRateLimit } from "../integrations/express/create-rate-limit/index.ts" import { handleError } from "../integrations/express/handle-error/index.ts" +import { preparePublishingRequest } from "../integrations/express/prepare-publishing-request/index.ts" import { respondNotFound } from "../integrations/express/respond-not-found/index.ts" import { createRequestLogger } from "../integrations/pino/create-request-logger/index.ts" import { omitRequestLogPayload } from "../integrations/pino/omit-request-log-payload/index.ts" @@ -32,6 +33,7 @@ export function createApp({ omitRequestLogPayload, ) app.use("/api/github/webhooks", express.raw({ type: "application/json", limit: "1mb" })) + app.use("/api/publishing", preparePublishingRequest) app.use( "/api/publishing", createRateLimit({ diff --git a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts index 79bb7303..e84f4867 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts @@ -833,6 +833,127 @@ describe("createArtifactPublishingService() database behavior", () => { ).toBe(1) }) + it("commits a promoted publication, version, channel, and audit together", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const input = publicationInput(prototype.id, artifact("publish-and-promote"), { + promoteChannel: "preview", + }) + + const result = await context.service.publish( + credential.authorization, + prototype.id, + "publish-and-promote", + input, + ) + const version = publishedVersion(result) + + expect(result).toMatchObject({ replayed: false, promotedChannel: "preview" }) + expect( + await context.prisma.prototypeVersion.findMany({ + where: { prototypeId: prototype.id }, + select: { id: true, workloadPublisherId: true }, + }), + ).toEqual([{ id: version.id, workloadPublisherId: credential.publisher.id }]) + expect( + await context.prisma.artifactPublication.findMany({ + where: { prototypeId: prototype.id }, + select: { prototypeVersionId: true, workloadPublisherId: true }, + }), + ).toEqual([ + { + prototypeVersionId: version.id, + workloadPublisherId: credential.publisher.id, + }, + ]) + expect( + await context.prisma.prototypeChannel.findMany({ + where: { prototypeId: prototype.id }, + select: { name: true, prototypeVersionId: true }, + }), + ).toEqual([{ name: "preview", prototypeVersionId: version.id }]) + expect( + await context.prisma.securityAudit.findMany({ + where: { prototypeId: prototype.id, event: "prototype.version.publish" }, + select: { prototypeVersionId: true, workloadPublisherId: true }, + }), + ).toEqual([ + { + prototypeVersionId: version.id, + workloadPublisherId: credential.publisher.id, + }, + ]) + }) + + it("does not move a later channel promotion when replaying an earlier publication", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const firstInput = publicationInput(prototype.id, artifact("earlier-publication"), { + promoteChannel: "current", + }) + const first = await context.service.publish( + credential.authorization, + prototype.id, + "earlier-publication", + firstInput, + ) + const second = await context.service.publish( + credential.authorization, + prototype.id, + "later-publication", + publicationInput(prototype.id, artifact("later-publication"), { + promoteChannel: "current", + }), + ) + + await expect( + context.service.publish( + credential.authorization, + prototype.id, + "earlier-publication", + firstInput, + ), + ).resolves.toMatchObject({ + replayed: true, + version: publishedVersion(first), + promotedChannel: "current", + }) + expect( + await context.prisma.prototypeChannel.findUniqueOrThrow({ + where: { prototypeId_name: { prototypeId: prototype.id, name: "current" } }, + select: { prototypeVersionId: true }, + }), + ).toEqual({ prototypeVersionId: publishedVersion(second).id }) + }) + + it("isolates status reads for publishers that reuse the same idempotency key", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const firstPublisher = await createCredential(context.prisma, prototype.id) + const secondPublisher = await createCredential(context.prisma, prototype.id) + const first = await context.service.publish( + firstPublisher.authorization, + prototype.id, + "shared-request-key", + publicationInput(prototype.id, artifact("first publisher version")), + ) + const second = await context.service.publish( + secondPublisher.authorization, + prototype.id, + "shared-request-key", + publicationInput(prototype.id, artifact("second publisher version")), + ) + + await expect( + context.service.getStatus(firstPublisher.authorization, prototype.id, "shared-request-key"), + ).resolves.toMatchObject({ version: publishedVersion(first) }) + await expect( + context.service.getStatus(secondPublisher.authorization, prototype.id, "shared-request-key"), + ).resolves.toMatchObject({ version: publishedVersion(second) }) + }) + it("rejects reuse of an idempotency key for a different publication", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) @@ -1048,7 +1169,7 @@ describe("createArtifactPublishingService() database behavior", () => { ).toBe(0) }) - it("rolls back a published version when its audit cannot be saved", async () => { + it("rolls back a promoted publication, version, channel, and audit together", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) const credential = await createCredential(context.prisma, prototype.id) @@ -1074,7 +1195,9 @@ describe("createArtifactPublishingService() database behavior", () => { credential.authorization, prototype.id, "audit-failure", - publicationInput(prototype.id, artifact("audit-failure")), + publicationInput(prototype.id, artifact("audit-failure"), { + promoteChannel: "current", + }), ), ).rejects.toThrow("audit unavailable") } finally { @@ -1090,6 +1213,9 @@ describe("createArtifactPublishingService() database behavior", () => { expect( await context.prisma.artifactPublication.count({ where: { prototypeId: prototype.id } }), ).toBe(0) + expect( + await context.prisma.prototypeChannel.count({ where: { prototypeId: prototype.id } }), + ).toBe(0) expect( await context.prisma.securityAudit.count({ where: { prototypeId: prototype.id, event: "prototype.version.publish" }, diff --git a/hub/server/domains/authentication/create-runtime/index.test.ts b/hub/server/domains/authentication/create-runtime/index.test.ts index 1fdf2b99..6e508317 100644 --- a/hub/server/domains/authentication/create-runtime/index.test.ts +++ b/hub/server/domains/authentication/create-runtime/index.test.ts @@ -5,7 +5,9 @@ import { describe, expect, it, vi } from "vitest" const { createLocalArtifactStorage } = vi.hoisted(() => ({ createLocalArtifactStorage: vi.fn((root: string) => { - void root + if (!path.isAbsolute(root)) { + throw new Error("HUB_ARTIFACT_STORAGE_PATH must be an absolute path.") + } return { put: vi.fn(), read: vi.fn(), @@ -22,15 +24,28 @@ vi.mock("#server/integrations/prisma", () => ({ prisma: {} })) import { createAuthenticationRuntime } from "#server/domains/authentication/create-runtime" describe("createAuthenticationRuntime()", () => { - it("keeps default artifact storage outside the checkout", async () => { - await createAuthenticationRuntime({}) - - expect(createLocalArtifactStorage).toHaveBeenCalledWith( - path.join(os.homedir(), ".replica", "artifacts"), - ) - const storagePath = createLocalArtifactStorage.mock.calls[0]?.[0] - expect(storagePath).toBeTypeOf("string") - if (!storagePath) throw new Error("Expected an artifact storage path") - expect(path.relative(process.cwd(), storagePath)).toMatch(/^\.\./) + it.each([undefined, "", " \t "])( + "keeps unset artifact storage outside the checkout for %j", + async (configuredPath) => { + await createAuthenticationRuntime({ + ...(configuredPath === undefined ? {} : { HUB_ARTIFACT_STORAGE_PATH: configuredPath }), + }) + + expect(createLocalArtifactStorage).toHaveBeenCalledWith( + path.join(os.homedir(), ".replica", "artifacts"), + ) + const storagePath = createLocalArtifactStorage.mock.calls[0]?.[0] + expect(storagePath).toBeTypeOf("string") + if (!storagePath) throw new Error("Expected an artifact storage path") + expect(path.relative(process.cwd(), storagePath)).toMatch(/^\.\./) + }, + ) + + it("passes a configured relative storage path unchanged to the storage adapter", async () => { + await expect( + createAuthenticationRuntime({ HUB_ARTIFACT_STORAGE_PATH: "relative/artifacts" }), + ).rejects.toThrow("HUB_ARTIFACT_STORAGE_PATH must be an absolute path.") + + expect(createLocalArtifactStorage).toHaveBeenCalledWith("relative/artifacts") }) }) diff --git a/hub/server/domains/authentication/create-runtime/index.ts b/hub/server/domains/authentication/create-runtime/index.ts index 3a61da70..123ee064 100644 --- a/hub/server/domains/authentication/create-runtime/index.ts +++ b/hub/server/domains/authentication/create-runtime/index.ts @@ -94,9 +94,9 @@ export async function createAuthenticationRuntime(environment: NodeJS.ProcessEnv admissionPolicy: admissionPolicy.name, trustedProxyHops, } - const artifactStoragePath = path.resolve( - environment.HUB_ARTIFACT_STORAGE_PATH ?? path.join(os.homedir(), ".replica", "artifacts"), - ) + const configuredArtifactStoragePath = environment.HUB_ARTIFACT_STORAGE_PATH?.trim() + const artifactStoragePath = + configuredArtifactStoragePath || path.join(os.homedir(), ".replica", "artifacts") return { administration: createAdministrationService({ store }), artifactPublishing: createArtifactPublishingService({ diff --git a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts index 9489304c..c0aac1fc 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts @@ -13,13 +13,14 @@ async function database(baseline?: PrototypeBaselinePolicy) { vi.stubGlobal("prisma", undefined) const { prisma } = await import("../../../integrations/prisma") onTestFinished(() => prisma.$disconnect()) + const store = createIdentityStore(prisma) return { prisma, - store: createIdentityStore(prisma), + store, service: createPrototypeAuthorizationService({ baseline, database: prisma, - store: createIdentityStore(prisma), + store, }), } } @@ -109,6 +110,20 @@ function deferred() { return { promise, resolve: resolvePromise } } +async function completesWithin(promise: Promise, milliseconds: number) { + let timer: ReturnType | undefined + try { + return await Promise.race([ + promise.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), milliseconds) + }), + ]) + } finally { + if (timer) clearTimeout(timer) + } +} + async function holdWorkspaceLock(database: PrismaClient, workspaceId: string) { const released = deferred() const locked = deferred() @@ -982,6 +997,46 @@ describe("createPrototypeAuthorizationService() database behavior", () => { expect(await prisma.workloadPublisher.count({ where: { prototypeId: prototype.id } })).toBe(0) }) + it.each(["viewer", "editor"] as const)( + "rejects a fresh %s provisioning a publisher without writing publisher state or an audit", + async (role) => { + const { prisma, service, store } = await database() + const administratorToken = `publisher-${role}-administrator-token` + const { prototype } = await createAdministeredPrototype({ + service, + store, + subject: `publisher-${role}-administrator`, + token: administratorToken, + }) + const actor = await createActor(store, { + subject: `publisher-${role}`, + token: `publisher-${role}-token`, + }) + await prisma.prototypeAccessGrant.create({ + data: { + prototypeId: prototype.id, + userId: actor.user.id, + effect: "allow", + role, + }, + }) + + await expect( + service.createPublisherCredential(`publisher-${role}-token`, prototype.id, { + name: `${role} publisher`, + actions: ["publish"], + }), + ).rejects.toMatchObject({ status: 403, code: "prototype_access_denied" }) + expect(await prisma.workloadPublisher.count({ where: { prototypeId: prototype.id } })).toBe(0) + expect(await prisma.workloadCredential.count()).toBe(0) + expect( + await prisma.securityAudit.count({ + where: { prototypeId: prototype.id, event: "prototype.publisher.create" }, + }), + ).toBe(0) + }, + ) + it("lists bounded publisher and credential metadata without reusable secrets", async () => { const { prisma, service, store } = await database() const token = "publisher-list-token" @@ -1029,6 +1084,76 @@ describe("createPrototypeAuthorizationService() database behavior", () => { expect(serialized).not.toContain("tokenDigest") }) + it.each(["stale-administrator", "fresh-editor"] as const)( + "rejects publisher metadata listing for a %s", + async (access) => { + const { prisma, service, store } = await database() + const administratorToken = `publisher-list-${access}-administrator-token` + const { prototype } = await createAdministeredPrototype({ + service, + store, + subject: `publisher-list-${access}-administrator`, + token: administratorToken, + }) + const token = + access === "stale-administrator" ? administratorToken : `publisher-list-${access}-token` + if (access === "stale-administrator") { + await prisma.session.updateMany({ + where: { tokenDigest: digestSecret(token) }, + data: { freshAuthorizedAt: new Date(Date.now() - 5 * 60_000) }, + }) + } else { + const actor = await createActor(store, { + subject: `publisher-list-${access}`, + token, + }) + await prisma.prototypeAccessGrant.create({ + data: { + prototypeId: prototype.id, + userId: actor.user.id, + effect: "allow", + role: "editor", + }, + }) + } + + await expect(service.listPublishers(token, prototype.id, {})).rejects.toMatchObject({ + status: 403, + code: + access === "stale-administrator" + ? "fresh_authorization_required" + : "prototype_access_denied", + }) + }, + ) + + it("lists publisher metadata without the identity-store or prototype write locks", async () => { + const { prisma, service, store } = await database() + const token = "publisher-list-lock-free-token" + const { prototype } = await createAdministeredPrototype({ + service, + store, + subject: "publisher-list-lock-free", + token, + }) + const provisioned = (await service.createPublisherCredential(token, prototype.id, { + name: "Lock-free publisher", + actions: ["publish"], + })) as { publisher: { id: string } } + const identityStoreLock = vi.spyOn(store, "lock") + const prototypeLock = await holdPrototypeLock(prisma, prototype.id) + const listing = service.listPublishers(token, prototype.id, {}) + + const completedWhileLocked = await completesWithin(listing, 1_000) + prototypeLock.release() + await prototypeLock.holder + const page = (await listing) as { items: { id: string }[] } + + expect(completedWhileLocked).toBe(true) + expect(page.items).toEqual([expect.objectContaining({ id: provisioned.publisher.id })]) + expect(identityStoreLock).not.toHaveBeenCalled() + }) + it("does not apply publisher or credential lifecycle operations across prototypes", async () => { const { prisma, service, store } = await database() const token = "publisher-scope-token" diff --git a/hub/server/domains/prototype-authorization/create-service/index.ts b/hub/server/domains/prototype-authorization/create-service/index.ts index e9b26dae..eed51dba 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.ts @@ -1008,43 +1008,35 @@ export function createPrototypeAuthorizationService({ async listPublishers(sessionToken, prototypeId, query) { const { cursor, limit } = parsePage(query) - return store.lock(async (_records, transaction) => { - const actor = await currentSession(store, sessionToken, transaction) - requireFreshAuthorization(actor) - requireIdentifier(prototypeId, "prototype_not_found", "The prototype was not found.") - await lockPrototype(transaction, prototypeId) - const { decision } = await prototypeDecision( - baseline, - transaction, - actor.userId, - prototypeId, - ) - requireDecision(decision, "administrator") - const items = await transaction.workloadPublisher.findMany({ - where: { prototypeId }, - orderBy: { id: "asc" }, - ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), - take: limit + 1, - select: { - id: true, - name: true, - actions: true, - active: true, - createdAt: true, - updatedAt: true, - credentials: { - orderBy: { createdAt: "asc" }, - select: { - id: true, - expiresAt: true, - revokedAt: true, - createdAt: true, - }, + const actor = await currentSession(store, sessionToken) + requireFreshAuthorization(actor) + requireIdentifier(prototypeId, "prototype_not_found", "The prototype was not found.") + const { decision } = await prototypeDecision(baseline, database, actor.userId, prototypeId) + requireDecision(decision, "administrator") + const items = await database.workloadPublisher.findMany({ + where: { prototypeId }, + orderBy: { id: "asc" }, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + take: limit + 1, + select: { + id: true, + name: true, + actions: true, + active: true, + createdAt: true, + updatedAt: true, + credentials: { + orderBy: { createdAt: "asc" }, + select: { + id: true, + expiresAt: true, + revokedAt: true, + createdAt: true, }, }, - }) - return pageResult(items, limit) + }, }) + return pageResult(items, limit) }, async setPublisherActive(sessionToken, prototypeId, publisherId, input) { diff --git a/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts b/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts index d0273e2a..43884abf 100644 --- a/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts +++ b/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts @@ -33,6 +33,16 @@ function bytesInShard(shard: string) { } } +function directoryChainLength(directory: string) { + let length = 1 + let current = directory + while (path.dirname(current) !== current) { + length += 1 + current = path.dirname(current) + } + return length +} + async function expectSymlinkRejected({ root, key, @@ -117,6 +127,72 @@ describe("createLocalArtifactStorage()", () => { expect(Buffer.from(await storage.read(results[0].key))).toEqual(bytes) }) + it("syncs the destination directory chain before new and deduplicated puts return", async () => { + const root = storageRoot() + const storage = createLocalArtifactStorage(root) + const bytes = Buffer.from("durable artifact") + const contentDigest = digest(bytes) + const location = artifactPath(root, contentDigest) + const expectedDirectorySyncs = directoryChainLength(location.shard) + const probePath = path.join(process.cwd(), `.artifact-sync-probe-${randomUUID()}`) + const probe = await open(probePath, "w") + onTestFinished(async () => { + await probe.close() + await rm(probePath, { force: true }) + }) + const fileHandlePrototype = Object.getPrototypeOf(probe) as typeof probe + let directorySyncs = 0 + vi.spyOn(fileHandlePrototype, "sync").mockImplementation(async function (this: typeof probe) { + if ((await this.stat()).isDirectory()) directorySyncs += 1 + }) + + await expect(storage.put(contentDigest, bytes)).resolves.toMatchObject({ created: true }) + expect(directorySyncs).toBe(expectedDirectorySyncs) + + await expect(storage.put(contentDigest, bytes)).resolves.toMatchObject({ created: false }) + expect(directorySyncs).toBe(expectedDirectorySyncs * 2) + }) + + it("syncs the destination directory chain after a concurrent EEXIST dedupe", async () => { + const root = storageRoot() + const storage = createLocalArtifactStorage(root) + const bytes = Buffer.from("concurrent durable artifact") + const contentDigest = digest(bytes) + const expectedDirectorySyncs = directoryChainLength(artifactPath(root, contentDigest).shard) + const probePath = path.join(process.cwd(), `.artifact-sync-probe-${randomUUID()}`) + const probe = await open(probePath, "w") + onTestFinished(async () => { + await probe.close() + await rm(probePath, { force: true }) + }) + const fileHandlePrototype = Object.getPrototypeOf(probe) as typeof probe + let regularFileSyncs = 0 + let directorySyncs = 0 + let releaseRegularFileSyncs!: () => void + const bothTemporaryFilesSynced = new Promise((resolve) => { + releaseRegularFileSyncs = resolve + }) + vi.spyOn(fileHandlePrototype, "sync").mockImplementation(async function (this: typeof probe) { + const status = await this.stat() + if (status.isDirectory()) { + directorySyncs += 1 + } else { + regularFileSyncs += 1 + if (regularFileSyncs === 2) releaseRegularFileSyncs() + await bothTemporaryFilesSynced + } + }) + + const results = await Promise.all([ + storage.put(contentDigest, bytes), + storage.put(contentDigest, bytes), + ]) + + expect(results.filter(({ created }) => created)).toHaveLength(1) + expect(results.filter(({ created }) => !created)).toHaveLength(1) + expect(directorySyncs).toBe(expectedDirectorySyncs * 2) + }) + it.each(["writeFile", "sync"] as const)( "removes its temporary file when %s fails", async (operation) => { diff --git a/hub/server/integrations/artifact-storage/create-local-storage/index.ts b/hub/server/integrations/artifact-storage/create-local-storage/index.ts index d2827ca6..d4c1854e 100644 --- a/hub/server/integrations/artifact-storage/create-local-storage/index.ts +++ b/hub/server/integrations/artifact-storage/create-local-storage/index.ts @@ -130,6 +130,21 @@ async function verifyExisting( } } +async function syncDirectoryChain(directory: string): Promise { + let current = directory + while (true) { + const handle = await open(current, constants.O_RDONLY | constants.O_DIRECTORY) + try { + await handle.sync() + } finally { + await handle.close() + } + const parent = path.dirname(current) + if (parent === current) return + current = parent + } +} + export function createLocalArtifactStorage(root: string): ArtifactStorage { if (!path.isAbsolute(root)) { throw new Error("HUB_ARTIFACT_STORAGE_PATH must be an absolute path.") @@ -143,6 +158,7 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { const { destination, resolvedRoot } = await resolveArtifactPath(root, key, true) try { await verifyExisting(resolvedRoot, destination, contentDigest) + await syncDirectoryChain(path.dirname(destination)) return { key, created: false } } catch (error) { if (!isFileSystemError(error, "ENOENT")) throw error @@ -167,10 +183,12 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { throw new Error("Artifact storage path changed during the write.") } await link(temporary, destination) + await syncDirectoryChain(path.dirname(destination)) return { key, created: true } } catch (error) { if (!isFileSystemError(error, "EEXIST")) throw error await verifyExisting(resolvedRoot, destination, contentDigest) + await syncDirectoryChain(path.dirname(destination)) return { key, created: false } } } finally { diff --git a/hub/server/integrations/express/prepare-publishing-request/index.ts b/hub/server/integrations/express/prepare-publishing-request/index.ts new file mode 100644 index 00000000..a9e46676 --- /dev/null +++ b/hub/server/integrations/express/prepare-publishing-request/index.ts @@ -0,0 +1,11 @@ +import type { NextFunction, Request, Response } from "express" + +export function preparePublishingRequest( + _request: Request, + response: Response, + next: NextFunction, +): void { + response.set("Cache-Control", "private, no-store") + response.set("Referrer-Policy", "no-referrer") + next() +} diff --git a/hub/server/routes/publishing.test.ts b/hub/server/routes/publishing.test.ts index 58fd5437..eccca88d 100644 --- a/hub/server/routes/publishing.test.ts +++ b/hub/server/routes/publishing.test.ts @@ -55,6 +55,11 @@ async function serve(authentication: AuthenticationRoutes, logger = createLogger return `http://127.0.0.1:${port}` } +function expectPublishingPrivacyHeaders(response: Response) { + expect(response.headers.get("cache-control")).toBe("private, no-store") + expect(response.headers.get("referrer-policy")).toBe("no-referrer") +} + describe("publishing routes", () => { it("passes workload authorization and idempotency to publication without browser state", async () => { const publish = vi.fn().mockResolvedValue({ @@ -82,6 +87,7 @@ describe("publishing routes", () => { }) expect(response.status).toBe(201) + expectPublishingPrivacyHeaders(response) expect(await response.json()).toEqual({ version: { id: "version-id" }, replayed: false, @@ -226,6 +232,7 @@ describe("publishing routes", () => { }) expect(response.status).toBe(413) + expectPublishingPrivacyHeaders(response) expect(await response.json()).toEqual({ error: "Invalid request" }) expect(publish).not.toHaveBeenCalled() await vi.waitFor(() => expect(records).toHaveLength(1)) @@ -258,6 +265,7 @@ describe("publishing routes", () => { }) expect(rejected.status).toBe(429) + expectPublishingPrivacyHeaders(rejected) expect(await rejected.json()).toEqual({ error: "rate_limited", message: "Too many publishing requests.", From 48aa5f3a29bfeec232cb68e9bc8ceb504ff90311 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Thu, 29 Oct 2026 05:17:20 -0700 Subject: [PATCH 08/13] test(hub): align artifact tests with final parent Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../create-service/index.db.test.ts | 7 +++++-- hub/server/routes/product/index.test.ts | 7 ------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts index c0aac1fc..a6825fba 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts @@ -588,7 +588,7 @@ describe("createPrototypeAuthorizationService() database behavior", () => { }, ) - it.each([{ action: "archive" }, { action: "create a version" }] as const)( + it.each([{ action: "archive" }, { action: "create a publisher" }] as const)( "rejects a commenter attempting to $action without changing product state or audits", async ({ action }) => { const { prisma, service, store } = await database() @@ -621,7 +621,10 @@ describe("createPrototypeAuthorizationService() database behavior", () => { const operation = action === "archive" ? service.archivePrototype(token, prototype.id) - : service.createVersion(token, prototype.id, { label: "Rejected version" }) + : service.createPublisherCredential(token, prototype.id, { + name: "Rejected publisher", + actions: ["publish"], + }) await expect(operation).rejects.toMatchObject({ status: 403, diff --git a/hub/server/routes/product/index.test.ts b/hub/server/routes/product/index.test.ts index 31f92a63..1c1babd7 100644 --- a/hub/server/routes/product/index.test.ts +++ b/hub/server/routes/product/index.test.ts @@ -172,13 +172,6 @@ describe("product routes", () => { call: "listVersions", expectedArguments: [sessionToken, "prototype-456", { cursor: "version-cursor", limit: "33" }], }, - { - method: "POST", - path: "/api/prototypes/prototype-456/versions", - call: "createVersion", - body: { label: "Mapped version" }, - expectedArguments: [sessionToken, "prototype-456", { label: "Mapped version" }], - }, { method: "GET", path: "/api/prototypes/prototype-456/versions/version-012", From 0d28ecc2f2d19fbac48d1eb979a02d83ee00acf6 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Thu, 29 Oct 2026 07:04:44 -0700 Subject: [PATCH 09/13] fix(hub): close artifact credential and storage gaps Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- hub/docs/artifact-ingestion.md | 2 +- hub/docs/testing-and-operations.md | 2 +- .../create-service/index.db.test.ts | 231 +++++++++++++++++- .../create-service/index.ts | 10 +- .../create-local-storage/index.test.ts | 98 +++++--- .../create-local-storage/index.ts | 20 +- 6 files changed, 309 insertions(+), 54 deletions(-) diff --git a/hub/docs/artifact-ingestion.md b/hub/docs/artifact-ingestion.md index 97b3826c..f781dea9 100644 --- a/hub/docs/artifact-ingestion.md +++ b/hub/docs/artifact-ingestion.md @@ -57,7 +57,7 @@ Publishing requests are rate-limited before JSON parsing. The version-upload rou `ArtifactStorage` separates ingestion from byte storage. The reference adapter writes content-addressed files under `HUB_ARTIFACT_STORAGE_PATH`, defaulting to `~/.replica/artifacts` outside the source checkout and Vite development root. Database rows store only the storage key and validated metadata, never the artifact bytes. -The local adapter is suitable for one Hub process with a durable mounted filesystem. It rejects relative configured paths and symbolic-link roots, shards, and artifact paths, resolves directories under the configured root, opens artifact files without following a final symbolic link, and syncs file and directory entries before reporting a successful write. Operators must keep the storage directory writable only by the Hub workload; the local adapter does not provide replication, remote durability, encryption-key management, hostile concurrent filesystem isolation, or multi-region consistency. A production object-store adapter must preserve the same content-addressed, immutable `put`, `read`, and `remove` boundary. +The local adapter is suitable for one Hub process with a durable mounted filesystem. It rejects relative configured paths and symbolic-link roots, shards, and artifact paths, normalizes root spellings before validation, resolves directories under the configured root, opens artifact files without following a final symbolic link, and syncs file and directory entries through the storage-root boundary before reporting a successful write. Operators must keep the storage directory writable only by the Hub workload; the local adapter does not provide replication, remote durability, encryption-key management, hostile concurrent filesystem isolation, or multi-region consistency. A production object-store adapter must preserve the same content-addressed, immutable `put`, `read`, and `remove` boundary. Storage happens before the database transaction. A failed transaction can therefore leave an unreferenced content-addressed file. Hub never deletes referenced bytes automatically. Operators may remove only files that have no matching `PrototypeVersion.storageKey`; retain unreferenced files long enough to exclude in-flight transactions and backups. Version metadata has no deletion API and must not be rewritten silently. diff --git a/hub/docs/testing-and-operations.md b/hub/docs/testing-and-operations.md index 2e780e97..f0b518f1 100644 --- a/hub/docs/testing-and-operations.md +++ b/hub/docs/testing-and-operations.md @@ -57,7 +57,7 @@ The database command uses `--passWithNoTests` so an empty selection exits succes A dedicated migration case applies the foundation migration, seeds retained lifecycle and security records, then applies the next migration. This catches upgrade behavior that an empty final schema cannot prove, including audit-outcome backfills and lifecycle constraints. -Prototype authorization cases cover direct-rule precedence, explicit deny, no-access defaults, GitHub baseline mapping and staleness, prototype-specific restrictions, cross-prototype and cross-repository isolation, installation membership versus product authority, stale sessions, deterministic overlapping administrator changes, membership lifecycle conflicts, audit rollback, route enforcement, scalar list responses, deny-aware and terminal pagination, version ownership, workload credential revocation, publisher suspension, provisioning authority, lock-free metadata reads, archive lifecycle, and explainable decisions. Artifact publishing cases cover manifest bounds, credential-free provenance URLs, digest and size verification, scoped workload actions, token non-storage, idempotent replay and conflicts, status isolation, concurrent sequence assignment, immutable rows, atomic publish-and-promotion, promotion rollback and replay, revocation and disable races, cross-prototype isolation, audit rollback, body limits, request-log omission, and the local non-GitHub producer contract. Local storage cases reject relative and symbolic-link paths and cover durable directory synchronization. +Prototype authorization cases cover direct-rule precedence, explicit deny, no-access defaults, GitHub baseline mapping and staleness, prototype-specific restrictions, cross-prototype and cross-repository isolation, installation membership versus product authority, stale sessions, deterministic overlapping administrator changes, membership lifecycle conflicts, audit rollback, route enforcement, scalar list responses, deny-aware and terminal pagination, version ownership, workload credential revocation, publisher suspension, provisioning authority, lock-free metadata reads, archive lifecycle, and explainable decisions. Artifact publishing cases cover manifest bounds, credential-free provenance URLs, digest and size verification, credential expiry and forgery, scoped workload actions, token non-storage, idempotent replay and conflicts, status isolation, concurrent sequence assignment, immutable rows, atomic publish-and-promotion, promotion rollback and replay, revocation and disable races, cross-prototype isolation, audit rollback, body limits, request-log omission, and the local non-GitHub producer contract. Local storage cases reject normalized relative and symbolic-link paths and cover durable directory synchronization without requiring access above the configured root. GitHub synchronization cases use mocked GitHub API responses and real database transactions. They cover transactional cache revalidation, invitation precedence during source outage, immutable-ID rename and 404 invalidation, token-404 installation removal, credential-independent webhook revocation, retryable source-generation changes, per-result confirmation ordering, active-identity filtering, bounded provider requests and apply batches, partial scans, dirty-event retry coalescing, first-enrollment and admission-failure auditing, repository and installation lifecycle, provider-response versus apply failures, scheduled failure logging, stale per-user and overlapping reconciliation or sign-in, manual suspension and audited administration preservation, compatibility membership isolation, synchronized session freshness, and rollback. diff --git a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts index e84f4867..79f88f23 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts @@ -134,6 +134,20 @@ async function holdPublisherAuthorizationRow( return { holder, release: released.resolve } } +async function holdPrototypeRow(database: PrismaClient, prototypeId: string) { + const released = deferred() + const locked = deferred() + const holder = database.$transaction(async (transaction) => { + await transaction.$queryRaw` + SELECT "id" FROM "prototypes" WHERE "id" = ${prototypeId}::uuid FOR UPDATE + ` + locked.resolve() + await released.promise + }) + await locked.promise + return { holder, release: released.resolve } +} + async function createPrototype(prisma: PrismaClient) { const user = await prisma.user.create({ data: { displayName: "Publisher owner" } }) const workspace = await prisma.workspace.create({ @@ -196,7 +210,7 @@ async function createAdministratorSession(prisma: PrismaClient, token: string) { return { actor, store } } -async function provisionPublisher(prisma: PrismaClient, sessionToken: string) { +async function createPublisherAdministrator(prisma: PrismaClient, sessionToken: string) { const { actor, store } = await createAdministratorSession(prisma, sessionToken) const workspace = await prisma.workspace.create({ data: { @@ -219,6 +233,11 @@ async function provisionPublisher(prisma: PrismaClient, sessionToken: string) { }, }) const authorization = createPrototypeAuthorizationService({ database: prisma, store }) + return { authorization, prototype } +} + +async function provisionPublisher(prisma: PrismaClient, sessionToken: string) { + const { authorization, prototype } = await createPublisherAdministrator(prisma, sessionToken) const provisioned = (await authorization.createPublisherCredential(sessionToken, prototype.id, { name: "CI publisher", actions: ["publish", "read-status"], @@ -437,6 +456,23 @@ describe("createArtifactPublishingService() database behavior", () => { ).resolves.toMatchObject({ replayed: false }) }) + it("reports a correctly shaped artifact over the decoded size limit as too large", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const publishedArtifact = artifactBytes(Buffer.alloc(maximumArtifactBytes + 1, 0x61)) + + await expect( + context.service.publish( + credential.authorization, + prototype.id, + "oversized-artifact", + publicationInput(prototype.id, publishedArtifact), + ), + ).rejects.toMatchObject({ status: 413, code: "artifact_too_large" }) + expect(await context.prisma.prototypeVersion.count()).toBe(0) + }) + it("rejects noncanonical base64 as an invalid artifact instead of an oversized artifact", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) @@ -554,6 +590,31 @@ describe("createArtifactPublishingService() database behavior", () => { expect(JSON.stringify(saved)).not.toContain(provisioned.credential.token) }) + it("rejects provisioning a workload credential that is already expired", async () => { + const context = await database() + const sessionToken = "expired-provisioning-session" + const { authorization, prototype } = await createPublisherAdministrator( + context.prisma, + sessionToken, + ) + + await expect( + authorization.createPublisherCredential(sessionToken, prototype.id, { + name: "Expired publisher", + actions: ["publish", "read-status"], + expiresAt: new Date(Date.now() - 1_000).toISOString(), + }), + ).rejects.toMatchObject({ status: 422, code: "invalid_workload_publisher" }) + expect( + await context.prisma.workloadPublisher.count({ where: { prototypeId: prototype.id } }), + ).toBe(0) + expect( + await context.prisma.securityAudit.count({ + where: { prototypeId: prototype.id, event: "prototype.publisher.create" }, + }), + ).toBe(0) + }) + it("accepts and persists an ordinary HTTPS provenance source URL", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) @@ -754,6 +815,92 @@ describe("createArtifactPublishingService() database behavior", () => { }, ) + it.each(["publish", "promote"] as const)( + "rejects an already-started %s when its credential expires while waiting for the prototype lock", + async (operation) => { + const context = await database() + const sessionToken = `expiring-${operation}-session` + const { prototype, provisioned, workloadAuthorization } = await provisionPublisher( + context.prisma, + sessionToken, + ) + const published = + operation === "promote" + ? await context.service.publish( + workloadAuthorization, + prototype.id, + "expiration-promotion-source", + publicationInput(prototype.id, artifact("expiration-promotion-source")), + ) + : null + const expiresAt = new Date(Date.now() + 60_000) + await context.prisma.workloadCredential.update({ + where: { id: provisioned.credential.id }, + data: { expiresAt }, + }) + const before = { + publications: await context.prisma.artifactPublication.count({ + where: { prototypeId: prototype.id }, + }), + versions: await context.prisma.prototypeVersion.count({ + where: { prototypeId: prototype.id }, + }), + channels: await context.prisma.prototypeChannel.findMany({ + where: { prototypeId: prototype.id }, + select: { name: true, prototypeVersionId: true }, + }), + workloadAudits: await context.prisma.securityAudit.count({ + where: { + prototypeId: prototype.id, + event: { in: ["prototype.version.publish", "prototype.channel.promote"] }, + }, + }), + } + const prototypeLock = await holdPrototypeRow(context.prisma, prototype.id) + onTestFinished(prototypeLock.release) + const workloadResult = + operation === "publish" + ? context.service.publish( + workloadAuthorization, + prototype.id, + "expired-while-blocked", + publicationInput(prototype.id, artifact("expired-while-blocked")), + ) + : context.service.promote(workloadAuthorization, prototype.id, "current", { + versionId: publishedVersion(published!).id, + }) + await waitForBlockedDatabaseQuery(context.prisma, "%prototypes%FOR UPDATE%") + + vi.useFakeTimers({ toFake: ["Date"] }) + vi.setSystemTime(expiresAt.getTime() + 1) + prototypeLock.release() + await prototypeLock.holder + + await expect(workloadResult).rejects.toMatchObject({ + status: 403, + code: "publisher_access_denied", + }) + expect({ + publications: await context.prisma.artifactPublication.count({ + where: { prototypeId: prototype.id }, + }), + versions: await context.prisma.prototypeVersion.count({ + where: { prototypeId: prototype.id }, + }), + channels: await context.prisma.prototypeChannel.findMany({ + where: { prototypeId: prototype.id }, + select: { name: true, prototypeVersionId: true }, + }), + workloadAudits: await context.prisma.securityAudit.count({ + where: { + prototypeId: prototype.id, + event: { in: ["prototype.version.publish", "prototype.channel.promote"] }, + }, + }), + }).toEqual(before) + }, + ) + it("enforces publish and read-status action scopes independently", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) @@ -781,6 +928,88 @@ describe("createArtifactPublishingService() database behavior", () => { ).rejects.toMatchObject({ status: 403, code: "publisher_access_denied" }) }) + it("rejects status reads after the workload credential expires", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + await context.service.publish( + credential.authorization, + prototype.id, + "expired-status", + publicationInput(prototype.id, artifact("expired-status")), + ) + await context.prisma.workloadCredential.update({ + where: { id: credential.credentialId }, + data: { expiresAt: new Date(Date.now() - 1_000) }, + }) + + await expect( + context.service.getStatus(credential.authorization, prototype.id, "expired-status"), + ).rejects.toMatchObject({ status: 403, code: "publisher_access_denied" }) + }) + + it("rejects a canonical wrong secret for an existing credential before storage or state changes", async () => { + const put = vi.fn((contentDigest: string) => + Promise.resolve({ + key: contentDigest.replace(":", "/"), + created: true, + }), + ) + const storage: ArtifactStorage = { + put, + read: vi.fn(), + remove: vi.fn(), + } + const context = await database(storage) + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + await context.service.publish( + credential.authorization, + prototype.id, + "wrong-secret-status", + publicationInput(prototype.id, artifact("wrong-secret-status")), + ) + put.mockClear() + const wrongToken = `${["rpl", "pub"].join("_")}_${credential.credentialId}.${"w".repeat(43)}` + const wrongAuthorization = `Bearer ${wrongToken}` + const before = { + publications: await context.prisma.artifactPublication.count({ + where: { prototypeId: prototype.id }, + }), + versions: await context.prisma.prototypeVersion.count({ + where: { prototypeId: prototype.id }, + }), + workloadAudits: await context.prisma.securityAudit.count({ + where: { prototypeId: prototype.id, event: "prototype.version.publish" }, + }), + } + + await expect( + context.service.getStatus(wrongAuthorization, prototype.id, "wrong-secret-status"), + ).rejects.toMatchObject({ status: 403, code: "publisher_access_denied" }) + await expect( + context.service.publish( + wrongAuthorization, + prototype.id, + "wrong-secret-publish", + publicationInput(prototype.id, artifact("wrong-secret-publish")), + ), + ).rejects.toMatchObject({ status: 403, code: "publisher_access_denied" }) + + expect(put).not.toHaveBeenCalled() + expect({ + publications: await context.prisma.artifactPublication.count({ + where: { prototypeId: prototype.id }, + }), + versions: await context.prisma.prototypeVersion.count({ + where: { prototypeId: prototype.id }, + }), + workloadAudits: await context.prisma.securityAudit.count({ + where: { prototypeId: prototype.id, event: "prototype.version.publish" }, + }), + }).toEqual(before) + }) + it("rejects a credential issued for a different prototype", async () => { const context = await database() const first = await createPrototype(context.prisma) diff --git a/hub/server/domains/artifact-publishing/create-service/index.ts b/hub/server/domains/artifact-publishing/create-service/index.ts index c0ae530c..af3e1cfa 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.ts @@ -38,7 +38,7 @@ const manifestSchema = z format: z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/), version: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,31}$/), digest: z.string().regex(/^sha256:[a-f0-9]{64}$/), - sizeBytes: z.number().int().min(0).max(maximumArtifactBytes), + sizeBytes: z.number().int().min(0), builtAt: z.iso.datetime({ offset: true }), }) .strict(), @@ -57,7 +57,7 @@ const manifestSchema = z const publishSchema = z .object({ manifest: manifestSchema, - artifactBase64: z.string().max(Math.ceil((maximumArtifactBytes * 4) / 3) + 4), + artifactBase64: z.string(), promoteChannel: z.enum(["current", "preview"]).optional(), }) .strict() @@ -244,6 +244,12 @@ export function createArtifactPublishingService({ if (!parsed.success || parsed.data.manifest.prototypeId !== prototypeId) { error(422, "invalid_artifact_manifest", "The artifact manifest is invalid.") } + if ( + parsed.data.manifest.artifact.sizeBytes > maximumArtifactBytes || + parsed.data.artifactBase64.length > Math.ceil((maximumArtifactBytes * 4) / 3) + 4 + ) { + error(413, "artifact_too_large", "The artifact exceeds the maximum size.") + } const keyDigest = digestSecret(parseIdempotencyKey(idempotencyKey)) const bytes = parseArtifactBase64(parsed.data.artifactBase64) if (bytes.length !== parsed.data.manifest.artifact.sizeBytes) { diff --git a/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts b/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts index 43884abf..1b369a50 100644 --- a/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts +++ b/hub/server/integrations/artifact-storage/create-local-storage/index.test.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto" -import { mkdir, open, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises" +import { mkdir, open, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises" import path from "node:path" import { describe, expect, it, onTestFinished, vi } from "vitest" @@ -33,14 +33,9 @@ function bytesInShard(shard: string) { } } -function directoryChainLength(directory: string) { - let length = 1 - let current = directory - while (path.dirname(current) !== current) { - length += 1 - current = path.dirname(current) - } - return length +async function directoryIdentity(directory: string) { + const status = await stat(directory) + return `${status.dev}:${status.ino}` } async function expectSymlinkRejected({ @@ -133,7 +128,6 @@ describe("createLocalArtifactStorage()", () => { const bytes = Buffer.from("durable artifact") const contentDigest = digest(bytes) const location = artifactPath(root, contentDigest) - const expectedDirectorySyncs = directoryChainLength(location.shard) const probePath = path.join(process.cwd(), `.artifact-sync-probe-${randomUUID()}`) const probe = await open(probePath, "w") onTestFinished(async () => { @@ -141,16 +135,25 @@ describe("createLocalArtifactStorage()", () => { await rm(probePath, { force: true }) }) const fileHandlePrototype = Object.getPrototypeOf(probe) as typeof probe - let directorySyncs = 0 + const syncedDirectories: string[] = [] vi.spyOn(fileHandlePrototype, "sync").mockImplementation(async function (this: typeof probe) { - if ((await this.stat()).isDirectory()) directorySyncs += 1 + const status = await this.stat() + if (status.isDirectory()) syncedDirectories.push(`${status.dev}:${status.ino}`) }) await expect(storage.put(contentDigest, bytes)).resolves.toMatchObject({ created: true }) - expect(directorySyncs).toBe(expectedDirectorySyncs) + const expectedDirectories = await Promise.all([ + directoryIdentity(location.shard), + directoryIdentity(location.algorithm), + directoryIdentity(root), + ]) + const outsideRoot = await directoryIdentity(path.dirname(root)) + expect(syncedDirectories).toEqual(expectedDirectories) + expect(syncedDirectories).not.toContain(outsideRoot) await expect(storage.put(contentDigest, bytes)).resolves.toMatchObject({ created: false }) - expect(directorySyncs).toBe(expectedDirectorySyncs * 2) + expect(syncedDirectories).toEqual([...expectedDirectories, ...expectedDirectories]) + expect(syncedDirectories).not.toContain(outsideRoot) }) it("syncs the destination directory chain after a concurrent EEXIST dedupe", async () => { @@ -158,7 +161,7 @@ describe("createLocalArtifactStorage()", () => { const storage = createLocalArtifactStorage(root) const bytes = Buffer.from("concurrent durable artifact") const contentDigest = digest(bytes) - const expectedDirectorySyncs = directoryChainLength(artifactPath(root, contentDigest).shard) + const location = artifactPath(root, contentDigest) const probePath = path.join(process.cwd(), `.artifact-sync-probe-${randomUUID()}`) const probe = await open(probePath, "w") onTestFinished(async () => { @@ -167,7 +170,7 @@ describe("createLocalArtifactStorage()", () => { }) const fileHandlePrototype = Object.getPrototypeOf(probe) as typeof probe let regularFileSyncs = 0 - let directorySyncs = 0 + const syncedDirectories: string[] = [] let releaseRegularFileSyncs!: () => void const bothTemporaryFilesSynced = new Promise((resolve) => { releaseRegularFileSyncs = resolve @@ -175,7 +178,7 @@ describe("createLocalArtifactStorage()", () => { vi.spyOn(fileHandlePrototype, "sync").mockImplementation(async function (this: typeof probe) { const status = await this.stat() if (status.isDirectory()) { - directorySyncs += 1 + syncedDirectories.push(`${status.dev}:${status.ino}`) } else { regularFileSyncs += 1 if (regularFileSyncs === 2) releaseRegularFileSyncs() @@ -190,7 +193,15 @@ describe("createLocalArtifactStorage()", () => { expect(results.filter(({ created }) => created)).toHaveLength(1) expect(results.filter(({ created }) => !created)).toHaveLength(1) - expect(directorySyncs).toBe(expectedDirectorySyncs * 2) + const expectedDirectories = await Promise.all([ + directoryIdentity(location.shard), + directoryIdentity(location.algorithm), + directoryIdentity(root), + ]) + expect(syncedDirectories.toSorted()).toEqual( + [...expectedDirectories, ...expectedDirectories].toSorted(), + ) + expect(syncedDirectories).not.toContain(await directoryIdentity(path.dirname(root))) }) it.each(["writeFile", "sync"] as const)( @@ -249,29 +260,36 @@ describe("createLocalArtifactStorage()", () => { readFile.mockRestore() }) - it("rejects a symbolic-link storage root without reading, replaying, or removing outside content", async () => { - const root = storageRoot() - const outside = storageRoot() - const bytes = Buffer.from("outside root sentinel") - const contentDigest = digest(bytes) - const location = artifactPath(outside, contentDigest) - await mkdir(location.shard, { recursive: true }) - await writeFile(location.artifact, bytes) - await symlink(outside, root, "dir") - const writeBytes = Buffer.from("outside write probe") + it.each([ + { name: "directly", suffix: "" }, + { name: "with a trailing separator", suffix: path.sep }, + { name: "with a trailing current-directory segment", suffix: `${path.sep}.` }, + ])( + "rejects a symbolic-link storage root $name without accessing outside content", + async ({ suffix }) => { + const root = storageRoot() + const outside = storageRoot() + const bytes = Buffer.from("outside root sentinel") + const contentDigest = digest(bytes) + const location = artifactPath(outside, contentDigest) + await mkdir(location.shard, { recursive: true }) + await writeFile(location.artifact, bytes) + await symlink(outside, root, "dir") + const writeBytes = Buffer.from("outside write probe") - await expectSymlinkRejected({ - root, - key: location.key, - contentDigest, - bytes, - sentinel: location.artifact, - writeProbe: { - bytes: writeBytes, - outsidePath: artifactPath(outside, digest(writeBytes)).artifact, - }, - }) - }) + await expectSymlinkRejected({ + root: `${root}${suffix}`, + key: location.key, + contentDigest, + bytes, + sentinel: location.artifact, + writeProbe: { + bytes: writeBytes, + outsidePath: artifactPath(outside, digest(writeBytes)).artifact, + }, + }) + }, + ) it.each(["sha256", "shard"] as const)( "rejects a symbolic-link %s directory without reading, replaying, or removing outside content", diff --git a/hub/server/integrations/artifact-storage/create-local-storage/index.ts b/hub/server/integrations/artifact-storage/create-local-storage/index.ts index d4c1854e..d10933c3 100644 --- a/hub/server/integrations/artifact-storage/create-local-storage/index.ts +++ b/hub/server/integrations/artifact-storage/create-local-storage/index.ts @@ -130,7 +130,7 @@ async function verifyExisting( } } -async function syncDirectoryChain(directory: string): Promise { +async function syncDirectoryChain(directory: string, root: string): Promise { let current = directory while (true) { const handle = await open(current, constants.O_RDONLY | constants.O_DIRECTORY) @@ -139,8 +139,9 @@ async function syncDirectoryChain(directory: string): Promise { } finally { await handle.close() } + if (current === root) return const parent = path.dirname(current) - if (parent === current) return + if (!isWithinRoot(root, parent)) return current = parent } } @@ -149,16 +150,17 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { if (!path.isAbsolute(root)) { throw new Error("HUB_ARTIFACT_STORAGE_PATH must be an absolute path.") } + const normalizedRoot = path.resolve(root) return { async put(contentDigest: string, bytes: Uint8Array): Promise { const key = artifactKey(contentDigest) if (calculateContentDigest(bytes) !== contentDigest) { throw new Error("Artifact bytes do not match the content digest.") } - const { destination, resolvedRoot } = await resolveArtifactPath(root, key, true) + const { destination, resolvedRoot } = await resolveArtifactPath(normalizedRoot, key, true) try { await verifyExisting(resolvedRoot, destination, contentDigest) - await syncDirectoryChain(path.dirname(destination)) + await syncDirectoryChain(path.dirname(destination), resolvedRoot) return { key, created: false } } catch (error) { if (!isFileSystemError(error, "ENOENT")) throw error @@ -178,17 +180,17 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { await file.close() } try { - const resolved = await resolveArtifactPath(root, key, false) + const resolved = await resolveArtifactPath(normalizedRoot, key, false) if (resolved.destination !== destination) { throw new Error("Artifact storage path changed during the write.") } await link(temporary, destination) - await syncDirectoryChain(path.dirname(destination)) + await syncDirectoryChain(path.dirname(destination), resolvedRoot) return { key, created: true } } catch (error) { if (!isFileSystemError(error, "EEXIST")) throw error await verifyExisting(resolvedRoot, destination, contentDigest) - await syncDirectoryChain(path.dirname(destination)) + await syncDirectoryChain(path.dirname(destination), resolvedRoot) return { key, created: false } } } finally { @@ -197,7 +199,7 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { }, async read(key: string): Promise { - const { destination, resolvedRoot } = await resolveArtifactPath(root, key, false) + const { destination, resolvedRoot } = await resolveArtifactPath(normalizedRoot, key, false) const file = await openArtifact(resolvedRoot, destination) try { return await file.readFile() @@ -208,7 +210,7 @@ export function createLocalArtifactStorage(root: string): ArtifactStorage { async remove(key: string): Promise { try { - const { destination, resolvedRoot } = await resolveArtifactPath(root, key, false) + const { destination, resolvedRoot } = await resolveArtifactPath(normalizedRoot, key, false) await validateArtifactPath(resolvedRoot, destination) await unlink(destination) } catch (error) { From 80f3afe863be469700f8bb367814115ae51f6961 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Thu, 29 Oct 2026 08:18:06 -0700 Subject: [PATCH 10/13] fix(hub): normalize artifact publishing inputs Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- hub/docs/artifact-ingestion.md | 4 +- hub/script/publish-artifact.mjs | 4 +- hub/script/tests/publish-artifact.test.mjs | 70 ++++++++++++++ .../create-service/index.db.test.ts | 91 +++++++++++++++++++ .../create-service/index.ts | 76 ++++++++++------ 5 files changed, 215 insertions(+), 30 deletions(-) create mode 100644 hub/script/tests/publish-artifact.test.mjs diff --git a/hub/docs/artifact-ingestion.md b/hub/docs/artifact-ingestion.md index f781dea9..09a58d6f 100644 --- a/hub/docs/artifact-ingestion.md +++ b/hub/docs/artifact-ingestion.md @@ -49,7 +49,7 @@ Archiving a prototype blocks new publishers, publication, publisher re-enablemen } ``` -The manifest and its nested objects are closed to unknown fields. Hub bounds every string, accepts only HTTP or HTTPS source URLs without embedded usernames, passwords, query strings, or fragments, limits decoded artifacts to 8 MiB in this reference layer, verifies canonical base64, checks the decoded size, and recalculates the SHA-256 digest before storing bytes. The manifest prototype ID must match the route. +The manifest and its nested objects are closed to unknown fields. Hub bounds every string, rejects NUL characters, accepts only HTTP or HTTPS source URLs without embedded usernames, passwords, query strings, or fragments, limits decoded artifacts to 8 MiB in this reference layer, verifies canonical base64, checks the decoded size, and recalculates the SHA-256 digest before storing bytes. Valid UUIDs are normalized before scope and idempotency checks, so hexadecimal casing does not change identity. The manifest prototype ID must match the route. Publishing requests are rate-limited before JSON parsing. The version-upload route alone accepts the bounded upload-sized JSON body; status and promotion retain the API's default JSON limit. @@ -87,4 +87,4 @@ REPLICA_PUBLISH_TOKEN=... npm run publish:artifact -- \ http://127.0.0.1:3001 PROTOTYPE_ID ./prototype.tar preview ``` -Optional `REPLICA_PROVENANCE_PROVIDER`, `REPLICA_SOURCE_URL`, `REPLICA_REVISION`, `REPLICA_REF`, `REPLICA_PIPELINE`, and `REPLICA_RUN_ID` values populate provenance without changing authorization. +The CLI prints its generated or configured idempotency key before sending the request. If the response is lost, retry with that value in `REPLICA_IDEMPOTENCY_KEY` to recover the committed result. Optional `REPLICA_PROVENANCE_PROVIDER`, `REPLICA_SOURCE_URL`, `REPLICA_REVISION`, `REPLICA_REF`, `REPLICA_PIPELINE`, and `REPLICA_RUN_ID` values populate provenance without changing authorization. diff --git a/hub/script/publish-artifact.mjs b/hub/script/publish-artifact.mjs index 05d66d35..85c64017 100644 --- a/hub/script/publish-artifact.mjs +++ b/hub/script/publish-artifact.mjs @@ -24,6 +24,8 @@ if ( const bytes = await readFile(artifactPath) const details = await stat(artifactPath) const digest = `sha256:${createHash("sha256").update(bytes).digest("hex")}` + const idempotencyKey = process.env.REPLICA_IDEMPOTENCY_KEY ?? randomUUID() + console.error(`Replica idempotency key: ${idempotencyKey}`) const response = await fetch( new URL(`/api/publishing/prototypes/${prototypeId}/versions`, hubUrl), { @@ -31,7 +33,7 @@ if ( headers: { authorization: `Bearer ${token}`, "content-type": "application/json", - "idempotency-key": process.env.REPLICA_IDEMPOTENCY_KEY ?? randomUUID(), + "idempotency-key": idempotencyKey, }, body: JSON.stringify({ manifest: { diff --git a/hub/script/tests/publish-artifact.test.mjs b/hub/script/tests/publish-artifact.test.mjs new file mode 100644 index 00000000..42221cb4 --- /dev/null +++ b/hub/script/tests/publish-artifact.test.mjs @@ -0,0 +1,70 @@ +import assert from "node:assert/strict" +import { spawn } from "node:child_process" +import { createServer } from "node:http" +import { once } from "node:events" +import { fileURLToPath } from "node:url" +import { it } from "node:test" + +const script = fileURLToPath(new URL("../publish-artifact.mjs", import.meta.url)) +const artifact = fileURLToPath(new URL("../../package.json", import.meta.url)) +const prototypeId = "50d2e31f-8aa1-4b67-8fb4-67f045b89a0d" + +function runPublisher(hubUrl, idempotencyKey) { + const child = spawn(process.execPath, [script, hubUrl, prototypeId, artifact], { + env: { + ...process.env, + REPLICA_PUBLISH_TOKEN: "test-token", + ...(idempotencyKey ? { REPLICA_IDEMPOTENCY_KEY: idempotencyKey } : {}), + }, + }) + let stdout = "" + let stderr = "" + child.stdout.setEncoding("utf8").on("data", (chunk) => { + stdout += chunk + }) + child.stderr.setEncoding("utf8").on("data", (chunk) => { + stderr += chunk + }) + return once(child, "close").then(([code, signal]) => ({ code, signal, stdout, stderr })) +} + +it("prints the idempotency key before a failed publish so it can be reused", async (t) => { + const receivedKeys = [] + let requestCount = 0 + const server = createServer((request, response) => { + receivedKeys.push(request.headers["idempotency-key"]) + request.resume() + request.on("end", () => { + requestCount += 1 + if (requestCount === 1) { + response.writeHead(503, { "content-type": "application/json" }) + response.end('{"error":"response lost"}') + } else { + response.writeHead(201, { "content-type": "application/json" }) + response.end('{"replayed":true}') + } + }) + }) + server.listen(0, "127.0.0.1") + await once(server, "listening") + t.after(() => server.close()) + const address = server.address() + assert.ok(address && typeof address === "object") + const hubUrl = `http://127.0.0.1:${address.port}` + + const failed = await runPublisher(hubUrl) + const generatedKey = failed.stderr.match( + /Replica idempotency key: ([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})/i, + )?.[1] + assert.equal(failed.code, 1) + assert.equal(failed.signal, null) + assert.ok(generatedKey) + assert.match(failed.stderr, /Publish failed \(503\)/) + + const retried = await runPublisher(hubUrl, generatedKey) + assert.equal(retried.code, 0) + assert.equal(retried.signal, null) + assert.equal(retried.stderr, `Replica idempotency key: ${generatedKey}\n`) + assert.equal(retried.stdout, '{"replayed":true}\n') + assert.deepEqual(receivedKeys, [generatedKey, generatedKey]) +}) diff --git a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts index 79f88f23..ca68cfc0 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts @@ -440,6 +440,41 @@ describe("createArtifactPublishingService() database behavior", () => { expect(await context.prisma.prototypeVersion.count()).toBe(0) }) + it.each(["provider", "sourceUrl", "revision", "ref", "pipeline", "runId"] as const)( + "rejects a NUL in provenance $field before storage or state changes", + async (field) => { + const put = vi.fn(() => Promise.resolve({ key: "unused", created: true })) + const storage: ArtifactStorage = { + put, + read: () => Promise.reject(new Error("Artifact not found")), + remove: () => Promise.resolve(), + } + const context = await database(storage) + const { prototype } = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const input = publicationInput(prototype.id, artifact(`nul-${field}`)) + input.manifest.provenance = { + ...input.manifest.provenance, + [field]: field === "sourceUrl" ? "https://ci.example/runs/\0hidden" : "visible\0hidden", + } + + await expect( + context.service.publish(credential.authorization, prototype.id, `nul-${field}`, input), + ).rejects.toMatchObject({ + status: 422, + code: "invalid_artifact_manifest", + }) + expect(put).not.toHaveBeenCalled() + await expect( + Promise.all([ + context.prisma.prototypeVersion.count(), + context.prisma.artifactPublication.count(), + context.prisma.securityAudit.count(), + ]), + ).resolves.toEqual([0, 0, 0]) + }, + ) + it("publishes a valid artifact at the decoded size limit", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) @@ -1062,6 +1097,62 @@ describe("createArtifactPublishingService() database behavior", () => { ).toBe(1) }) + it("normalizes uppercase prototype identifiers across publishing operations", async () => { + const context = await database() + const { prototype } = await createPrototype(context.prisma) + const other = await createPrototype(context.prisma) + const credential = await createCredential(context.prisma, prototype.id) + const uppercasePrototypeId = prototype.id.toUpperCase() + const uppercaseInput = publicationInput( + uppercasePrototypeId, + artifact("uppercase-prototype-id"), + ) + + const first = await context.service.publish( + credential.authorization, + uppercasePrototypeId, + "uppercase-request", + uppercaseInput, + ) + const replay = await context.service.publish( + credential.authorization, + prototype.id, + "uppercase-request", + publicationInput(prototype.id, artifact("uppercase-prototype-id")), + ) + const status = await context.service.getStatus( + credential.authorization, + uppercasePrototypeId, + "uppercase-request", + ) + await context.service.promote(credential.authorization, uppercasePrototypeId, "current", { + versionId: publishedVersion(first).id, + }) + + expect(first).toMatchObject({ replayed: false }) + expect(replay).toMatchObject({ replayed: true, version: publishedVersion(first) }) + expect(status).toMatchObject({ version: publishedVersion(first) }) + await expect( + context.prisma.prototypeChannel.findUniqueOrThrow({ + where: { prototypeId_name: { prototypeId: prototype.id, name: "current" } }, + select: { prototypeVersionId: true }, + }), + ).resolves.toEqual({ prototypeVersionId: publishedVersion(first).id }) + await expect( + context.service.publish( + credential.authorization, + other.prototype.id.toUpperCase(), + "uppercase-cross-prototype", + publicationInput(other.prototype.id.toUpperCase(), artifact("isolated-uppercase")), + ), + ).rejects.toMatchObject({ status: 403, code: "publisher_access_denied" }) + expect( + await context.prisma.prototypeVersion.count({ + where: { prototypeId: other.prototype.id }, + }), + ).toBe(0) + }) + it("commits a promoted publication, version, channel, and audit together", async () => { const context = await database() const { prototype } = await createPrototype(context.prisma) diff --git a/hub/server/domains/artifact-publishing/create-service/index.ts b/hub/server/domains/artifact-publishing/create-service/index.ts index af3e1cfa..230b0870 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.ts @@ -13,8 +13,14 @@ import type { ArtifactPublishingService, PrototypeChannelName, PublisherAction } import type { Prisma, PrismaClient } from "../../../generated/prisma/client.ts" -const boundedIdentifier = z.string().trim().min(1).max(255) +const boundedIdentifier = z + .string() + .trim() + .min(1) + .max(255) + .refine((value) => !value.includes("\0")) function isSafeProvenanceUrl(value: string): boolean { + if (value.includes("\0")) return false try { const url = new URL(value) return ( @@ -76,10 +82,12 @@ function error(status: number, code: string, message: string): never { throw new AuthenticationError(status, code, message) } -function parsePrototypeId(prototypeId: string): void { - if (!prototypeIdSchema.safeParse(prototypeId).success) { +function parsePrototypeId(prototypeId: string): string { + const parsed = prototypeIdSchema.safeParse(prototypeId) + if (!parsed.success) { error(404, "prototype_not_found", "The prototype was not found.") } + return parsed.data.toLowerCase() } function parseIdempotencyKey(value: string): string { @@ -167,6 +175,7 @@ export function createArtifactPublishingService({ type PublisherAuthentication = { credentialId: string publisherId: string + prototypeId: string tokenDigest: string } @@ -182,7 +191,7 @@ export function createArtifactPublishingService({ } async function authenticate(authorization: string, prototypeId: string, action: PublisherAction) { - parsePrototypeId(prototypeId) + const normalizedPrototypeId = parsePrototypeId(prototypeId) const match = authorizationSchema.exec(authorization) if (!match?.[1] || !match[2]) { error(401, "publisher_authentication_required", "Publisher authentication failed.") @@ -198,7 +207,7 @@ export function createArtifactPublishingService({ credential.revokedAt || (credential.expiresAt && credential.expiresAt <= new Date()) || !credential.workloadPublisher.active || - credential.workloadPublisher.prototypeId !== prototypeId || + credential.workloadPublisher.prototypeId !== normalizedPrototypeId || !credential.workloadPublisher.actions.includes(action) || !secretsMatch(credential.tokenDigest, tokenDigest) ) { @@ -207,6 +216,7 @@ export function createArtifactPublishingService({ return { credentialId: credential.id, publisherId: credential.workloadPublisher.id, + prototypeId: normalizedPrototypeId, tokenDigest, } } @@ -227,7 +237,7 @@ export function createArtifactPublishingService({ (credential.expiresAt && credential.expiresAt <= new Date()) || !credential.workloadPublisher.active || credential.workloadPublisher.id !== authentication.publisherId || - credential.workloadPublisher.prototypeId !== prototypeId || + credential.workloadPublisher.prototypeId !== authentication.prototypeId || !credential.workloadPublisher.actions.includes(action) || !secretsMatch(credential.tokenDigest, authentication.tokenDigest) ) { @@ -239,9 +249,13 @@ export function createArtifactPublishingService({ return { async publish(authorization, prototypeId, idempotencyKey, input) { const authentication = await authenticate(authorization, prototypeId, "publish") - await requireUnarchivedPrototype(prototypeId) + const normalizedPrototypeId = authentication.prototypeId + await requireUnarchivedPrototype(normalizedPrototypeId) const parsed = publishSchema.safeParse(input) - if (!parsed.success || parsed.data.manifest.prototypeId !== prototypeId) { + if ( + !parsed.success || + parsed.data.manifest.prototypeId.toLowerCase() !== normalizedPrototypeId + ) { error(422, "invalid_artifact_manifest", "The artifact manifest is invalid.") } if ( @@ -260,21 +274,23 @@ export function createArtifactPublishingService({ } const requestDigest = digestSecret( JSON.stringify({ - manifest: parsed.data.manifest, + manifest: { ...parsed.data.manifest, prototypeId: normalizedPrototypeId }, promoteChannel: parsed.data.promoteChannel ?? null, }), ) const stored = await storage.put(parsed.data.manifest.artifact.digest, bytes) return database.$transaction(async (transaction) => { - await lockPrototype(transaction, prototypeId) + await lockPrototype(transaction, normalizedPrototypeId) const publisher = await requireCurrentPublisherAuthorization( transaction, authentication, - prototypeId, + normalizedPrototypeId, "publish", ) - const prototype = await transaction.prototype.findUnique({ where: { id: prototypeId } }) + const prototype = await transaction.prototype.findUnique({ + where: { id: normalizedPrototypeId }, + }) if (!prototype) error(404, "prototype_not_found", "The prototype was not found.") if (prototype.archivedAt) { error(409, "prototype_archived", "Archived prototypes cannot receive versions.") @@ -304,7 +320,7 @@ export function createArtifactPublishingService({ } const duplicate = await transaction.prototypeVersion.findFirst({ where: { - prototypeId, + prototypeId: normalizedPrototypeId, contentDigest: parsed.data.manifest.artifact.digest, }, }) @@ -312,13 +328,13 @@ export function createArtifactPublishingService({ error(409, "artifact_already_published", "This artifact was already published.") } const latest = await transaction.prototypeVersion.findFirst({ - where: { prototypeId }, + where: { prototypeId: normalizedPrototypeId }, orderBy: { sequence: "desc" }, }) const manifest = parsed.data.manifest const version = await transaction.prototypeVersion.create({ data: { - prototypeId, + prototypeId: normalizedPrototypeId, workloadPublisherId: publisher.id, sequence: (latest?.sequence ?? 0) + 1, artifactFormat: manifest.artifact.format, @@ -337,7 +353,7 @@ export function createArtifactPublishingService({ }) await transaction.artifactPublication.create({ data: { - prototypeId, + prototypeId: normalizedPrototypeId, workloadPublisherId: publisher.id, prototypeVersionId: version.id, idempotencyKeyDigest: keyDigest, @@ -347,10 +363,13 @@ export function createArtifactPublishingService({ if (parsed.data.promoteChannel) { await transaction.prototypeChannel.upsert({ where: { - prototypeId_name: { prototypeId, name: parsed.data.promoteChannel }, + prototypeId_name: { + prototypeId: normalizedPrototypeId, + name: parsed.data.promoteChannel, + }, }, create: { - prototypeId, + prototypeId: normalizedPrototypeId, prototypeVersionId: version.id, name: parsed.data.promoteChannel, }, @@ -363,7 +382,7 @@ export function createArtifactPublishingService({ sessionId: null, authenticationProviderRevisionId: null, workspaceId: prototype.workspaceId, - prototypeId, + prototypeId: normalizedPrototypeId, prototypeVersionId: version.id, workloadPublisherId: publisher.id, context: { @@ -392,7 +411,7 @@ export function createArtifactPublishingService({ }, include: { prototypeVersion: true }, }) - if (!publication || publication.prototypeId !== prototypeId) { + if (!publication || publication.prototypeId !== authentication.prototypeId) { error(404, "publication_not_found", "The publication was not found.") } return { @@ -403,37 +422,40 @@ export function createArtifactPublishingService({ async promote(authorization, prototypeId, channel, input) { const authentication = await authenticate(authorization, prototypeId, "publish") + const normalizedPrototypeId = authentication.prototypeId const parsedChannel = channelSchema.safeParse(channel) const parsed = promotionSchema.safeParse(input) if (!parsedChannel.success || !parsed.success) { error(422, "invalid_promotion", "The promotion is invalid.") } return database.$transaction(async (transaction) => { - await lockPrototype(transaction, prototypeId) + await lockPrototype(transaction, normalizedPrototypeId) const publisher = await requireCurrentPublisherAuthorization( transaction, authentication, - prototypeId, + normalizedPrototypeId, "publish", ) - const prototype = await transaction.prototype.findUnique({ where: { id: prototypeId } }) + const prototype = await transaction.prototype.findUnique({ + where: { id: normalizedPrototypeId }, + }) if (!prototype) error(404, "prototype_not_found", "The prototype was not found.") if (prototype.archivedAt) { error(409, "prototype_archived", "Archived prototypes cannot be promoted.") } const version = await transaction.prototypeVersion.findFirst({ - where: { id: parsed.data.versionId, prototypeId }, + where: { id: parsed.data.versionId, prototypeId: normalizedPrototypeId }, }) if (!version) error(404, "version_not_found", "The version was not found.") const pointer = await transaction.prototypeChannel.upsert({ where: { prototypeId_name: { - prototypeId, + prototypeId: normalizedPrototypeId, name: parsedChannel.data satisfies PrototypeChannelName, }, }, create: { - prototypeId, + prototypeId: normalizedPrototypeId, prototypeVersionId: version.id, name: parsedChannel.data, }, @@ -445,7 +467,7 @@ export function createArtifactPublishingService({ sessionId: null, authenticationProviderRevisionId: null, workspaceId: prototype.workspaceId, - prototypeId, + prototypeId: normalizedPrototypeId, prototypeVersionId: version.id, workloadPublisherId: publisher.id, context: { From 243c5209cf3535a12e8e7df619136a76dee08fba Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Thu, 29 Oct 2026 09:58:05 -0700 Subject: [PATCH 11/13] fix(hub): integrate latest synchronized access Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../create-service/index.db.test.ts | 50 ++++++++----------- .../create-service/index.db.test.ts | 1 - 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts index ca68cfc0..0300bc57 100644 --- a/hub/server/domains/artifact-publishing/create-service/index.db.test.ts +++ b/hub/server/domains/artifact-publishing/create-service/index.db.test.ts @@ -1,5 +1,4 @@ import { createHash, randomUUID } from "node:crypto" -import { readFile } from "node:fs/promises" import { describe, expect, it, onTestFinished, vi } from "vitest" @@ -1318,35 +1317,26 @@ describe("createArtifactPublishingService() database behavior", () => { expect(results.map(({ replayed }) => replayed)).toEqual([false, false]) expect(versions.map(({ sequence }) => sequence)).toEqual([1, 2]) - const migration = await readFile( - new URL( - "../../../../schema/migrations/20261027230000_artifact_ingestion/migration.sql", - import.meta.url, - ), - "utf8", - ) - const immutableVersionStart = migration.indexOf( - "CREATE FUNCTION reject_prototype_version_mutation()", - ) - expect(immutableVersionStart).toBeGreaterThanOrEqual(0) - const immutableVersionSql = migration.slice(immutableVersionStart) - await context.prisma.$executeRawUnsafe(immutableVersionSql) - try { - await expect( - context.prisma.prototypeVersion.update({ - where: { id: versions[0]!.id }, - data: { label: "rewritten" }, - }), - ).rejects.toThrow(/prototype versions are immutable/i) - await expect( - context.prisma.prototypeVersion.delete({ where: { id: versions[1]!.id } }), - ).rejects.toThrow(/prototype versions are immutable/i) - } finally { - await context.prisma.$executeRawUnsafe( - "DROP TRIGGER prototype_versions_immutable ON prototype_versions", - ) - await context.prisma.$executeRawUnsafe("DROP FUNCTION reject_prototype_version_mutation()") - } + expect( + await context.prisma.$queryRaw<{ exists: boolean }[]>` + SELECT EXISTS ( + SELECT 1 + FROM pg_trigger + WHERE tgname = 'prototype_versions_immutable' + AND tgrelid = 'prototype_versions'::regclass + AND NOT tgisinternal + ) AS exists + `, + ).toEqual([{ exists: true }]) + await expect( + context.prisma.prototypeVersion.update({ + where: { id: versions[0]!.id }, + data: { label: "rewritten" }, + }), + ).rejects.toThrow(/prototype versions are immutable/i) + await expect( + context.prisma.prototypeVersion.delete({ where: { id: versions[1]!.id } }), + ).rejects.toThrow(/prototype versions are immutable/i) }) it("rejects an already-started publication when archive acquires the prototype lock first", async () => { diff --git a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts index a6825fba..3df634e2 100644 --- a/hub/server/domains/prototype-authorization/create-service/index.db.test.ts +++ b/hub/server/domains/prototype-authorization/create-service/index.db.test.ts @@ -3188,5 +3188,4 @@ describe("createPrototypeAuthorizationService() database behavior", () => { nextCursor: null, }) }) - }) From dc71cacdd1fbfa4087967ba11fbd57e22cfc020a Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Thu, 29 Oct 2026 10:16:13 -0700 Subject: [PATCH 12/13] fix(hub): defer retained dirty retries Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../create-service/index.db.test.ts | 223 ++++++++++++++---- .../create-service/index.ts | 30 ++- 2 files changed, 190 insertions(+), 63 deletions(-) diff --git a/hub/server/domains/github-authorization/create-service/index.db.test.ts b/hub/server/domains/github-authorization/create-service/index.db.test.ts index 7bc69011..98fb8611 100644 --- a/hub/server/domains/github-authorization/create-service/index.db.test.ts +++ b/hub/server/domains/github-authorization/create-service/index.db.test.ts @@ -4492,6 +4492,7 @@ describe("createGitHubAuthorizationService() database behavior", () => { it("keeps the dirty retry guard through a bounded batch, retains new IDs, and logs failures", async () => { const { prisma, service } = await database() vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }) + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout") const initialSources = await Promise.all( ["100", "200", "300", "400"].map((githubRepositoryId) => prisma.gitHubRepositorySource.create({ @@ -4507,15 +4508,33 @@ describe("createGitHubAuthorizationService() database behavior", () => { }), ), ) - const initialRepositoryRequests: Array<{ + type PendingRepositoryRequest = { githubRepositoryId: string response: ReturnType> - }> = [] - const initialBatchStarted = deferred() + } + const firstRepositoryRequests: PendingRepositoryRequest[] = [] + const secondRepositoryRequests: PendingRepositoryRequest[] = [] + const thirdRepositoryRequests: string[] = [] + const firstBatchStarted = deferred() + const secondBatchStarted = deferred() + const thirdBatchStarted = deferred() const requestedRepositoryIds: string[] = [] - let firstBatchPending = true + let pendingRepositoryRequests: PendingRepositoryRequest[] | null = firstRepositoryRequests let concurrentRepositoryRequests = 0 let maximumConcurrentRepositoryRequests = 0 + function successfulRepositoryResponse(githubRepositoryId: string) { + return new Response( + JSON.stringify({ + id: Number(githubRepositoryId), + full_name: `owner/repository-${githubRepositoryId}`, + name: `repository-${githubRepositoryId}`, + owner: { login: "owner" }, + default_branch: "main", + visibility: "private", + }), + { status: 200 }, + ) + } const request = vi.fn((input: string | URL | Request) => { const url = requestUrl(input) if (url.endsWith("/access_tokens")) { @@ -4531,37 +4550,71 @@ describe("createGitHubAuthorizationService() database behavior", () => { maximumConcurrentRepositoryRequests, concurrentRepositoryRequests, ) - if (firstBatchPending) { + if (pendingRepositoryRequests) { const response = deferred() - initialRepositoryRequests.push({ githubRepositoryId, response }) - if (initialRepositoryRequests.length === 4) initialBatchStarted.resolve() + const batch = pendingRepositoryRequests + batch.push({ githubRepositoryId, response }) + if (batch.length === 4) { + if (batch === firstRepositoryRequests) firstBatchStarted.resolve() + if (batch === secondRepositoryRequests) secondBatchStarted.resolve() + } return response.promise.finally(() => { concurrentRepositoryRequests -= 1 }) } + thirdRepositoryRequests.push(githubRepositoryId) + thirdBatchStarted.resolve() concurrentRepositoryRequests -= 1 - return Promise.resolve( - new Response( - JSON.stringify({ - id: Number(githubRepositoryId), - full_name: `owner/repository-${githubRepositoryId}`, - name: `repository-${githubRepositoryId}`, - owner: { login: "owner" }, - default_branch: "main", - visibility: "private", - }), - { status: 200 }, - ), - ) + return Promise.resolve(successfulRepositoryResponse(githubRepositoryId)) }) vi.stubGlobal("fetch", request) const errorLog = vi.spyOn(logger, "error").mockImplementation(() => undefined) + async function waitForCurrentSources(githubRepositoryIds: string[]) { + for (let attempt = 0; attempt < 100; attempt += 1) { + const sources = await prisma.gitHubRepositorySource.findMany({ + where: { githubRepositoryId: { in: githubRepositoryIds } }, + select: { synchronizationStatus: true }, + }) + if ( + sources.length === githubRepositoryIds.length && + sources.every(({ synchronizationStatus }) => synchronizationStatus === "current") + ) { + return + } + await new Promise((resolve) => setImmediate(resolve)) + } + throw new Error(`Sources did not become current: ${githubRepositoryIds.join(", ")}`) + } + function scheduledDirtyRetryCount() { + return setTimeoutSpy.mock.calls.filter(([, timeout]) => timeout === 1_000).length + } + async function waitForScheduledBatch( + scheduledRetryCountWhileRunning: number, + startedRequests: readonly unknown[], + ) { + for (let attempt = 0; attempt < 1_000; attempt += 1) { + if ( + scheduledDirtyRetryCount() > scheduledRetryCountWhileRunning || + startedRequests.length > 0 + ) { + return + } + await new Promise((resolve) => setImmediate(resolve)) + } + throw new Error("The next dirty retry batch was not scheduled.") + } await service.processWebhook( webhook({ action: "edited", installation: { id: 7 } }, "membership", "initial-dirty-batch"), ) await vi.advanceTimersByTimeAsync(1_000) - await initialBatchStarted.promise + await firstBatchStarted.promise + const initialGithubRepositoryIds = initialSources.map( + ({ githubRepositoryId }) => githubRepositoryId, + ) + expect(firstRepositoryRequests.map(({ githubRepositoryId }) => githubRepositoryId)).toEqual( + expect.arrayContaining(initialGithubRepositoryIds), + ) const laterSource = await prisma.gitHubRepositorySource.create({ data: { githubRepositoryId: "500", @@ -4591,46 +4644,116 @@ describe("createGitHubAuthorizationService() database behavior", () => { ) await vi.advanceTimersByTimeAsync(10_000) - expect(initialRepositoryRequests).toHaveLength(4) + expect(firstRepositoryRequests).toHaveLength(4) + expect(secondRepositoryRequests).toHaveLength(0) + expect(thirdRepositoryRequests).toHaveLength(0) expect(requestedRepositoryIds).not.toContain("500") + expect(maximumConcurrentRepositoryRequests).toBe(4) - firstBatchPending = false - initialRepositoryRequests.forEach(({ githubRepositoryId, response }, index) => { + const scheduledRetryCountDuringFirstBatch = scheduledDirtyRetryCount() + pendingRepositoryRequests = secondRepositoryRequests + firstRepositoryRequests.forEach(({ githubRepositoryId, response }, index) => { response.resolve( index === 0 ? new Response(null, { status: 503 }) - : new Response( - JSON.stringify({ - id: Number(githubRepositoryId), - full_name: `owner/repository-${githubRepositoryId}`, - name: `repository-${githubRepositoryId}`, - owner: { login: "owner" }, - default_branch: "main", - visibility: "private", - }), - { status: 200 }, - ), + : successfulRepositoryResponse(githubRepositoryId), ) }) - await vi.waitFor(() => - expect(errorLog).toHaveBeenCalledWith( - expect.objectContaining({ reason: "reconciliation_failed" }), - "GitHub authorization dirty retry failed", - ), + const failedGithubRepositoryId = firstRepositoryRequests[0]!.githubRepositoryId + const failedSource = initialSources.find( + ({ githubRepositoryId }) => githubRepositoryId === failedGithubRepositoryId, ) - await vi.advanceTimersByTimeAsync(1_000) - await vi.waitFor(() => expect(requestedRepositoryIds).toContain("500")) + for (let attempt = 0; attempt < 100; attempt += 1) { + const settledSources = await prisma.gitHubRepositorySource.findMany({ + where: { id: { in: initialSources.map(({ id }) => id) } }, + select: { id: true, synchronizationStatus: true }, + }) + if ( + settledSources.length === initialSources.length && + settledSources.every(({ id, synchronizationStatus }) => + id === failedSource!.id + ? synchronizationStatus === "failed" + : synchronizationStatus === "current", + ) + ) { + break + } + await new Promise((resolve) => setImmediate(resolve)) + } + for (let attempt = 0; attempt < 100 && errorLog.mock.calls.length === 0; attempt += 1) { + await new Promise((resolve) => setImmediate(resolve)) + } + expect( + errorLog.mock.calls.some( + ([fields, message]) => + message === "GitHub authorization dirty retry failed" && + fields.repositorySourceId === failedSource!.id && + fields.reason === "reconciliation_failed", + ), + ).toBe(true) + await waitForScheduledBatch(scheduledRetryCountDuringFirstBatch, secondRepositoryRequests) + expect(secondRepositoryRequests).toHaveLength(0) + expect(scheduledDirtyRetryCount()).toBe(scheduledRetryCountDuringFirstBatch + 1) + + await vi.advanceTimersByTimeAsync(999) + expect(secondRepositoryRequests).toHaveLength(0) + await vi.advanceTimersByTimeAsync(1) + await secondBatchStarted.promise + const queuedGithubRepositoryIds = [ + ...initialGithubRepositoryIds, + laterSource.githubRepositoryId, + ] + const secondBatchIds = secondRepositoryRequests.map( + ({ githubRepositoryId }) => githubRepositoryId, + ) + expect(secondBatchIds).toHaveLength(4) + expect(new Set(secondBatchIds).size).toBe(4) + expect(secondBatchIds.every((sourceId) => queuedGithubRepositoryIds.includes(sourceId))).toBe( + true, + ) + const runningSecondBatchSources = await prisma.gitHubRepositorySource.findMany({ + where: { githubRepositoryId: { in: secondBatchIds } }, + select: { synchronizationStatus: true }, + }) + expect(runningSecondBatchSources).toHaveLength(4) + expect( + runningSecondBatchSources.every( + ({ synchronizationStatus }) => synchronizationStatus === "dirty", + ), + ).toBe(true) + expect(thirdRepositoryRequests).toHaveLength(0) expect(maximumConcurrentRepositoryRequests).toBe(4) - expect(requestedRepositoryIds).toContain(laterSource.githubRepositoryId) - await vi.waitFor(async () => - expect( - await prisma.gitHubRepositorySource.findUniqueOrThrow({ - where: { id: laterSource.id }, - }), - ).toMatchObject({ synchronizationStatus: "current" }), + + await vi.advanceTimersByTimeAsync(10_000) + expect(secondRepositoryRequests).toHaveLength(4) + expect(thirdRepositoryRequests).toHaveLength(0) + + const scheduledRetryCountDuringSecondBatch = scheduledDirtyRetryCount() + pendingRepositoryRequests = null + secondRepositoryRequests.forEach(({ githubRepositoryId, response }) => { + response.resolve(successfulRepositoryResponse(githubRepositoryId)) + }) + await waitForCurrentSources(secondBatchIds) + await waitForScheduledBatch(scheduledRetryCountDuringSecondBatch, thirdRepositoryRequests) + expect(thirdRepositoryRequests).toHaveLength(0) + expect(scheduledDirtyRetryCount()).toBe(scheduledRetryCountDuringSecondBatch + 1) + + const thirdBatchIds = queuedGithubRepositoryIds.filter( + (sourceId) => !secondBatchIds.includes(sourceId), ) - expect(initialSources).toHaveLength(4) + expect(thirdBatchIds).toHaveLength(1) + await vi.advanceTimersByTimeAsync(999) + expect(thirdRepositoryRequests).toHaveLength(0) + await vi.advanceTimersByTimeAsync(1) + await thirdBatchStarted.promise + + expect(thirdRepositoryRequests).toEqual(thirdBatchIds) + expect(new Set([...secondBatchIds, ...thirdRepositoryRequests])).toEqual( + new Set(queuedGithubRepositoryIds), + ) + expect(maximumConcurrentRepositoryRequests).toBe(4) + await waitForCurrentSources(queuedGithubRepositoryIds) }) it("keeps the synchronized authorization migration constraints and compatibility backfill", () => { diff --git a/hub/server/domains/github-authorization/create-service/index.ts b/hub/server/domains/github-authorization/create-service/index.ts index 4fbc4dfb..dcde8093 100644 --- a/hub/server/domains/github-authorization/create-service/index.ts +++ b/hub/server/domains/github-authorization/create-service/index.ts @@ -524,19 +524,23 @@ export function createGitHubAuthorizationService({ dirtyRetryTimer = null dirtyRetryRunning = true void (async () => { - while (dirtyRetrySources.size > 0) { - const sourceIds = [...dirtyRetrySources].slice(0, 4) - for (const sourceId of sourceIds) dirtyRetrySources.delete(sourceId) - const results = await Promise.allSettled(sourceIds.map((sourceId) => reconcile(sourceId))) - results.forEach((result, index) => { - if (result.status === "rejected") { - log("error", "GitHub authorization dirty retry failed", { - repositorySourceId: sourceIds[index], - reason: "reconciliation_failed", - }) - } - }) - } + const sourceIds = [...dirtyRetrySources].slice(0, 4) + for (const sourceId of sourceIds) dirtyRetrySources.delete(sourceId) + const results = await Promise.allSettled(sourceIds.map((sourceId) => reconcile(sourceId))) + results.forEach((result, index) => { + if ( + result.status === "rejected" || + result.value?.synchronizationStatus === "failed" + ) { + log("error", "GitHub authorization dirty retry failed", { + repositorySourceId: sourceIds[index], + reason: + result.status === "fulfilled" + ? (result.value?.synchronizationError ?? "reconciliation_failed") + : "reconciliation_failed", + }) + } + }) })().finally(() => { dirtyRetryRunning = false if (dirtyRetrySources.size > 0) scheduleDirtyReconciliation([]) From be0d40384916a2b163e485b4b6dd72bb216fd208 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Thu, 29 Oct 2026 10:34:08 -0700 Subject: [PATCH 13/13] fix(hub): stabilize dirty retry logging Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../github-authorization/create-service/index.db.test.ts | 6 ++++-- .../domains/github-authorization/create-service/index.ts | 5 +---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/hub/server/domains/github-authorization/create-service/index.db.test.ts b/hub/server/domains/github-authorization/create-service/index.db.test.ts index 98fb8611..a46be310 100644 --- a/hub/server/domains/github-authorization/create-service/index.db.test.ts +++ b/hub/server/domains/github-authorization/create-service/index.db.test.ts @@ -4687,8 +4687,10 @@ describe("createGitHubAuthorizationService() database behavior", () => { errorLog.mock.calls.some( ([fields, message]) => message === "GitHub authorization dirty retry failed" && - fields.repositorySourceId === failedSource!.id && - fields.reason === "reconciliation_failed", + typeof fields === "object" && + fields !== null && + Reflect.get(fields, "repositorySourceId") === failedSource!.id && + Reflect.get(fields, "reason") === "reconciliation_failed", ), ).toBe(true) await waitForScheduledBatch(scheduledRetryCountDuringFirstBatch, secondRepositoryRequests) diff --git a/hub/server/domains/github-authorization/create-service/index.ts b/hub/server/domains/github-authorization/create-service/index.ts index dcde8093..be36a3a3 100644 --- a/hub/server/domains/github-authorization/create-service/index.ts +++ b/hub/server/domains/github-authorization/create-service/index.ts @@ -528,10 +528,7 @@ export function createGitHubAuthorizationService({ for (const sourceId of sourceIds) dirtyRetrySources.delete(sourceId) const results = await Promise.allSettled(sourceIds.map((sourceId) => reconcile(sourceId))) results.forEach((result, index) => { - if ( - result.status === "rejected" || - result.value?.synchronizationStatus === "failed" - ) { + if (result.status === "rejected" || result.value?.synchronizationStatus === "failed") { log("error", "GitHub authorization dirty retry failed", { repositorySourceId: sourceIds[index], reason: