Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,8 @@ jobs:
run: git tag --list '@forestadmin/workflow-executor@*' | sort > /tmp/we-tags-before.txt
- name: Snapshot agent-bff tags (pre-release)
run: git tag --list '@forestadmin/agent-bff@*' | sort > /tmp/bff-tags-before.txt
- name: Snapshot gateway tags (pre-release)
run: git tag --list '@forestadmin/gateway@*' | sort > /tmp/gateway-tags-before.txt
- name: "Run multi-semantic-release"
run: "$(yarn bin)/multi-semantic-release --deps.bump=override"
env:
Expand Down Expand Up @@ -311,6 +313,30 @@ jobs:
echo "agent-bff@${VERSION} released — dispatching docker-publish-bff.yml."
gh workflow run docker-publish-bff.yml --ref "$GITHUB_REF_NAME" -f version="$VERSION"

# Same reasoning as the workflow-executor dispatch above: the release commit
# carries `[skip ci]`, so the tag push cannot build the image itself.
- name: Trigger gateway Docker image publish (if released)
if: ${{ !cancelled() }}
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
run: |
if [ ! -f /tmp/gateway-tags-before.txt ]; then
echo "Pre-release snapshot missing — release step never ran; nothing to publish."
exit 0
fi
git tag --list '@forestadmin/gateway@*' | sort > /tmp/gateway-tags-after.txt
NEW_TAGS=$(comm -13 /tmp/gateway-tags-before.txt /tmp/gateway-tags-after.txt)
if [ -z "$NEW_TAGS" ]; then
echo "No new @forestadmin/gateway tag in this run — nothing to publish."
exit 0
fi
# If more than one appears, pick the highest so :latest never regresses.
VERSION=$(echo "$NEW_TAGS" \
| sed 's|^@forestadmin/gateway@||' \
| sort -V | tail -n1)
echo "gateway@${VERSION} released — dispatching docker-publish-gateway.yml."
gh workflow run docker-publish-gateway.yml --ref "$GITHUB_REF_NAME" -f version="$VERSION"

publish-api-reference:
name: Publish API Reference
runs-on: ubuntu-latest
Expand Down
461 changes: 461 additions & 0 deletions .github/workflows/docker-publish-gateway.yml

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion packages/agent-bff/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
export { default as BFFHttpServer } from './http/bff-http-server';
export { parseConfig, REQUIRED_KEYS } from './config/env-config';
export { CONFIG_KEYS, parseConfig, parsePublicUrl, REQUIRED_KEYS } from './config/env-config';
export type {
BFFConfig,
PresenceMap,
Expand Down
14 changes: 8 additions & 6 deletions packages/agent-bff/src/openapi/render-openapi.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { ConfigLabels } from '../config/env-config';
import type { Logger } from '../ports/logger-port';

import { generateOpenApiDocument, serializeOpenApi } from './openapi-document';
Expand All @@ -18,9 +19,9 @@ function wantsUnfolding(env: NodeJS.ProcessEnv): boolean {
return UNFOLD_VARS.every(name => (env[name] ?? '').trim() !== '');
}

function publishesAiQuery(env: NodeJS.ProcessEnv, logger: Logger): boolean {
function publishesAiQuery(env: NodeJS.ProcessEnv, logger: Logger, labels: ConfigLabels): boolean {
try {
return resolveOAuthConfig(parseConfig(env)) !== undefined;
return resolveOAuthConfig(parseConfig(env, labels)) !== undefined;
} catch (error) {
const reason = extractErrorMessage(error);

Expand All @@ -42,8 +43,9 @@ function publishesAiQuery(env: NodeJS.ProcessEnv, logger: Logger): boolean {
export default async function renderOpenApi(
env: NodeJS.ProcessEnv,
logger: Logger,
options: { basePath?: string } = {},
options: { basePath?: string; labels?: ConfigLabels } = {},
): Promise<string> {
const labels = options.labels ?? {};
const basePath = normalizeBasePath(options.basePath);
const authSecret = env.FOREST_AUTH_SECRET;

Expand All @@ -53,11 +55,11 @@ export default async function renderOpenApi(
// the variables unfolding needs are all present — and from there a bad value is a real failure.
const unfoldable =
authSecret && wantsUnfolding(env)
? { source: resolveUnfoldSource(parseConfig(env), logger), authSecret }
? { source: resolveUnfoldSource(parseConfig(env, labels), logger), authSecret }
: undefined;

const publicUrl = parsePublicUrl(env.BFF_PUBLIC_URL, 'BFF_PUBLIC_URL');
const hasAiQueryRoute = publishesAiQuery(env, logger);
const publicUrl = parsePublicUrl(env.BFF_PUBLIC_URL, labels.BFF_PUBLIC_URL ?? 'BFF_PUBLIC_URL');
const hasAiQueryRoute = publishesAiQuery(env, logger, labels);

if (!unfoldable?.source) {
logger('Warn', `Emitting the generic OpenAPI document: ${NOTHING_TO_UNFOLD}`);
Expand Down
18 changes: 18 additions & 0 deletions packages/agent-bff/test/openapi/openapi-cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,24 @@ describe('renderOpenApi', () => {
/BFF_PUBLIC_URL/,
);
});

it('should name the labelled key of a malformed public url', async () => {
await expect(
renderOpenApi({ BFF_PUBLIC_URL: 'bff.example.com' }, noopLogger, {
labels: { BFF_PUBLIC_URL: 'FOREST_GATEWAY_API_PUBLIC_URL' },
}),
).rejects.toThrow(
'Invalid configuration: FOREST_GATEWAY_API_PUBLIC_URL must be a valid http(s) URL.',
);
});

it('should name the labelled key of a broken setting once unfolding is asked', async () => {
await expect(
renderOpenApi({ ...VALID_ENV, BFF_AGENT_TIMEOUT_MS: 'nope' }, noopLogger, {
labels: { BFF_AGENT_TIMEOUT_MS: 'FOREST_GATEWAY_API_AGENT_TIMEOUT_MS' },
}),
).rejects.toThrow('FOREST_GATEWAY_API_AGENT_TIMEOUT_MS must be an integer');
});
});

describe('dispatchCli', () => {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
export default function normalizeDomainList(domains?: string[]): string[] | undefined {
export default function normalizeDomainList(
domains?: string[],
label = 'allowedOAuthClients',
): string[] | undefined {
if (domains === undefined) return undefined;

const entries = domains.map(domain => domain.trim()).filter(Boolean);

if (entries.length === 0) {
throw new Error(
'Invalid allowedOAuthClients: no domains to allow. List at least one domain ' +
`Invalid ${label}: no domains to allow. List at least one domain ` +
'(e.g. dust.tt), or omit the option to accept any registered client.',
);
}
Expand All @@ -15,7 +18,7 @@ export default function normalizeDomainList(domains?: string[]): string[] | unde
// hostname; '\' acts as a path separator in WHATWG URLs, silently truncating.
if (/[/:@?#\\\s]/.test(entry)) {
throw new Error(
`Invalid allowedOAuthClients entry "${entry}": list bare domains (e.g. dust.tt) ` +
`Invalid ${label} entry "${entry}": list bare domains (e.g. dust.tt) ` +
'without scheme, port, path, or spaces.',
);
}
Expand All @@ -27,7 +30,7 @@ export default function normalizeDomainList(domains?: string[]): string[] | unde
// carry one, so keeping it would silently reject every client for that domain.
return new URL(`https://${entry}`).hostname.replace(/\.$/, '');
} catch {
throw new Error(`Invalid allowedOAuthClients entry "${entry}": not a valid domain.`);
throw new Error(`Invalid ${label} entry "${entry}": not a valid domain.`);
}
});
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,22 @@ describe('normalizeDomainList', () => {
expect(() => normalizeDomainList(['dust.tt#x'])).toThrow(/bare domains/);
expect(() => normalizeDomainList(['dust.tt\\unexpected'])).toThrow(/bare domains/);
});

it('should name the given label on an empty list', () => {
expect(() => normalizeDomainList([], 'FOREST_GATEWAY_ALLOWED_OAUTH_CLIENTS')).toThrow(
'Invalid FOREST_GATEWAY_ALLOWED_OAUTH_CLIENTS: no domains to allow.',
);
});

it('should name the given label on an invalid entry', () => {
expect(() =>
normalizeDomainList(['https://dust.tt'], 'FOREST_MCP_ALLOWED_OAUTH_CLIENTS'),
).toThrow('Invalid FOREST_MCP_ALLOWED_OAUTH_CLIENTS entry "https://dust.tt"');
});

it('should name allowedOAuthClients by default', () => {
expect(() => normalizeDomainList(['https://dust.tt'])).toThrow(
'Invalid allowedOAuthClients entry "https://dust.tt"',
);
});
});
117 changes: 117 additions & 0 deletions packages/gateway/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
# syntax=docker/dockerfile:1
#
# Production image for @forestadmin/gateway: the MCP, the API, or both, on one port.
# The build context MUST be the monorepo root (yarn workspaces + yarn.lock):
#
# docker build -f packages/gateway/Dockerfile -t ghcr.io/forestadmin/gateway:latest .
#
# Configuration is passed entirely via environment variables; FOREST_GATEWAY_SERVICES
# (mcp, api or mcp,api) is required.

# Base image pinned by digest (node:22-bookworm-slim). Bump via Renovate/Dependabot.
FROM node:22-bookworm-slim@sha256:813a7480f28fdadac1f7f5c824bcdad435b5bc1322a5968bbbdef8d058f9dff4 AS base

# ── Stage 1: build ────────────────────────────────────────────────────────────
FROM base AS builder
WORKDIR /app

# Same install as the agent-bff image: manifests + lockfile first so a source-only
# change keeps the install layer cached, --frozen-lockfile so the tree matches
# yarn.lock, --ignore-scripts to skip husky and native builds never loaded here.
COPY --parents package.json yarn.lock packages/*/package.json packages/*/*/package.json ./
RUN yarn install --frozen-lockfile --ignore-scripts

COPY . .

# Build the Gateway and only its 6 workspace dependencies, in topological order.
# agent-bff's `build` also copies the Redoc bundle into dist/docs.
RUN node_modules/.bin/lerna run build \
--scope @forestadmin/gateway \
--include-dependencies

# ── Stage 2: isolated prod deps ───────────────────────────────────────────────
# Only the external runtime deps of the 7 workspace packages, from the committed
# lockfile (see docker/README.md): a dependency change it does not cover fails here.
FROM base AS prod-deps
WORKDIR /deps

COPY packages/gateway/docker/build-deps-manifest.js ./build-deps-manifest.js
COPY packages/gateway/docker/deps/yarn.lock ./yarn.lock
COPY packages/gateway/package.json ./packages/gateway/package.json
COPY packages/agent-bff/package.json ./packages/agent-bff/package.json
COPY packages/agent-client/package.json ./packages/agent-client/package.json
COPY packages/agent-toolkit/package.json ./packages/agent-toolkit/package.json
COPY packages/datasource-toolkit/package.json ./packages/datasource-toolkit/package.json
COPY packages/forestadmin-client/package.json ./packages/forestadmin-client/package.json
COPY packages/mcp-server/package.json ./packages/mcp-server/package.json

RUN node build-deps-manifest.js packages package.json
RUN yarn install --frozen-lockfile --ignore-scripts

# ── Stage 3: runtime ──────────────────────────────────────────────────────────
FROM base AS runtime
WORKDIR /app
ENV NODE_ENV=production

# The digest pin freezes Debian packages too, so security fixes can only land here. APT_EPOCH is
# part of this layer's cache key: CI passes the run id so every run refetches; local builds reuse
# the cached layer until --no-cache.
ARG APT_EPOCH
RUN echo "$APT_EPOCH" \
&& apt-get update -o APT::Update::Error-Mode=any \
&& DEBIAN_FRONTEND=noninteractive apt-get upgrade -y --with-new-pkgs -o Dpkg::Options::=--force-confold \
&& rm -rf /var/lib/apt/lists/*

COPY --from=prod-deps /deps/node_modules ./node_modules

# @forestadmin/* workspace packages placed directly in node_modules, resolved by
# walking up to /app/node_modules/@forestadmin/<pkg>/ and its package.json "main".
COPY --from=builder /app/packages/agent-bff/dist ./node_modules/@forestadmin/agent-bff/dist
COPY --from=builder /app/packages/agent-bff/package.json ./node_modules/@forestadmin/agent-bff/package.json
COPY --from=builder /app/packages/agent-client/dist ./node_modules/@forestadmin/agent-client/dist
COPY --from=builder /app/packages/agent-client/package.json ./node_modules/@forestadmin/agent-client/package.json
COPY --from=builder /app/packages/agent-toolkit/dist ./node_modules/@forestadmin/agent-toolkit/dist
COPY --from=builder /app/packages/agent-toolkit/package.json ./node_modules/@forestadmin/agent-toolkit/package.json
COPY --from=builder /app/packages/datasource-toolkit/dist ./node_modules/@forestadmin/datasource-toolkit/dist
COPY --from=builder /app/packages/datasource-toolkit/package.json ./node_modules/@forestadmin/datasource-toolkit/package.json
COPY --from=builder /app/packages/forestadmin-client/dist ./node_modules/@forestadmin/forestadmin-client/dist
COPY --from=builder /app/packages/forestadmin-client/package.json ./node_modules/@forestadmin/forestadmin-client/package.json
COPY --from=builder /app/packages/mcp-server/dist ./node_modules/@forestadmin/mcp-server/dist
COPY --from=builder /app/packages/mcp-server/package.json ./node_modules/@forestadmin/mcp-server/package.json

# Gateway entry point.
COPY --from=builder /app/packages/gateway/dist ./packages/gateway/dist
COPY --from=builder /app/packages/gateway/package.json ./packages/gateway/package.json

# The working directory, and only it: `openapi --output` defaults to ./openapi.json. Not
# recursive — node_modules and dist stay root-owned, so the process cannot rewrite its code.
RUN chown node:node /app

USER node

# Not `forestadmin-agent-bff`, the default of the preload this image borrows from agent-bff.
# Setting OTEL_SERVICE_NAME at run time still wins.
ENV OTEL_SERVICE_NAME=forest-gateway

# 3931 when the mcp service is on, 3450 with api only; PORT overrides both. A real port only:
# with PORT=0 nothing outside the process learns the port, and the healthcheck fails.
EXPOSE 3450 3931

# The probe resolves the port the way the Gateway does (PORT, then its legacy aliases, then the
# per-service default), so the two never disagree. Only a 200 passes: a degraded service makes
# /health answer 503, and a misconfigured container is meant to read unhealthy.
HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
CMD ["node", "/app/packages/gateway/dist/healthcheck.js"]

# OCI image metadata (https://github.com/opencontainers/image-spec).
LABEL org.opencontainers.image.title="forestadmin-gateway" \
org.opencontainers.image.description="Forest Admin MCP and API services on one port." \
org.opencontainers.image.source="https://github.com/ForestAdmin/agent-nodejs" \
org.opencontainers.image.licenses="GPL-3.0"

# Exec form, no wrapper script: the CLI takes positional subcommands (`openapi`).
#
# The preload is agent-bff's, required from the same path the Gateway imports agent-bff by, so
# the tracing handle it parks is the one the shutdown path flushes. Inert until an OTLP endpoint
# or OTEL_TRACES_EXPORTER is set.
ENTRYPOINT ["node", "--require", "/app/node_modules/@forestadmin/agent-bff/dist/tracing-preload.js", "/app/packages/gateway/dist/cli.js"]
63 changes: 63 additions & 0 deletions packages/gateway/docker/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# Docker image build assets

The production image installs its runtime dependencies into an **isolated**
`node_modules` (rather than shipping the whole monorepo's hoisted tree). This
keeps the image small with a fully pinned dependency tree.

## How it works

- [`build-deps-manifest.js`](./build-deps-manifest.js) merges the external
(non-`@forestadmin`) runtime dependencies of the Gateway and its 6
workspace dependencies, plus the OpenTelemetry packages, into a single `package.json`.
Three OTel packages are named there; `sdk-node` drags in every OTLP exporter
and the Zipkin one behind them, which is what lets the image honour
`OTEL_TRACES_EXPORTER` and `OTEL_EXPORTER_OTLP_PROTOCOL` without the app ever
choosing an exporter.
- [`deps/yarn.lock`](./deps/) pins every transitive version.
- The Docker build regenerates the manifest from the live `package.json` files
and runs `yarn install --frozen-lockfile`. If a workspace dependency changes
in a way the committed lock does not cover, **the build fails** instead of
silently shipping an unpinned version.
- [`check-deps-closure.js`](./check-deps-closure.js) recomputes the real
transitive `@forestadmin` closure of the Gateway and fails when the hardcoded list
or the Dockerfile `COPY` lines drift from it.
- [`smoke-test.sh`](./smoke-test.sh) runs a locally-loaded image before it is
published: CLI surface, module graph, Redoc bundle, boot with `mcp,api` and
an unreachable Forest server, the Docker `HEALTHCHECK` per service set, and a
clean stop on SIGTERM.

## Running the smoke test

```bash
sh packages/gateway/docker/smoke-test.sh <image-ref>
```

It needs `docker`, `curl`, `openssl` and **`python3`** on the host. The last one
serves a two-line stub of the Forest server so the last boot can be fully
configured and reach `/health` 200 — the path the image's own HEALTHCHECK
demands, which nothing else exercises. All four are present on the GitHub
runners; on a slim box python3 is the one likely to be missing.

## Updating the lockfile

Only `deps/yarn.lock` is committed — the manifest is generated on demand (the
Docker build regenerates it too), so there is no stale `package.json` to drift.
Run this whenever a runtime dependency of one of the 7 workspace packages
changes **or when the `OTEL_DEPENDENCIES` versions in `build-deps-manifest.js`
are bumped** (the build will fail with `--frozen-lockfile` until you do):

The generated manifest carries the repo's pinned `packageManager`
(`yarn@1.22.19`), so with Corepack enabled (`corepack enable`) the refresh uses
the same Yarn as the Docker build — a global Yarn 4 would otherwise emit an
incompatible lockfile format.

```bash
# from the monorepo root (Corepack enabled)
TMP=$(mktemp -d)
node packages/gateway/docker/build-deps-manifest.js packages "$TMP/package.json"
( cd "$TMP" && yarn install --ignore-scripts )
cp "$TMP/yarn.lock" packages/gateway/docker/deps/yarn.lock
rm -rf "$TMP"
```

Then commit the updated `deps/yarn.lock`.
Loading
Loading