Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 100 additions & 23 deletions .github/workflows/deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,12 @@ jobs:
path: backend/console/target/higress-console.jar
retention-days: 7

deploy:
name: Deploy to server
image:
name: Build and push image
needs: build
runs-on: ubuntu-latest
outputs:
image: ${{ steps.meta.outputs.image }}
steps:
- uses: actions/checkout@v5
with:
Expand All @@ -55,32 +57,107 @@ jobs:
name: higress-console-jar
path: release

- name: Prepare release bundle
id: bundle
- name: Image name
id: meta
env:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
run: |
version="$(date -u +%Y%m%d%H%M)-${GITHUB_SHA::7}"
cp deploy/Dockerfile deploy/deploy.sh release/
echo "version=$version" >> "$GITHUB_OUTPUT"
echo "Release: \`$version\`" >> "$GITHUB_STEP_SUMMARY"
image="docker.io/${DOCKER_USERNAME}/higress-console:${version}"
echo "image=$image" >> "$GITHUB_OUTPUT"
echo "Image: \`$image\`" >> "$GITHUB_STEP_SUMMARY"

- name: Configure SSH
- uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

- name: Build and push
env:
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
IMAGE: ${{ steps.meta.outputs.image }}
run: |
install -m 700 -d ~/.ssh
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key
chmod 600 ~/.ssh/deploy_key
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
set -euo pipefail
cp deploy/Dockerfile release/
docker build -t "$IMAGE" release
docker push "$IMAGE"

- name: Upload and deploy
deploy:
name: Deploy to K3s
needs: image
runs-on: ubuntu-latest
env:
# Production runs on the K3s cluster (k3s.seasonschange.cn), namespace higress.
# KUBE_CONFIG is a kubeconfig for the namespace-scoped ServiceAccount higress/ci-deployer.
NAMESPACE: higress
IMAGE: ${{ needs.image.outputs.image }}
PUBLIC_URL: https://mcp.tmlab.top
steps:
- uses: actions/checkout@v5
with:
sparse-checkout: deploy

- name: Configure cluster access
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
VERSION: ${{ steps.bundle.outputs.version }}
KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
run: |
set -euo pipefail
[[ -n "$KUBE_CONFIG" ]] || { echo "::error::Repository secret KUBE_CONFIG is required."; exit 1; }
install -m 700 -d ~/.kube
printf '%s\n' "$KUBE_CONFIG" > ~/.kube/config
chmod 600 ~/.kube/config
kubectl -n "$NAMESPACE" get deployment higress

# The Deployment uses strategy Recreate (one instance owns /data), so make sure the node can
# pull the new image before the old pod is stopped. This also pre-pulls it (~2.3 GB).
- name: Pre-pull image on the node
run: |
set -euo pipefail
kubectl -n "$NAMESPACE" delete pod image-prepull --ignore-not-found --wait=true
kubectl -n "$NAMESPACE" run image-prepull --image="$IMAGE" --restart=Never \
--overrides='{"spec":{"nodeSelector":{"role":"legacy"},"automountServiceAccountToken":false}}' \
--command -- /bin/true
phase=""
for _ in $(seq 1 120); do
phase="$(kubectl -n "$NAMESPACE" get pod image-prepull -o jsonpath='{.status.phase}')"
reason="$(kubectl -n "$NAMESPACE" get pod image-prepull -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}')"
case "$phase/$reason" in
Succeeded/*) echo "image pulled"; break ;;
*/ErrImagePull|*/ImagePullBackOff|*/InvalidImageName|Failed/*)
kubectl -n "$NAMESPACE" describe pod image-prepull | tail -20
kubectl -n "$NAMESPACE" delete pod image-prepull --ignore-not-found
echo "::error::The cluster cannot pull $IMAGE; the running Higress was not touched."
exit 1 ;;
esac
sleep 5
done
kubectl -n "$NAMESPACE" delete pod image-prepull --ignore-not-found
[[ "$phase" == "Succeeded" ]] || { echo "::error::Timed out pulling $IMAGE"; exit 1; }

- name: Deploy
run: |
set -euo pipefail
previous="$(kubectl -n "$NAMESPACE" get deployment higress -o jsonpath='{.spec.template.spec.containers[?(@.name=="higress")].image}')"
echo "Deploying $IMAGE (previous: $previous)"
sed "s|__IMAGE__|$IMAGE|" deploy/k3s/higress.yaml | kubectl -n "$NAMESPACE" apply -f -
if ! kubectl -n "$NAMESPACE" rollout status deployment/higress --timeout=10m; then
kubectl -n "$NAMESPACE" get pods -o wide || true
kubectl -n "$NAMESPACE" logs deployment/higress -c higress --tail=80 || true
echo "::warning::Rolling back to $previous"
kubectl -n "$NAMESPACE" rollout undo deployment/higress
kubectl -n "$NAMESPACE" rollout status deployment/higress --timeout=10m || true
echo "::error::Release failed health checks and was rolled back."
exit 1
fi

- name: Check public gateway
run: |
target="$DEPLOY_USER@$DEPLOY_HOST"
dir="/opt/higress/releases/$VERSION"
ssh -i ~/.ssh/deploy_key "$target" "mkdir -p '$dir'"
scp -i ~/.ssh/deploy_key release/higress-console.jar release/Dockerfile release/deploy.sh "$target:$dir/"
ssh -i ~/.ssh/deploy_key "$target" "bash '$dir/deploy.sh' '$VERSION'"
set -euo pipefail
# MCP servers require a consumer key, so an unauthenticated call must be rejected with 401.
code=""
for _ in $(seq 1 24); do
code="$(curl -s -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' -d '{}' "$PUBLIC_URL/mcp-servers/higress-admin" || true)"
if [[ "$code" == "401" ]]; then echo "gateway healthy ($code)"; exit 0; fi
sleep 5
done
echo "::error::$PUBLIC_URL/mcp-servers/higress-admin returned ${code:-nothing}, expected 401."
exit 1
9 changes: 5 additions & 4 deletions deploy/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# Overlay our console build on top of the pinned Higress all-in-one image that already runs on the server.
# The base image is tagged locally on the server (see setup-server.sh), so upstream "latest" drift never
# changes the gateway/controller versions without an explicit re-pin.
ARG BASE_IMAGE=higress-aio-base:pinned
# Overlay our console build on top of a pinned Higress all-in-one image.
# The base is pinned by digest so upstream "latest" drift never changes the gateway/controller
# versions without an explicit re-pin. To move to a newer Higress, pull the new all-in-one,
# take its digest and update BASE_IMAGE here.
ARG BASE_IMAGE=higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/all-in-one@sha256:55f5fa3ec77d5c00c5622c05019e6deb030b195bb0efa7212eff3555c071bed0
FROM ${BASE_IMAGE}

COPY higress-console.jar /app/higress-console.jar
49 changes: 28 additions & 21 deletions deploy/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Deployment (Time-Machine-Lab fork)

This fork adds static Bearer API tokens to the console and deploys it onto our single Higress all-in-one host.
This fork adds static Bearer API tokens to the console and runs it inside the Higress all-in-one image on our K3s cluster.

## Console API token

Expand All @@ -11,31 +11,38 @@ admin user. Failures are rate limited per client IP (same limits as form/basic l
On the server the plaintext token is `/opt/higress/secrets/console-api-token`; the container only gets its digest
through `/opt/higress/console.env`.

## Layout on the server (`/opt/higress`)
## Where it runs

| Path | Purpose |
Since 2026-11-11 Higress runs on the K3s cluster (`k3s.seasonschange.cn`), namespace `higress`.
The manifest is `k3s/higress.yaml`: one pod with the all-in-one container plus a Redis sidecar
(Higress MCP expects Redis on `127.0.0.1:6379`).

| Object | Purpose |
|---|---|
| `data/` | Higress all-in-one data volume |
| `releases/<version>/` | Uploaded jar + Dockerfile + deploy.sh (last 3 kept) |
| `console.env` | Env file for the container (token digest) |
| `ports.env` | Loopback host ports (`CONSOLE_PORT`, `GATEWAY_PORT`, `GATEWAY_TLS_PORT`; default 8001/8080/8443) |
| `secrets/` | console API token, `ops-admin` / `claude-ops` consumer keys |
| `redis.pass` | Password of the `higress-redis` container used by Higress MCP |
| `setup/` | Copy of this directory used for one-time setup |
| PVC `higress-data` | `/data` of the all-in-one: routes, plugins, MCP servers, consumers (node `agent-1`) |
| Secret `higress-console-env` | `HIGRESS_CONSOLE_ADMIN_API_TOKENS` (sha256 digest of the console API token) |
| Secret `higress-redis` | Redis password (also referenced by the MCP redis config inside `/data`) |
| Service `higress` | `8080` gateway, `8001` console (cluster-internal only) |
| Ingress `mcp` | `mcp.tmlab.top` → gateway |

The plaintext console token and the `ops-admin` / `claude-ops` consumer keys are still kept in
`/opt/higress/secrets/` on 207.57.124.102. To reach the console: `kubectl -n higress port-forward svc/higress 8001`.

## Flow

1. `setup-server.sh` (one time, idempotent): pins the running all-in-one image as `higress-aio-base:pinned`,
generates the console token, installs nginx + certbot and issues the certificate for `$DOMAIN`.
nginx terminates HTTPS and proxies to the gateway on `127.0.0.1:$GATEWAY_PORT`; certbot renews via `certbot.timer`
and reloads nginx through a deploy hook.
2. Push to `main` → `.github/workflows/deploy.yaml` builds the jar, uploads it with `deploy.sh`, which builds
`higress-aio-tml:<version>`, swaps the container, checks console readiness + token auth + gateway, and rolls back
on failure. Manual rollback: `bash /opt/higress/releases/<older-version>/deploy.sh <older-version>`.
1. Push to `main` → `.github/workflows/deploy.yaml` builds the jar, builds `Dockerfile` (the jar on top of the
all-in-one base pinned by digest) and pushes `docker.io/<DOCKER_USERNAME>/higress-console:<version>`.
2. The deploy job pre-pulls the image on the node with a throw-away pod (the Deployment is `Recreate`, so the old
pod is only stopped once the new image is known to be pullable), applies `k3s/higress.yaml` with the new image,
waits for the console readiness probe, and runs `kubectl rollout undo` if it fails. It then checks that
`https://mcp.tmlab.top/mcp-servers/higress-admin` rejects an unauthenticated call with 401.
Manual rollback: `kubectl -n higress rollout undo deployment/higress`.
3. `mcp/setup_higress_admin.py` registers the console itself as the `higress-admin` MCP server
(`https://<domain>/mcp-servers/higress-admin`, consumer `ops-admin`). Re-run after editing
`mcp/higress-admin-openapi.yaml`:
`python3 setup_higress_admin.py higress-admin-openapi.yaml` on the server.
`mcp/higress-admin-openapi.yaml`.

Required secrets: repository secret `KUBE_CONFIG` (kubeconfig of the namespace-scoped ServiceAccount
`higress/ci-deployer`) and the organization secrets `DOCKER_USERNAME` / `DOCKER_PASSWORD`.
To move to a newer Higress all-in-one, update the digest in `Dockerfile` and push.

Required repository secrets: `DEPLOY_HOST`, `DEPLOY_USER`, `DEPLOY_SSH_KEY`, `DEPLOY_KNOWN_HOSTS`.
To move to a newer Higress all-in-one, pull it on the server and re-tag it as `higress-aio-base:pinned`, then redeploy.
`deploy.sh`, `setup-server.sh` and `nginx/` describe the previous single-host Docker setup and are kept for reference.
87 changes: 87 additions & 0 deletions deploy/k3s/higress.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
# Higress all-in-one (gateway + controller + embedded apiserver + this console) on the K3s cluster,
# namespace `higress`. Applied by .github/workflows/deploy.yaml with __IMAGE__ substituted.
#
# Pre-created in the cluster (not in git):
# Secret higress-console-env HIGRESS_CONSOLE_ADMIN_API_TOKENS (sha256 digest of the console API token)
# Secret higress-redis password (also referenced by the MCP redis config in /data)
# /data (routes, plugins, MCP servers, consumers) lives in PVC higress-data on node agent-1.
# Redis runs as a sidecar so Higress MCP still finds it on 127.0.0.1:6379.
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: higress-data, namespace: higress}
spec:
accessModes: [ReadWriteOnce]
storageClassName: local-path
resources: {requests: {storage: 1Gi}}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: higress, namespace: higress}
spec:
replicas: 1
strategy: {type: Recreate}
selector: {matchLabels: {app: higress}}
template:
metadata: {labels: {app: higress}}
spec:
nodeSelector: {role: legacy}
automountServiceAccountToken: false
enableServiceLinks: false
terminationGracePeriodSeconds: 30
containers:
- name: higress
# Replaced by CI with docker.io/<user>/higress-console:<version>.
image: __IMAGE__
imagePullPolicy: IfNotPresent
envFrom: [{secretRef: {name: higress-console-env}}]
ports:
- {name: console, containerPort: 8001}
- {name: http, containerPort: 8080}
- {name: https, containerPort: 8443}
volumeMounts: [{name: data, mountPath: /data}]
resources:
requests: {cpu: 200m, memory: 768Mi}
limits: {memory: 2Gi}
startupProbe:
httpGet: {path: /healthz/ready, port: 8001}
periodSeconds: 5
failureThreshold: 72
readinessProbe:
tcpSocket: {port: 8080}
periodSeconds: 10
livenessProbe:
tcpSocket: {port: 8080}
periodSeconds: 30
failureThreshold: 5
- name: redis
image: docker.io/library/redis:7-alpine
command: ["sh","-c","exec redis-server --bind 127.0.0.1 --port 6379 --requirepass \"$REDIS_PASSWORD\" --maxmemory 64mb --maxmemory-policy allkeys-lru --save ''"]
env:
- name: REDIS_PASSWORD
valueFrom: {secretKeyRef: {name: higress-redis, key: password}}
resources:
requests: {cpu: 10m, memory: 16Mi}
limits: {memory: 128Mi}
volumes:
- name: data
persistentVolumeClaim: {claimName: higress-data}
---
apiVersion: v1
kind: Service
metadata: {name: higress, namespace: higress}
spec:
selector: {app: higress}
ports:
- {name: http, port: 8080, targetPort: 8080}
- {name: console, port: 8001, targetPort: 8001}
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata: {name: mcp, namespace: higress}
spec:
ingressClassName: traefik
rules:
- host: mcp.tmlab.top
http:
paths:
- {path: /, pathType: Prefix, backend: {service: {name: higress, port: {number: 8080}}}}
Loading