From 50715ef519cbf34c8dbd9b0e13b09ec4f1b9f6c4 Mon Sep 17 00:00:00 2001 From: Seasons Change <1204992313@qq.com> Date: Wed, 11 Nov 2026 11:52:12 +0800 Subject: [PATCH] ci: build a console image and deploy it to the K3s cluster Higress moved from the Docker host 207.57.124.102 to the K3s cluster (namespace higress). The workflow no longer uploads the jar over SSH and builds the image on the server: - image job: builds deploy/Dockerfile (console jar on top of the Higress all-in-one, now pinned by digest instead of a server-local tag) and pushes docker.io//higress-console: - deploy job: pre-pulls the image on the node with a throw-away pod so the Recreate rollout never stops the running gateway for an unpullable image, applies deploy/k3s/higress.yaml, waits for readiness, rolls back on failure and checks that mcp.tmlab.top still enforces auth - add deploy/k3s/higress.yaml (the manifest running in the cluster) and update deploy/README.md - needs repository secret KUBE_CONFIG (ServiceAccount higress/ci-deployer); DEPLOY_HOST/USER/SSH_KEY/KNOWN_HOSTS are no longer used Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yaml | 123 +++++++++++++++++++++++++++------- deploy/Dockerfile | 9 +-- deploy/README.md | 49 ++++++++------ deploy/k3s/higress.yaml | 87 ++++++++++++++++++++++++ 4 files changed, 220 insertions(+), 48 deletions(-) create mode 100644 deploy/k3s/higress.yaml diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 4df50df7..3cb45a97 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -41,10 +41,12 @@ jobs: path: backend/console/target/higress-console.jar retention-days: 7 - deploy: - name: Deploy to server + image: + name: Build and push image needs: build runs-on: ubuntu-latest + outputs: + image: ${{ steps.meta.outputs.image }} steps: - uses: actions/checkout@v5 with: @@ -55,32 +57,107 @@ jobs: name: higress-console-jar path: release - - name: Prepare release bundle - id: bundle + - name: Image name + id: meta + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} run: | version="$(date -u +%Y%m%d%H%M)-${GITHUB_SHA::7}" - cp deploy/Dockerfile deploy/deploy.sh release/ - echo "version=$version" >> "$GITHUB_OUTPUT" - echo "Release: \`$version\`" >> "$GITHUB_STEP_SUMMARY" + image="docker.io/${DOCKER_USERNAME}/higress-console:${version}" + echo "image=$image" >> "$GITHUB_OUTPUT" + echo "Image: \`$image\`" >> "$GITHUB_STEP_SUMMARY" - - name: Configure SSH + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push env: - DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + IMAGE: ${{ steps.meta.outputs.image }} run: | - install -m 700 -d ~/.ssh - printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key - chmod 600 ~/.ssh/deploy_key - printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts + set -euo pipefail + cp deploy/Dockerfile release/ + docker build -t "$IMAGE" release + docker push "$IMAGE" - - name: Upload and deploy + deploy: + name: Deploy to K3s + needs: image + runs-on: ubuntu-latest + env: + # Production runs on the K3s cluster (k3s.seasonschange.cn), namespace higress. + # KUBE_CONFIG is a kubeconfig for the namespace-scoped ServiceAccount higress/ci-deployer. + NAMESPACE: higress + IMAGE: ${{ needs.image.outputs.image }} + PUBLIC_URL: https://mcp.tmlab.top + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: deploy + + - name: Configure cluster access env: - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - DEPLOY_USER: ${{ secrets.DEPLOY_USER }} - VERSION: ${{ steps.bundle.outputs.version }} + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + run: | + set -euo pipefail + [[ -n "$KUBE_CONFIG" ]] || { echo "::error::Repository secret KUBE_CONFIG is required."; exit 1; } + install -m 700 -d ~/.kube + printf '%s\n' "$KUBE_CONFIG" > ~/.kube/config + chmod 600 ~/.kube/config + kubectl -n "$NAMESPACE" get deployment higress + + # The Deployment uses strategy Recreate (one instance owns /data), so make sure the node can + # pull the new image before the old pod is stopped. This also pre-pulls it (~2.3 GB). + - name: Pre-pull image on the node + run: | + set -euo pipefail + kubectl -n "$NAMESPACE" delete pod image-prepull --ignore-not-found --wait=true + kubectl -n "$NAMESPACE" run image-prepull --image="$IMAGE" --restart=Never \ + --overrides='{"spec":{"nodeSelector":{"role":"legacy"},"automountServiceAccountToken":false}}' \ + --command -- /bin/true + phase="" + for _ in $(seq 1 120); do + phase="$(kubectl -n "$NAMESPACE" get pod image-prepull -o jsonpath='{.status.phase}')" + reason="$(kubectl -n "$NAMESPACE" get pod image-prepull -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}')" + case "$phase/$reason" in + Succeeded/*) echo "image pulled"; break ;; + */ErrImagePull|*/ImagePullBackOff|*/InvalidImageName|Failed/*) + kubectl -n "$NAMESPACE" describe pod image-prepull | tail -20 + kubectl -n "$NAMESPACE" delete pod image-prepull --ignore-not-found + echo "::error::The cluster cannot pull $IMAGE; the running Higress was not touched." + exit 1 ;; + esac + sleep 5 + done + kubectl -n "$NAMESPACE" delete pod image-prepull --ignore-not-found + [[ "$phase" == "Succeeded" ]] || { echo "::error::Timed out pulling $IMAGE"; exit 1; } + + - name: Deploy + run: | + set -euo pipefail + previous="$(kubectl -n "$NAMESPACE" get deployment higress -o jsonpath='{.spec.template.spec.containers[?(@.name=="higress")].image}')" + echo "Deploying $IMAGE (previous: $previous)" + sed "s|__IMAGE__|$IMAGE|" deploy/k3s/higress.yaml | kubectl -n "$NAMESPACE" apply -f - + if ! kubectl -n "$NAMESPACE" rollout status deployment/higress --timeout=10m; then + kubectl -n "$NAMESPACE" get pods -o wide || true + kubectl -n "$NAMESPACE" logs deployment/higress -c higress --tail=80 || true + echo "::warning::Rolling back to $previous" + kubectl -n "$NAMESPACE" rollout undo deployment/higress + kubectl -n "$NAMESPACE" rollout status deployment/higress --timeout=10m || true + echo "::error::Release failed health checks and was rolled back." + exit 1 + fi + + - name: Check public gateway run: | - target="$DEPLOY_USER@$DEPLOY_HOST" - dir="/opt/higress/releases/$VERSION" - ssh -i ~/.ssh/deploy_key "$target" "mkdir -p '$dir'" - scp -i ~/.ssh/deploy_key release/higress-console.jar release/Dockerfile release/deploy.sh "$target:$dir/" - ssh -i ~/.ssh/deploy_key "$target" "bash '$dir/deploy.sh' '$VERSION'" + set -euo pipefail + # MCP servers require a consumer key, so an unauthenticated call must be rejected with 401. + code="" + for _ in $(seq 1 24); do + code="$(curl -s -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' -d '{}' "$PUBLIC_URL/mcp-servers/higress-admin" || true)" + if [[ "$code" == "401" ]]; then echo "gateway healthy ($code)"; exit 0; fi + sleep 5 + done + echo "::error::$PUBLIC_URL/mcp-servers/higress-admin returned ${code:-nothing}, expected 401." + exit 1 diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 61de0621..9f4aeadc 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,7 +1,8 @@ -# Overlay our console build on top of the pinned Higress all-in-one image that already runs on the server. -# The base image is tagged locally on the server (see setup-server.sh), so upstream "latest" drift never -# changes the gateway/controller versions without an explicit re-pin. -ARG BASE_IMAGE=higress-aio-base:pinned +# Overlay our console build on top of a pinned Higress all-in-one image. +# The base is pinned by digest so upstream "latest" drift never changes the gateway/controller +# versions without an explicit re-pin. To move to a newer Higress, pull the new all-in-one, +# take its digest and update BASE_IMAGE here. +ARG BASE_IMAGE=higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/all-in-one@sha256:55f5fa3ec77d5c00c5622c05019e6deb030b195bb0efa7212eff3555c071bed0 FROM ${BASE_IMAGE} COPY higress-console.jar /app/higress-console.jar diff --git a/deploy/README.md b/deploy/README.md index d04e1dc3..6ea1949a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,6 +1,6 @@ # Deployment (Time-Machine-Lab fork) -This fork adds static Bearer API tokens to the console and deploys it onto our single Higress all-in-one host. +This fork adds static Bearer API tokens to the console and runs it inside the Higress all-in-one image on our K3s cluster. ## Console API token @@ -11,31 +11,38 @@ admin user. Failures are rate limited per client IP (same limits as form/basic l On the server the plaintext token is `/opt/higress/secrets/console-api-token`; the container only gets its digest through `/opt/higress/console.env`. -## Layout on the server (`/opt/higress`) +## Where it runs -| Path | Purpose | +Since 2026-11-11 Higress runs on the K3s cluster (`k3s.seasonschange.cn`), namespace `higress`. +The manifest is `k3s/higress.yaml`: one pod with the all-in-one container plus a Redis sidecar +(Higress MCP expects Redis on `127.0.0.1:6379`). + +| Object | Purpose | |---|---| -| `data/` | Higress all-in-one data volume | -| `releases//` | Uploaded jar + Dockerfile + deploy.sh (last 3 kept) | -| `console.env` | Env file for the container (token digest) | -| `ports.env` | Loopback host ports (`CONSOLE_PORT`, `GATEWAY_PORT`, `GATEWAY_TLS_PORT`; default 8001/8080/8443) | -| `secrets/` | console API token, `ops-admin` / `claude-ops` consumer keys | -| `redis.pass` | Password of the `higress-redis` container used by Higress MCP | -| `setup/` | Copy of this directory used for one-time setup | +| PVC `higress-data` | `/data` of the all-in-one: routes, plugins, MCP servers, consumers (node `agent-1`) | +| Secret `higress-console-env` | `HIGRESS_CONSOLE_ADMIN_API_TOKENS` (sha256 digest of the console API token) | +| Secret `higress-redis` | Redis password (also referenced by the MCP redis config inside `/data`) | +| Service `higress` | `8080` gateway, `8001` console (cluster-internal only) | +| Ingress `mcp` | `mcp.tmlab.top` → gateway | + +The plaintext console token and the `ops-admin` / `claude-ops` consumer keys are still kept in +`/opt/higress/secrets/` on 207.57.124.102. To reach the console: `kubectl -n higress port-forward svc/higress 8001`. ## Flow -1. `setup-server.sh` (one time, idempotent): pins the running all-in-one image as `higress-aio-base:pinned`, - generates the console token, installs nginx + certbot and issues the certificate for `$DOMAIN`. - nginx terminates HTTPS and proxies to the gateway on `127.0.0.1:$GATEWAY_PORT`; certbot renews via `certbot.timer` - and reloads nginx through a deploy hook. -2. Push to `main` → `.github/workflows/deploy.yaml` builds the jar, uploads it with `deploy.sh`, which builds - `higress-aio-tml:`, swaps the container, checks console readiness + token auth + gateway, and rolls back - on failure. Manual rollback: `bash /opt/higress/releases//deploy.sh `. +1. Push to `main` → `.github/workflows/deploy.yaml` builds the jar, builds `Dockerfile` (the jar on top of the + all-in-one base pinned by digest) and pushes `docker.io//higress-console:`. +2. The deploy job pre-pulls the image on the node with a throw-away pod (the Deployment is `Recreate`, so the old + pod is only stopped once the new image is known to be pullable), applies `k3s/higress.yaml` with the new image, + waits for the console readiness probe, and runs `kubectl rollout undo` if it fails. It then checks that + `https://mcp.tmlab.top/mcp-servers/higress-admin` rejects an unauthenticated call with 401. + Manual rollback: `kubectl -n higress rollout undo deployment/higress`. 3. `mcp/setup_higress_admin.py` registers the console itself as the `higress-admin` MCP server (`https:///mcp-servers/higress-admin`, consumer `ops-admin`). Re-run after editing - `mcp/higress-admin-openapi.yaml`: - `python3 setup_higress_admin.py higress-admin-openapi.yaml` on the server. + `mcp/higress-admin-openapi.yaml`. + +Required secrets: repository secret `KUBE_CONFIG` (kubeconfig of the namespace-scoped ServiceAccount +`higress/ci-deployer`) and the organization secrets `DOCKER_USERNAME` / `DOCKER_PASSWORD`. +To move to a newer Higress all-in-one, update the digest in `Dockerfile` and push. -Required repository secrets: `DEPLOY_HOST`, `DEPLOY_USER`, `DEPLOY_SSH_KEY`, `DEPLOY_KNOWN_HOSTS`. -To move to a newer Higress all-in-one, pull it on the server and re-tag it as `higress-aio-base:pinned`, then redeploy. +`deploy.sh`, `setup-server.sh` and `nginx/` describe the previous single-host Docker setup and are kept for reference. diff --git a/deploy/k3s/higress.yaml b/deploy/k3s/higress.yaml new file mode 100644 index 00000000..22f0573b --- /dev/null +++ b/deploy/k3s/higress.yaml @@ -0,0 +1,87 @@ +# Higress all-in-one (gateway + controller + embedded apiserver + this console) on the K3s cluster, +# namespace `higress`. Applied by .github/workflows/deploy.yaml with __IMAGE__ substituted. +# +# Pre-created in the cluster (not in git): +# Secret higress-console-env HIGRESS_CONSOLE_ADMIN_API_TOKENS (sha256 digest of the console API token) +# Secret higress-redis password (also referenced by the MCP redis config in /data) +# /data (routes, plugins, MCP servers, consumers) lives in PVC higress-data on node agent-1. +# Redis runs as a sidecar so Higress MCP still finds it on 127.0.0.1:6379. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: higress-data, namespace: higress} +spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path + resources: {requests: {storage: 1Gi}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: higress, namespace: higress} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: {matchLabels: {app: higress}} + template: + metadata: {labels: {app: higress}} + spec: + nodeSelector: {role: legacy} + automountServiceAccountToken: false + enableServiceLinks: false + terminationGracePeriodSeconds: 30 + containers: + - name: higress + # Replaced by CI with docker.io//higress-console:. + image: __IMAGE__ + imagePullPolicy: IfNotPresent + envFrom: [{secretRef: {name: higress-console-env}}] + ports: + - {name: console, containerPort: 8001} + - {name: http, containerPort: 8080} + - {name: https, containerPort: 8443} + volumeMounts: [{name: data, mountPath: /data}] + resources: + requests: {cpu: 200m, memory: 768Mi} + limits: {memory: 2Gi} + startupProbe: + httpGet: {path: /healthz/ready, port: 8001} + periodSeconds: 5 + failureThreshold: 72 + readinessProbe: + tcpSocket: {port: 8080} + periodSeconds: 10 + livenessProbe: + tcpSocket: {port: 8080} + periodSeconds: 30 + failureThreshold: 5 + - name: redis + image: docker.io/library/redis:7-alpine + command: ["sh","-c","exec redis-server --bind 127.0.0.1 --port 6379 --requirepass \"$REDIS_PASSWORD\" --maxmemory 64mb --maxmemory-policy allkeys-lru --save ''"] + env: + - name: REDIS_PASSWORD + valueFrom: {secretKeyRef: {name: higress-redis, key: password}} + resources: + requests: {cpu: 10m, memory: 16Mi} + limits: {memory: 128Mi} + volumes: + - name: data + persistentVolumeClaim: {claimName: higress-data} +--- +apiVersion: v1 +kind: Service +metadata: {name: higress, namespace: higress} +spec: + selector: {app: higress} + ports: + - {name: http, port: 8080, targetPort: 8080} + - {name: console, port: 8001, targetPort: 8001} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: {name: mcp, namespace: higress} +spec: + ingressClassName: traefik + rules: + - host: mcp.tmlab.top + http: + paths: + - {path: /, pathType: Prefix, backend: {service: {name: higress, port: {number: 8080}}}}