diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 4df50df7..3cb45a97 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -41,10 +41,12 @@ jobs: path: backend/console/target/higress-console.jar retention-days: 7 - deploy: - name: Deploy to server + image: + name: Build and push image needs: build runs-on: ubuntu-latest + outputs: + image: ${{ steps.meta.outputs.image }} steps: - uses: actions/checkout@v5 with: @@ -55,32 +57,107 @@ jobs: name: higress-console-jar path: release - - name: Prepare release bundle - id: bundle + - name: Image name + id: meta + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} run: | version="$(date -u +%Y%m%d%H%M)-${GITHUB_SHA::7}" - cp deploy/Dockerfile deploy/deploy.sh release/ - echo "version=$version" >> "$GITHUB_OUTPUT" - echo "Release: \`$version\`" >> "$GITHUB_STEP_SUMMARY" + image="docker.io/${DOCKER_USERNAME}/higress-console:${version}" + echo "image=$image" >> "$GITHUB_OUTPUT" + echo "Image: \`$image\`" >> "$GITHUB_STEP_SUMMARY" - - name: Configure SSH + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push env: - DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + IMAGE: ${{ steps.meta.outputs.image }} run: | - install -m 700 -d ~/.ssh - printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key - chmod 600 ~/.ssh/deploy_key - printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts + set -euo pipefail + cp deploy/Dockerfile release/ + docker build -t "$IMAGE" release + docker push "$IMAGE" - - name: Upload and deploy + deploy: + name: Deploy to K3s + needs: image + runs-on: ubuntu-latest + env: + # Production runs on the K3s cluster (k3s.seasonschange.cn), namespace higress. + # KUBE_CONFIG is a kubeconfig for the namespace-scoped ServiceAccount higress/ci-deployer. + NAMESPACE: higress + IMAGE: ${{ needs.image.outputs.image }} + PUBLIC_URL: https://mcp.tmlab.top + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: deploy + + - name: Configure cluster access env: - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - DEPLOY_USER: ${{ secrets.DEPLOY_USER }} - VERSION: ${{ steps.bundle.outputs.version }} + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + run: | + set -euo pipefail + [[ -n "$KUBE_CONFIG" ]] || { echo "::error::Repository secret KUBE_CONFIG is required."; exit 1; } + install -m 700 -d ~/.kube + printf '%s\n' "$KUBE_CONFIG" > ~/.kube/config + chmod 600 ~/.kube/config + kubectl -n "$NAMESPACE" get deployment higress + + # The Deployment uses strategy Recreate (one instance owns /data), so make sure the node can + # pull the new image before the old pod is stopped. This also pre-pulls it (~2.3 GB). + - name: Pre-pull image on the node + run: | + set -euo pipefail + kubectl -n "$NAMESPACE" delete pod image-prepull --ignore-not-found --wait=true + kubectl -n "$NAMESPACE" run image-prepull --image="$IMAGE" --restart=Never \ + --overrides='{"spec":{"nodeSelector":{"role":"legacy"},"automountServiceAccountToken":false}}' \ + --command -- /bin/true + phase="" + for _ in $(seq 1 120); do + phase="$(kubectl -n "$NAMESPACE" get pod image-prepull -o jsonpath='{.status.phase}')" + reason="$(kubectl -n "$NAMESPACE" get pod image-prepull -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}')" + case "$phase/$reason" in + Succeeded/*) echo "image pulled"; break ;; + */ErrImagePull|*/ImagePullBackOff|*/InvalidImageName|Failed/*) + kubectl -n "$NAMESPACE" describe pod image-prepull | tail -20 + kubectl -n "$NAMESPACE" delete pod image-prepull --ignore-not-found + echo "::error::The cluster cannot pull $IMAGE; the running Higress was not touched." + exit 1 ;; + esac + sleep 5 + done + kubectl -n "$NAMESPACE" delete pod image-prepull --ignore-not-found + [[ "$phase" == "Succeeded" ]] || { echo "::error::Timed out pulling $IMAGE"; exit 1; } + + - name: Deploy + run: | + set -euo pipefail + previous="$(kubectl -n "$NAMESPACE" get deployment higress -o jsonpath='{.spec.template.spec.containers[?(@.name=="higress")].image}')" + echo "Deploying $IMAGE (previous: $previous)" + sed "s|__IMAGE__|$IMAGE|" deploy/k3s/higress.yaml | kubectl -n "$NAMESPACE" apply -f - + if ! kubectl -n "$NAMESPACE" rollout status deployment/higress --timeout=10m; then + kubectl -n "$NAMESPACE" get pods -o wide || true + kubectl -n "$NAMESPACE" logs deployment/higress -c higress --tail=80 || true + echo "::warning::Rolling back to $previous" + kubectl -n "$NAMESPACE" rollout undo deployment/higress + kubectl -n "$NAMESPACE" rollout status deployment/higress --timeout=10m || true + echo "::error::Release failed health checks and was rolled back." + exit 1 + fi + + - name: Check public gateway run: | - target="$DEPLOY_USER@$DEPLOY_HOST" - dir="/opt/higress/releases/$VERSION" - ssh -i ~/.ssh/deploy_key "$target" "mkdir -p '$dir'" - scp -i ~/.ssh/deploy_key release/higress-console.jar release/Dockerfile release/deploy.sh "$target:$dir/" - ssh -i ~/.ssh/deploy_key "$target" "bash '$dir/deploy.sh' '$VERSION'" + set -euo pipefail + # MCP servers require a consumer key, so an unauthenticated call must be rejected with 401. + code="" + for _ in $(seq 1 24); do + code="$(curl -s -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' -d '{}' "$PUBLIC_URL/mcp-servers/higress-admin" || true)" + if [[ "$code" == "401" ]]; then echo "gateway healthy ($code)"; exit 0; fi + sleep 5 + done + echo "::error::$PUBLIC_URL/mcp-servers/higress-admin returned ${code:-nothing}, expected 401." + exit 1 diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 61de0621..9f4aeadc 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,7 +1,8 @@ -# Overlay our console build on top of the pinned Higress all-in-one image that already runs on the server. -# The base image is tagged locally on the server (see setup-server.sh), so upstream "latest" drift never -# changes the gateway/controller versions without an explicit re-pin. -ARG BASE_IMAGE=higress-aio-base:pinned +# Overlay our console build on top of a pinned Higress all-in-one image. +# The base is pinned by digest so upstream "latest" drift never changes the gateway/controller +# versions without an explicit re-pin. To move to a newer Higress, pull the new all-in-one, +# take its digest and update BASE_IMAGE here. +ARG BASE_IMAGE=higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/all-in-one@sha256:55f5fa3ec77d5c00c5622c05019e6deb030b195bb0efa7212eff3555c071bed0 FROM ${BASE_IMAGE} COPY higress-console.jar /app/higress-console.jar diff --git a/deploy/README.md b/deploy/README.md index d04e1dc3..6ea1949a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,6 +1,6 @@ # Deployment (Time-Machine-Lab fork) -This fork adds static Bearer API tokens to the console and deploys it onto our single Higress all-in-one host. +This fork adds static Bearer API tokens to the console and runs it inside the Higress all-in-one image on our K3s cluster. ## Console API token @@ -11,31 +11,38 @@ admin user. Failures are rate limited per client IP (same limits as form/basic l On the server the plaintext token is `/opt/higress/secrets/console-api-token`; the container only gets its digest through `/opt/higress/console.env`. -## Layout on the server (`/opt/higress`) +## Where it runs -| Path | Purpose | +Since 2026-11-11 Higress runs on the K3s cluster (`k3s.seasonschange.cn`), namespace `higress`. +The manifest is `k3s/higress.yaml`: one pod with the all-in-one container plus a Redis sidecar +(Higress MCP expects Redis on `127.0.0.1:6379`). + +| Object | Purpose | |---|---| -| `data/` | Higress all-in-one data volume | -| `releases//` | Uploaded jar + Dockerfile + deploy.sh (last 3 kept) | -| `console.env` | Env file for the container (token digest) | -| `ports.env` | Loopback host ports (`CONSOLE_PORT`, `GATEWAY_PORT`, `GATEWAY_TLS_PORT`; default 8001/8080/8443) | -| `secrets/` | console API token, `ops-admin` / `claude-ops` consumer keys | -| `redis.pass` | Password of the `higress-redis` container used by Higress MCP | -| `setup/` | Copy of this directory used for one-time setup | +| PVC `higress-data` | `/data` of the all-in-one: routes, plugins, MCP servers, consumers (node `agent-1`) | +| Secret `higress-console-env` | `HIGRESS_CONSOLE_ADMIN_API_TOKENS` (sha256 digest of the console API token) | +| Secret `higress-redis` | Redis password (also referenced by the MCP redis config inside `/data`) | +| Service `higress` | `8080` gateway, `8001` console (cluster-internal only) | +| Ingress `mcp` | `mcp.tmlab.top` → gateway | + +The plaintext console token and the `ops-admin` / `claude-ops` consumer keys are still kept in +`/opt/higress/secrets/` on 207.57.124.102. To reach the console: `kubectl -n higress port-forward svc/higress 8001`. ## Flow -1. `setup-server.sh` (one time, idempotent): pins the running all-in-one image as `higress-aio-base:pinned`, - generates the console token, installs nginx + certbot and issues the certificate for `$DOMAIN`. - nginx terminates HTTPS and proxies to the gateway on `127.0.0.1:$GATEWAY_PORT`; certbot renews via `certbot.timer` - and reloads nginx through a deploy hook. -2. Push to `main` → `.github/workflows/deploy.yaml` builds the jar, uploads it with `deploy.sh`, which builds - `higress-aio-tml:`, swaps the container, checks console readiness + token auth + gateway, and rolls back - on failure. Manual rollback: `bash /opt/higress/releases//deploy.sh `. +1. Push to `main` → `.github/workflows/deploy.yaml` builds the jar, builds `Dockerfile` (the jar on top of the + all-in-one base pinned by digest) and pushes `docker.io//higress-console:`. +2. The deploy job pre-pulls the image on the node with a throw-away pod (the Deployment is `Recreate`, so the old + pod is only stopped once the new image is known to be pullable), applies `k3s/higress.yaml` with the new image, + waits for the console readiness probe, and runs `kubectl rollout undo` if it fails. It then checks that + `https://mcp.tmlab.top/mcp-servers/higress-admin` rejects an unauthenticated call with 401. + Manual rollback: `kubectl -n higress rollout undo deployment/higress`. 3. `mcp/setup_higress_admin.py` registers the console itself as the `higress-admin` MCP server (`https:///mcp-servers/higress-admin`, consumer `ops-admin`). Re-run after editing - `mcp/higress-admin-openapi.yaml`: - `python3 setup_higress_admin.py higress-admin-openapi.yaml` on the server. + `mcp/higress-admin-openapi.yaml`. + +Required secrets: repository secret `KUBE_CONFIG` (kubeconfig of the namespace-scoped ServiceAccount +`higress/ci-deployer`) and the organization secrets `DOCKER_USERNAME` / `DOCKER_PASSWORD`. +To move to a newer Higress all-in-one, update the digest in `Dockerfile` and push. -Required repository secrets: `DEPLOY_HOST`, `DEPLOY_USER`, `DEPLOY_SSH_KEY`, `DEPLOY_KNOWN_HOSTS`. -To move to a newer Higress all-in-one, pull it on the server and re-tag it as `higress-aio-base:pinned`, then redeploy. +`deploy.sh`, `setup-server.sh` and `nginx/` describe the previous single-host Docker setup and are kept for reference. diff --git a/deploy/k3s/higress.yaml b/deploy/k3s/higress.yaml new file mode 100644 index 00000000..22f0573b --- /dev/null +++ b/deploy/k3s/higress.yaml @@ -0,0 +1,87 @@ +# Higress all-in-one (gateway + controller + embedded apiserver + this console) on the K3s cluster, +# namespace `higress`. Applied by .github/workflows/deploy.yaml with __IMAGE__ substituted. +# +# Pre-created in the cluster (not in git): +# Secret higress-console-env HIGRESS_CONSOLE_ADMIN_API_TOKENS (sha256 digest of the console API token) +# Secret higress-redis password (also referenced by the MCP redis config in /data) +# /data (routes, plugins, MCP servers, consumers) lives in PVC higress-data on node agent-1. +# Redis runs as a sidecar so Higress MCP still finds it on 127.0.0.1:6379. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: higress-data, namespace: higress} +spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path + resources: {requests: {storage: 1Gi}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: higress, namespace: higress} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: {matchLabels: {app: higress}} + template: + metadata: {labels: {app: higress}} + spec: + nodeSelector: {role: legacy} + automountServiceAccountToken: false + enableServiceLinks: false + terminationGracePeriodSeconds: 30 + containers: + - name: higress + # Replaced by CI with docker.io//higress-console:. + image: __IMAGE__ + imagePullPolicy: IfNotPresent + envFrom: [{secretRef: {name: higress-console-env}}] + ports: + - {name: console, containerPort: 8001} + - {name: http, containerPort: 8080} + - {name: https, containerPort: 8443} + volumeMounts: [{name: data, mountPath: /data}] + resources: + requests: {cpu: 200m, memory: 768Mi} + limits: {memory: 2Gi} + startupProbe: + httpGet: {path: /healthz/ready, port: 8001} + periodSeconds: 5 + failureThreshold: 72 + readinessProbe: + tcpSocket: {port: 8080} + periodSeconds: 10 + livenessProbe: + tcpSocket: {port: 8080} + periodSeconds: 30 + failureThreshold: 5 + - name: redis + image: docker.io/library/redis:7-alpine + command: ["sh","-c","exec redis-server --bind 127.0.0.1 --port 6379 --requirepass \"$REDIS_PASSWORD\" --maxmemory 64mb --maxmemory-policy allkeys-lru --save ''"] + env: + - name: REDIS_PASSWORD + valueFrom: {secretKeyRef: {name: higress-redis, key: password}} + resources: + requests: {cpu: 10m, memory: 16Mi} + limits: {memory: 128Mi} + volumes: + - name: data + persistentVolumeClaim: {claimName: higress-data} +--- +apiVersion: v1 +kind: Service +metadata: {name: higress, namespace: higress} +spec: + selector: {app: higress} + ports: + - {name: http, port: 8080, targetPort: 8080} + - {name: console, port: 8001, targetPort: 8001} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: {name: mcp, namespace: higress} +spec: + ingressClassName: traefik + rules: + - host: mcp.tmlab.top + http: + paths: + - {path: /, pathType: Prefix, backend: {service: {name: higress, port: {number: 8080}}}}