Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 58 additions & 73 deletions .github/workflows/release-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ concurrency:

env:
PUBLIC_URL: https://plane.tmlab.top
PLANE_HTTP_PORT: "6767"

jobs:
prepare:
Expand All @@ -32,29 +31,19 @@ jobs:
- name: Validate deployment configuration
shell: bash
env:
SERVER_IP: ${{ secrets.SERVER_IP }}
SERVER_USER: ${{ secrets.SERVER_USER }}
SERVER_PWD: ${{ secrets.SERVER_PWD }}
KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: |
set -euo pipefail

for secret_name in SERVER_IP SERVER_USER SERVER_PWD DOCKER_USERNAME DOCKER_PASSWORD; do
for secret_name in KUBE_CONFIG DOCKER_USERNAME DOCKER_PASSWORD; do
if [[ -z "${!secret_name:-}" ]]; then
echo "::error::Required GitHub Secret ${secret_name} is not configured."
exit 1
fi
done

[[ "$SERVER_IP" =~ ^[A-Za-z0-9][A-Za-z0-9.:-]*$ ]] || {
echo "::error::SERVER_IP contains unsupported characters."
exit 1
}
[[ "$SERVER_USER" =~ ^[A-Za-z0-9._-]+$ ]] || {
echo "::error::SERVER_USER contains unsupported characters."
exit 1
}
[[ "$DOCKER_USERNAME" =~ ^[a-z0-9]+([._-][a-z0-9]+)*$ ]] || {
echo "::error::DOCKER_USERNAME is not a valid Docker Hub namespace."
exit 1
Expand Down Expand Up @@ -146,99 +135,95 @@ jobs:
runs-on: ubuntu-24.04
environment: production
env:
# Production runs on the K3s cluster (k3s.seasonschange.cn), namespace plane.
# KUBE_CONFIG is a kubeconfig for the namespace-scoped ServiceAccount plane/ci-deployer.
NAMESPACE: plane
VERSION: ${{ needs.prepare.outputs.version }}
SERVER_IP: ${{ secrets.SERVER_IP }}
SERVER_USER: ${{ secrets.SERVER_USER }}
SSHPASS: ${{ secrets.SERVER_PWD }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
APP_DEPLOYMENTS: api worker beat-worker web admin space live mcp proxy
steps:
- name: Check out repository
uses: actions/checkout@v6

- name: Install SSH client dependencies
run: |
sudo apt-get update
sudo apt-get install --yes sshpass

- name: Trust production SSH host key
- name: Configure cluster access
shell: bash
env:
KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
run: |
set -euo pipefail
expected_fingerprint="SHA256:jmzl6o5sZWlfwF2Ncg6hrEf7lluI0iDfp7t1W2MJwRk"
scanned_key_file="$RUNNER_TEMP/production-ed25519-host-key"
known_hosts_file="$HOME/.ssh/known_hosts"

install -d -m 700 "$HOME/.ssh"
if ! ssh-keyscan -p 22 -T 10 -t ed25519 -H "$SERVER_IP" > "$scanned_key_file" 2>/dev/null; then
echo "::error::Could not retrieve the production ED25519 SSH host key."
exit 1
fi
if [[ ! -s "$scanned_key_file" ]]; then
echo "::error::The production server returned no ED25519 SSH host key."
exit 1
fi

mapfile -t fingerprints < <(
ssh-keygen -lf "$scanned_key_file" -E sha256 \
| awk '$NF == "(ED25519)" {print $2}' \
| sort -u
)
if [[ "${#fingerprints[@]}" -ne 1 || "${fingerprints[0]:-}" != "$expected_fingerprint" ]]; then
echo "::error::Production ED25519 SSH host key fingerprint does not match the manually verified fingerprint."
exit 1
fi
install -m 700 -d "$HOME/.kube"
printf '%s\n' "$KUBE_CONFIG" > "$HOME/.kube/config"
chmod 600 "$HOME/.kube/config"
kubectl -n "$NAMESPACE" get deployment api

install -m 600 "$scanned_key_file" "$known_hosts_file"

- name: Prepare deployment directory
- name: Render manifests
shell: bash
run: |
set -euo pipefail
sshpass -e ssh -p 22 -o StrictHostKeyChecking=yes \
"$SERVER_USER@$SERVER_IP" 'install -d -m 700 "$HOME/plane"'
mkdir -p "$RUNNER_TEMP/k3s"
for f in deployments/k3s/*.yaml; do
sed -e "s|__DOCKER_USERNAME__|$DOCKER_USERNAME|g" -e "s|__VERSION__|$VERSION|g" "$f" \
> "$RUNNER_TEMP/k3s/$(basename "$f")"
done
! grep -n "__[A-Z_]*__" "$RUNNER_TEMP"/k3s/*.yaml
echo "Previous release: $(kubectl -n "$NAMESPACE" get deployment api -o jsonpath='{.spec.template.spec.containers[0].image}')"

- name: Log in to Docker Hub on server
# Django migrations are not reversible, so dump the database before running them.
- name: Back up database
shell: bash
run: |
set -euo pipefail
remote_docker_username=$(printf '%q' "$DOCKER_USERNAME")
printf '%s' "$DOCKER_PASSWORD" | \
sshpass -e ssh -p 22 -o StrictHostKeyChecking=yes \
"$SERVER_USER@$SERVER_IP" \
"docker login --username $remote_docker_username --password-stdin"
kubectl -n "$NAMESPACE" delete job plane-predeploy-backup --ignore-not-found --wait=true
kubectl -n "$NAMESPACE" apply -f "$RUNNER_TEMP/k3s/backup-job.yaml"
if ! kubectl -n "$NAMESPACE" wait --for=condition=complete job/plane-predeploy-backup --timeout=10m; then
kubectl -n "$NAMESPACE" logs job/plane-predeploy-backup --tail=50 || true
echo "::error::Pre-release database backup failed; nothing was changed."
exit 1
fi
kubectl -n "$NAMESPACE" logs job/plane-predeploy-backup

- name: Copy deployment assets
- name: Run database migrations
shell: bash
run: |
set -euo pipefail
sshpass -e scp -P 22 -o StrictHostKeyChecking=yes \
deployments/cli/community/docker-compose.yml \
deployments/production/docker-compose.override.yml \
deployments/production/deploy.sh \
"$SERVER_USER@$SERVER_IP:~/plane/"
kubectl -n "$NAMESPACE" delete job plane-migrator --ignore-not-found --wait=true
kubectl -n "$NAMESPACE" apply -f "$RUNNER_TEMP/k3s/migrator-job.yaml"
if ! kubectl -n "$NAMESPACE" wait --for=condition=complete job/plane-migrator --timeout=15m; then
kubectl -n "$NAMESPACE" logs job/plane-migrator --tail=100 || true
echo "::error::Migrations failed. The running release was not changed; the pre-release dump is in PVC plane-predeploy-backups."
exit 1
fi
kubectl -n "$NAMESPACE" logs job/plane-migrator --tail=20

- name: Run production deployment
- name: Roll out release
shell: bash
run: |
set -euo pipefail
remote_version=$(printf '%q' "$VERSION")
remote_docker_username=$(printf '%q' "$DOCKER_USERNAME")
remote_public_url=$(printf '%q' "$PUBLIC_URL")
remote_http_port=$(printf '%q' "$PLANE_HTTP_PORT")

sshpass -e ssh -p 22 -o StrictHostKeyChecking=yes \
"$SERVER_USER@$SERVER_IP" \
"cd \"\$HOME/plane\" && chmod 700 deploy.sh && ./deploy.sh $remote_version $remote_docker_username $remote_public_url $remote_http_port"
kubectl -n "$NAMESPACE" apply -f "$RUNNER_TEMP/k3s/plane.yaml"
failed=""
for d in $APP_DEPLOYMENTS; do
kubectl -n "$NAMESPACE" rollout status "deployment/$d" --timeout=10m || failed="$failed $d"
done
if [[ -n "$failed" ]]; then
kubectl -n "$NAMESPACE" get pods -o wide || true
for d in $failed; do kubectl -n "$NAMESPACE" logs "deployment/$d" --tail=60 || true; done
echo "::warning::Rolling back application deployments"
for d in $APP_DEPLOYMENTS; do kubectl -n "$NAMESPACE" rollout undo "deployment/$d" || true; done
for d in $APP_DEPLOYMENTS; do kubectl -n "$NAMESPACE" rollout status "deployment/$d" --timeout=10m || true; done
echo "::error::Release $VERSION failed to become ready on:$failed (application images rolled back; migrations were not reversed)."
exit 1
fi

- name: Verify production externally
shell: bash
run: |
set -euo pipefail
curl --fail --show-error --silent \
--retry 30 --retry-delay 10 --retry-all-errors \
--resolve "plane.tmlab.top:443:$SERVER_IP" \
"$PUBLIC_URL/" > /dev/null
curl --fail --show-error --silent \
--retry 10 --retry-delay 5 --retry-all-errors \
"$PUBLIC_URL/api/instances/" > /dev/null

promote:
name: Promote ${{ matrix.image }} to latest
Expand Down
34 changes: 34 additions & 0 deletions deployments/k3s/backup-job.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Pre-release pg_dump of the `plane` database into PVC plane-predeploy-backups (last 5 kept).
# Migrations are not reversible, so CI runs this before the migrator. The cluster also takes a
# daily dump of every pg-main database (database/pg-main-dump CronJob).
apiVersion: batch/v1
kind: Job
metadata: {name: plane-predeploy-backup, namespace: plane}
spec:
backoffLimit: 0
template:
spec:
restartPolicy: Never
enableServiceLinks: false
automountServiceAccountToken: false
nodeSelector: {role: legacy}
securityContext: {fsGroup: 26}
containers:
- name: dump
image: ghcr.io/cloudnative-pg/postgresql:17-standard-trixie
command: ["/bin/bash", "-c"]
args:
- |
set -euo pipefail
f="/backup/plane-$(date -u +%Y%m%dT%H%M%SZ)-before-__VERSION__.dump"
pg_dump "$DATABASE_URL" -Fc -f "$f"
ls -la "$f"
ls -1t /backup/*.dump | tail -n +6 | xargs -r rm -f
env:
- name: DATABASE_URL
valueFrom: {secretKeyRef: {name: plane-app-env, key: DATABASE_URL}}
resources: {requests: {cpu: 50m, memory: 64Mi}, limits: {memory: 512Mi}}
volumeMounts: [{name: backup, mountPath: /backup}]
volumes:
- name: backup
persistentVolumeClaim: {claimName: plane-predeploy-backups}
20 changes: 20 additions & 0 deletions deployments/k3s/migrator-job.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Runs Django migrations before the new release starts. Recreated by CI on every release.
apiVersion: batch/v1
kind: Job
metadata: {name: plane-migrator, namespace: plane}
spec:
backoffLimit: 2
template:
spec:
restartPolicy: Never
enableServiceLinks: false
automountServiceAccountToken: false
nodeSelector: {role: legacy}
containers:
- name: migrator
image: docker.io/__DOCKER_USERNAME__/plane-api:__VERSION__
args: ["./bin/docker-entrypoint-migrator.sh"]
envFrom: [{secretRef: {name: plane-app-env}}]
resources: {requests: {cpu: 50m, memory: 128Mi}, limits: {memory: 1Gi}}
volumeMounts: [{name: logs, mountPath: /code/plane/logs}]
volumes: [{name: logs, emptyDir: {}}]
Loading
Loading