diff --git a/.github/workflows/k3s-ca.yml b/.github/workflows/k3s-ca.yml new file mode 100644 index 0000000..381bf76 --- /dev/null +++ b/.github/workflows/k3s-ca.yml @@ -0,0 +1,64 @@ +name: k3s-ca + +# The k3s custom-CA library (pkg/templates/scripts/tn-k3s-ca.sh) and its harness. +# Unit tests run on every change; the docker matrix against real rancher/k3s +# releases is started by hand (it needs privileged containers and several minutes). + +on: + pull_request: + paths: + - 'pkg/templates/**' + - 'test/k3s-ca/**' + - '.github/workflows/k3s-ca.yml' + workflow_dispatch: + inputs: + suites: + description: 'run.sh --suite value' + default: 'fresh,remediation' + versions: + description: 'k3s versions (space separated)' + default: 'v1.34.11 v1.36.5' + +permissions: + contents: read + +jobs: + unit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + cache: true + - name: shellcheck + run: | + shellcheck -s sh -S warning pkg/templates/scripts/tn-k3s-ca.sh + - name: go test + run: go test ./pkg/templates/... ./test/k3s-ca/... + + docker-matrix: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + cache: true + - name: run the harness + env: + TN_K3S_CA_LIB: ${{ github.workspace }}/pkg/templates/scripts/tn-k3s-ca.sh + SUITES: ${{ inputs.suites }} + VERSIONS: ${{ inputs.versions }} + run: | + test/k3s-ca/run.sh --suite "$SUITES" --versions "$VERSIONS" --gens bs-o111 -j 3 --out "$RUNNER_TEMP/k3sca" + - name: logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: k3sca-results + path: | + ${{ runner.temp }}/k3sca/results.tsv + ${{ runner.temp }}/k3sca/logs diff --git a/README.md b/README.md index 9b6082c..ee197da 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,22 @@ Securely connect to your edge devices via Teknoir's proxy infrastructure. tnctl socks-proxy [device] 1080 ``` +### 4. Bootstrap a device + +```bash +tnctl bootstrap my-device # writes bootstrap_{agent,server,docker_server}_my-device.sh +tnctl bootstrap my-device --k3s-version v1.34.11+k3s1 # install another k3s release +``` + +The scripts install a **pinned k3s release** (`v1.36.5+k3s1` by default, the `stable` channel when this +was written and the release the CA tests run against) instead of whatever the `stable` channel happens +to be on the day the script runs. `--k3s-version` takes an exact release, or `stable` / `latest` for the +installer's own channels; at run time `INSTALL_K3S_VERSION=... sh bootstrap_server_my-device.sh` wins. +An agent must run the version of its server: generate both scripts with the same `--k3s-version`. +k3s cannot be downgraded, so do not pin below what a device already runs. The server script also +repairs the k3s CA of devices that were bootstrapped with an older script, see +[`docs/commands/bootstrap.md`](docs/commands/bootstrap.md) and [`docs/troubleshooting.md`](docs/troubleshooting.md). + ## Global Flags These flags are available to all subcommands: diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index bf1e3da..87c6ddb 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -20,6 +20,10 @@ var bootstrapCmd = &cobra.Command{ Use: "bootstrap [device-name]", Short: "Generate bootstrap scripts for a device", RunE: func(cmd *cobra.Command, args []string) error { + k3sVersion, _ := cmd.Flags().GetString("k3s-version") + if err := templates.ValidateK3SVersion(k3sVersion); err != nil { + return err + } cfg, err := config.Load() if err != nil { return err @@ -101,18 +105,27 @@ var bootstrapCmd = &cobra.Command{ } dockerSecret, _ := cmd.Flags().GetString("docker-secret") + k3sUpgrade, _ := cmd.Flags().GetBool("k3s-upgrade") + force, _ := cmd.Flags().GetBool("force") + if force && !k3sUpgrade { + return fmt.Errorf("--force only applies together with --k3s-upgrade") + } data := templates.TemplateData{ - Domain: domain, - Namespace: namespace, - DeviceID: deviceName, - RSAPrivate: rsaPrivate, - RSAPublic: rsaPublic, - Username: username, - Password: password, - PublicSSHKey: publicSSHKey, - K3SToken: k3sToken, - DockerSecret: dockerSecret, + Domain: domain, + Namespace: namespace, + DeviceID: deviceName, + RSAPrivate: rsaPrivate, + RSAPublic: rsaPublic, + Username: username, + Password: password, + PublicSSHKey: publicSSHKey, + K3SToken: k3sToken, + DockerSecret: dockerSecret, + K3SVersion: k3sVersion, + K3SVersionExplicit: cmd.Flags().Changed("k3s-version"), + K3SUpgrade: k3sUpgrade, + K3SForce: force, } generate := func(tmpl *template.Template, filename string) error { @@ -162,6 +175,12 @@ func generateK3SToken() (string, error) { func init() { bootstrapCmd.Flags().Bool("skip-upload", false, "Skip uploading the generated scripts") + bootstrapCmd.Flags().String("k3s-version", templates.DefaultK3SVersion, + "k3s release the scripts install (INSTALL_K3S_VERSION): an exact release like v1.36.5+k3s1, or stable / latest for the installer's channels. Server and agent scripts use the same value") + bootstrapCmd.Flags().Bool("k3s-upgrade", false, + "let the scripts replace an installed k3s with --k3s-version (TN_K3S_UPGRADE=true is the default in the scripts). Without it a re-run keeps the installed k3s; a downgrade is always refused") + bootstrapCmd.Flags().Bool("force", false, + "with --k3s-upgrade: allow an upgrade over more than one minor version (TN_K3S_FORCE=true is the default in the scripts)") bootstrapCmd.Flags().String("docker-secret", "", "The contents of the .dockerconfigjson") rootCmd.AddCommand(bootstrapCmd) } diff --git a/cmd/bootstrap_test.go b/cmd/bootstrap_test.go new file mode 100644 index 0000000..4292023 --- /dev/null +++ b/cmd/bootstrap_test.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "testing" + + "teknoir/cli/pkg/templates" +) + +func TestBootstrapK3SVersionFlag(t *testing.T) { + f := bootstrapCmd.Flags().Lookup("k3s-version") + if f == nil { + t.Fatal("no --k3s-version flag") + } + if f.DefValue != templates.DefaultK3SVersion { + t.Errorf("default %q", f.DefValue) + } +} + +func TestBootstrapK3SUpgradeFlags(t *testing.T) { + for _, name := range []string{"k3s-upgrade", "force"} { + f := bootstrapCmd.Flags().Lookup(name) + if f == nil { + t.Fatalf("no --%s flag", name) + } + // a re-run keeps the installed k3s unless the operator asks otherwise + if f.DefValue != "false" { + t.Errorf("--%s defaults to %q", name, f.DefValue) + } + } +} diff --git a/docs/commands/bootstrap.md b/docs/commands/bootstrap.md index 03dcebc..3a18c2b 100644 --- a/docs/commands/bootstrap.md +++ b/docs/commands/bootstrap.md @@ -15,6 +15,9 @@ The device may be given as an argument, via the `--device` global flag, or selec | Flag | Default | Description | | --- | --- | --- | | `--skip-upload` | `false` | Skip uploading the generated scripts. | +| `--k3s-version` | `v1.36.5+k3s1` | k3s release the server script installs: an exact release (`v1.36.5+k3s1`), or `stable` / `latest` for the installer's channels. The agent script installs the release of its server instead; a release given here explicitly is installed only if it is not newer than the server's. | +| `--k3s-upgrade` | `false` | Let the scripts replace an installed k3s with `--k3s-version` (sets `TN_K3S_UPGRADE=true` as the script's default). Without it a re-run **keeps** the installed k3s. See [Re-running a script](#re-running-a-script-the-installed-k3s-is-kept). | +| `--force` | `false` | Only with `--k3s-upgrade`: allow an upgrade over more than one minor version (`TN_K3S_FORCE=true`). A downgrade is never allowed. | | `--docker-secret` | `""` | Contents of the `.dockerconfigjson` to embed. | Plus all [global flags](index.md#global-flags). @@ -33,6 +36,152 @@ The scripts are rendered from templates and populated with the device's decoded (RSA keypair, username/password, public SSH key), the domain/namespace, the device ID, a freshly generated K3S token, and the optional Docker secret. +## The k3s version + +The server and agent scripts install a **pinned** k3s release: `INSTALL_K3S_VERSION` is set from +`--k3s-version`, default `v1.36.5+k3s1` (the `stable` channel of + when this was written, and the release that the CA tests in +`test/k3s-ca` prove with the re-issued root). Without a pin the installer takes whatever `stable` is on +the day the script runs, and that moved the fleet onto a k3s that refused the device CA once already. + +| `--k3s-version` | What the script does | +| --- | --- | +| *(default)* or `v1.36.5+k3s1` | `INSTALL_K3S_VERSION=v1.36.5+k3s1` | +| any other `vX.Y.Z+k3sN` | that release | +| `stable` | nothing: the old behaviour, the installer picks the `stable` channel | +| `latest` | `INSTALL_K3S_CHANNEL=latest` | + +- At run time, `INSTALL_K3S_VERSION=v1.xx.y+k3sN sh bootstrap_server_.sh` overrides the pin. +- The pin is the release for a **new server**. A device that already has k3s keeps what it has, see + below. +- **An agent follows its server.** A kubelet must never be newer than the apiserver it joins, so the + agent script does not install the pin. It reads the release of the server and installs exactly that + one. k3s does **not** answer `/version` to an anonymous caller (its apiserver runs with + `anonymous-auth=false`: HTTP 401 on every release, checked with v1.30, v1.34 and v1.36), and the join + token only opens the supervisor endpoints, so the **server script publishes the release it runs** + as `/server/static/teknoir/k3s-version` after its installer has run. k3s serves + `server/static` without authentication, so an agent reads + `https:///static/teknoir/k3s-version` (and, for a server that does allow it, `/version`). + A release that you asked for (`--k3s-version`, or `INSTALL_K3S_VERSION` when running the script) is + installed if it is not newer than the server's (older gets a warning) and **refused** if it is, + unless `TN_K3S_FORCE=true`. A channel (`stable`, `latest`) cannot be compared: the server's release + is installed instead. If the release of the server cannot be read, the script **stops** and says + why, unless a release was asked for (it is then installed as asked, with a warning) or + `TN_K3S_FORCE=true`: the server is down or not reachable (tried three times), or it was + bootstrapped by a script from before this change and publishes nothing. **For such a server, run + its bootstrap script again**: that keeps k3s as it is and publishes the release. An out-of-band + upgrade of k3s leaves the file at the older release, which is the safe direction. An OS image build + (`OS_BUILD`) cannot reach a server and installs the pin. +- Raise the default in `pkg/templates/bootstrap_templates.go` (`DefaultK3SVersion`) together with the + harness runs in `test/k3s-ca` for the new release. + +## Re-running a script: the installed k3s is kept + +A bootstrap script is meant to be run again (to repair the CA, refresh the manifests, ...). It must not +upgrade k3s as a side effect, so the pin only applies to a device **without** k3s. Before it runs the +installer, the server and agent scripts read `k3s --version` and decide: + +| State of the device | What the script does | +| --- | --- | +| k3s is not installed | Installs the pin (`INSTALL_K3S_VERSION`); an **agent** installs the release of its server, see above | +| a k3s binary or `server/db` exists, but `k3s --version` gives no release | **Refuses** to install anything (the installer would replace k3s without any of the checks below); `TN_K3S_FORCE=true` goes on as if k3s were not installed | +| k3s is installed, any pin | **Keeps the installed release**: it passes that exact release to the installer (which then only rewrites the unit and the symlinks) and says that the pin was not applied | +| k3s is installed, `--k3s-upgrade` (or `TN_K3S_UPGRADE=true`) | Upgrades to the pin (an agent: to the release of its server), with these checks: a pin **below** the installed release is **refused** (k3s cannot go back, the datastore has been migrated; this also holds with `--force`), and so is a pin more than **one minor version** above (`v1.34` to `v1.36`: upgrade to `v1.35.x` first) unless `--force` (`TN_K3S_FORCE=true`), which turns the refusal into a warning | +| installed, upgrade requested, but the pin is a channel (`stable`, `latest`) | Upgrades, with a warning: the installer picks the release, so downgrade and minor jump cannot be checked. Pin an exact release to get the checks | + +Upgrade servers before their agents, and one minor version at a time. The environment of the +script overrides what `tnctl bootstrap` baked in (`TN_K3S_UPGRADE=false sh bootstrap_*.sh`). + +**Order in the server script**: (0) the arguments of the installer (`INSTALL_K3S_EXEC`, including a +value you gave: the cgroup step reads it), the data dir and its check against `config.yaml`, (1) this +release decision and the cgroup flag below, (2) the CA repair, (3) the k3s installer. A refusal in (0) +or (1) happens before anything is changed ("Nothing was changed"). The CA is still repaired before the installer, so an upgrade never meets a root that the +new k3s would reject. Should a later step ever refuse after the CA step has changed the device (the CA +rotated, files replaced, k3s restarted), the message lists exactly that, instead of "Nothing was +changed". + +## cgroup v1 devices + +Kubernetes 1.35 and later make the kubelet **refuse to start on a cgroup v1 host** +(`failCgroupV1` defaults to true) and k3s does not change that. Devices on Jetson L4T r32/r35, +Ubuntu 18.04 or 20.04 are cgroup v1 (or hybrid). Both scripts therefore check +`stat -fc %T /sys/fs/cgroup`: anything but `cgroup2fs` is v1 or hybrid. On such a host, when the k3s +that will run is **v1.31 or later** (the kept installed release, or the pin, or the release of the +server, or a channel; the kubelet knows the flag since 1.31, and an older one would not start with an +unknown argument), the script writes + +```yaml +# /etc/rancher/k3s/config.yaml.d/99-teknoir-cgroupv1.yaml +kubelet-arg+: + - "fail-cgroupv1=false" +``` + +and warns that cgroup v1 is deprecated. The flag only matters from v1.35, but it is set from v1.31 so +that a later upgrade (also one made by hand, outside the scripts) does not stop the kubelet. The file +is written atomically (temporary file, then `mv`). `kubelet-arg+` **appends** to the `kubelet-arg` of +`config.yaml` and the other drop-ins, so the arguments of the device owner are kept; the file is +rewritten only when it differs, so a re-run changes nothing. A `fail-cgroupv1` that the device sets +itself in a file that k3s reads (`config.yaml` or another `config.yaml.d/*.yaml` or `*.yml`) is respected +and our file is removed. Only an uncommented line that sets `fail-cgroupv1=false` counts: a comment, or +`fail-cgroupv1=true`, does not, and our file is written as usual (setting `true` on a cgroup v1 host on +purpose is not supported). +A flag only in `INSTALL_K3S_EXEC` reaches the unit when the installer has run, so our file stays (or is +written) until then: the same value twice is harmless, and the command line wins. For an installed k3s +older than v1.31 that is being upgraded, the flag is written after the installer (`tn_inst_finish`), as the +old kubelet cannot take it. The installer starts the new release as soon as it has rewritten the unit, and +v1.35 or later does not start on cgroup v1 without the flag, so when the release is a channel (`stable`, +`latest`) or an exact v1.35 or later, `--kubelet-arg=fail-cgroupv1=false` is also appended to +`INSTALL_K3S_EXEC` (once): the new unit has it from its first start, and the old kubelet never reads it. +An exact v1.31 to v1.34 does not need it. On cgroup v2 the file is removed. An OS image build (`OS_BUILD=true`) runs on another machine than the +device and cannot tell: the flag is set there, it is harmless on cgroup v2. `TN_K3S_CGROUPV1=true|false` +forces the answer. The way out is cgroup v2 (kernel command line `systemd.unified_cgroup_hierarchy=1`, +or a newer OS release); then delete the file. + +The flag is written **before** the CA step, which may restart k3s: a k3s that a cgroup v1 host made +exit (an old script upgraded it to v1.35 or later) restarts itself every few seconds and reads the +flag then, so it is up for `rotate-ca` and the re-run ends with a working k3s. `rotate-ca` is tried +again for about a minute while the unit is starting. + +The server script no longer replaces an existing `/etc/rancher/k3s/config.yaml` on a re-run: it adds +`disable`, `node-name` and (for a non-default data dir) `data-dir` when they are missing. + +## The k3s CA + +The server scripts (`bootstrap_server_.sh`) use the device certificate (`rsa_public.pem`) +and key from the device's settings as the **root of the cluster CA** and issue the intermediate and +leaf CAs below it. k3s v1.34.12, v1.35.9, v1.36.5, v1.37.1 and later only accept a root with the +`keyCertSign` key usage, so the script **re-issues the root from the same key and subject** with +`keyCertSign` and CA:TRUE (valid 20 years; the device files are not modified). The code is +[`pkg/templates/scripts/tn-k3s-ca.sh`](https://github.com/teknoir/cli/blob/main/pkg/templates/scripts/tn-k3s-ca.sh), +embedded in the script and usable stand-alone (`sh tn-k3s-ca.sh check|remediate`). + +Re-running a server script is safe: + +| State of the device | What the script does with the CA | +| --- | --- | +| k3s never started | Creates the PKI (keeps a valid one) | +| k3s started, root is fine | Leaves it alone | +| k3s started, root has no `keyCertSign` (bootstrapped by an older script) | Repairs it first with `k3s certificate rotate-ca` and restarts k3s, so the k3s installer that follows cannot brick the device. `TN_K3S_CA_AUTO_REMEDIATE=false` stops with advice instead; a failed repair stops the script | +| k3s runs, but from another data dir than the one found (no `server/token` there) | Stops before anything is changed: generating a CA in the wrong directory would not touch the CA that k3s uses. Pass `DATA_DIR=` | +| k3s too old for `rotate-ca` (before v1.24.12, v1.25.8, v1.26.3) | Stops before anything is changed, with advice to upgrade k3s one minor version at a time to a release that has `rotate-ca` but is older than v1.34.12 | + +The data dir is the one k3s uses on the device, resolved the way k3s does: `--data-dir`/`-d` in the +`ExecStart` of the `k3s.service` unit, the last `data-dir` in `/etc/rancher/k3s/config.yaml.d/*.yaml`, +`data-dir` in `/etc/rancher/k3s/config.yaml`, `K3S_DATA_DIR` of the unit, then +`/var/lib/rancher/k3s`. Set `DATA_DIR` to override it. The server script writes the same data dir to +`config.yaml` when it is not the default, so the installer's unit keeps using it, and puts the Teknoir +manifests there. If the data dir found (from the `ExecStart`, `K3S_DATA_DIR` or `DATA_DIR`) differs from +the `data-dir` of `config.yaml` or `config.yaml.d`, the script **stops** before anything is changed: the +installer writes a unit without `--data-dir`, after which k3s would start from the `config.yaml` +directory, with a new CA, and leave the cluster behind. Make them agree and run the script again. + +`k3s certificate rotate-ca` and the restart wait for a slow device: a stop, start or restart of k3s may +take `TN_K3S_SERVICE_TIMEOUT` seconds (default **600**), and after a timeout the unit is watched for +`TN_K3S_READY_GRACE` (120) more seconds before the restart is called failed. + +See [Troubleshooting](../troubleshooting.md#bootstrapping-k3s) for devices that hang or were +bricked by a k3s upgrade. The agent script has no CA step (an agent has no CA of its own); it follows the k3s release of its server, see above. + ## Requirements A domain and namespace must be set, and you must be authenticated. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index ffce9c6..d747f70 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -53,6 +53,193 @@ including resolved URLs and the exact `ssh` invocation used. - Re-run with `--debug` to see the full `ssh` command and any `ncat`/proxy errors. - Verify network access to `deadend-.:2222`. +## Bootstrapping & k3s + +??? failure "k3s never becomes Ready: `Failed to validate connection to cluster ... CA cert validation failed ... x509: certificate signed by unknown authority`" + k3s v1.34.12, v1.35.9, v1.36.5, v1.37.1 and later (k3s-io/k3s#14699, a CVE fix) drop every + certificate without the `keyCertSign` key usage from the CA bundle. The device certificate of + the platform (`/etc/teknoir/rsa_public.pem`) is the **root** of the cluster CA, and the + certificates generated before this was fixed never had `keyCertSign`. The get.k3s.io `stable` + channel has been v1.36.5 since 2026-10-01, so a new device bootstrapped with an old script, or an + old device whose k3s was upgraded, hits this. + + The bootstrap scripts generated by a `tnctl` with this fix re-issue the root from the **same key + and subject** with `keyCertSign` (valid 20 years); the device files in `/etc/teknoir` are never + changed, and everything signed below the old root stays valid. Check that your `tnctl` has it: + `grep -c tn_k3s_ca_setup bootstrap_server_.sh` prints a number above 0. + + Find out where the device is with the CA tool (`tn-k3s-ca.sh`, run as root on the device; it is + [`pkg/templates/scripts/tn-k3s-ca.sh`](https://github.com/teknoir/cli/blob/main/pkg/templates/scripts/tn-k3s-ca.sh) + in this repository and is embedded in every generated `bootstrap_server_*.sh`): + + ```bash + sh tn-k3s-ca.sh check # exit 0 = ok, 2 = needs rotation, 3 = needs regeneration, 4 = foreign CA or no server data found + ``` + + Without `--data-dir`, the tool finds the k3s data dir the way k3s does (`--data-dir`/`-d` in the + `ExecStart` of `k3s.service`, then the last `data-dir` in `/etc/rancher/k3s/config.yaml.d/*.yaml`, + then `data-dir` in `/etc/rancher/k3s/config.yaml`, then `K3S_DATA_DIR` of the unit, then + `/var/lib/rancher/k3s`) and tells which one it used (`data dir ... (source)` on stderr). If a k3s + **server unit exists but the data dir it found has no `server/tls/server-ca.crt`**, `check` is never + "ok": it exits 4 (`no-server-data`) and names the directory it looked in. Either k3s has not + started on this device yet, or its data dir is somewhere the tool cannot see; pass `--data-dir DIR`. + + | Situation | How to tell | What to do | + | --- | --- | --- | + | **Fresh bootstrap that hangs**: k3s was never Ready on this device | `check` says `needs-regeneration`; `/server/token` (`/var/lib/rancher/k3s/server/token`) does not exist | Run the fixed `bootstrap_server_*.sh` again, or `sh tn-k3s-ca.sh remediate`. Nothing was ever stored in the cluster, so there is nothing to rotate: k3s is stopped while `server/tls` is replaced (the old dir is moved aside, the result is validated, and the old dir is put back if anything fails), the old files are saved as `server/tls-backup-tn-*.tar`, and k3s is started again. `rotate-ca` cannot be used here, it needs the token a started server writes | + | **Bricked by an upgrade**: the device worked, then k3s was upgraded and does not start | `check` says `needs-rotation`; k3s logs the error above | `sh tn-k3s-ca.sh remediate`. k3s must be running (a k3s that loops on the error is fine, it listens). The tool stages the new CA files, runs `k3s certificate rotate-ca` against the running server, copies the files and restarts k3s. Workloads and the datastore are kept. Details below | + | **Still works, not upgraded yet** | `check` says `needs-rotation`; `k3s --version` shows v1.34.11, v1.35.8, v1.36.4 or v1.37.0 or older | Do it now: `sh tn-k3s-ca.sh remediate`, or run the fixed bootstrap script again (it repairs the CA before it touches k3s; `TN_K3S_CA_AUTO_REMEDIATE=false` makes it stop with advice instead) | + | **`foreign`** | `check` says `foreign` | The root is not the Teknoir device CA (no matching key in `server/tls/root-ca.key` or `/etc/teknoir/rsa_private.pem`). The tool does not touch it | + + `remediate` runs in phases, which you can also run one at a time (`--phase`): `prepare` + (offline: checks, backs up `server/tls`, stages `server/.tn-ca-rotate-