From 8302fd90cfcc36ffc0da7278f90f467e0367ee34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85slund?= Date: Fri, 16 Oct 2026 17:56:40 +0000 Subject: [PATCH 1/2] fix(bootstrap): refuse scripts without the device key pair; key-only user Onboarding a new device right after creating it: the device-controller generates the key pair after creation and the Backstage catalog only picks it up on its next device refresh (5 min), so tnctl bootstrap wrote empty rsa_private.pem/rsa_public.pem and the server script failed later at "Cannot generate the k3s CA". tnctl now says so and stops. setup_user no longer aborts the script when the platform has no password for the device user (chpasswd under set -e): it warns and keeps key-only login, so the SSH key for the tunnel is still installed. Found onboarding a fresh VM device (vm-tnctl-01) to staging with tnctl. Co-Authored-By: Claude Opus 5.5 --- cmd/bootstrap.go | 6 ++++++ pkg/templates/partials/user_setup.sh.tmpl | 4 +++- pkg/templates/testdata/agent.golden.sh | 4 +++- pkg/templates/testdata/docker_server.golden.sh | 4 +++- pkg/templates/testdata/server.golden.sh | 4 +++- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index ab7f9a2..570d6c5 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -86,6 +86,12 @@ var bootstrapCmd = &cobra.Command{ if err != nil { return err } + // The device-controller generates the key pair after the Device is created, and the + // Backstage catalog picks it up on its next device refresh (every few minutes). Scripts + // without it install empty keys and fail at the k3s CA step on the device. + if rsaPrivate == "" || rsaPublic == "" { + return fmt.Errorf("device %s/%s has no key pair in the catalog yet (the platform adds it a few minutes after the device is created); run tnctl bootstrap again shortly", namespace, deviceName) + } username, err := decode("username", device.Spec.Settings.Username) if err != nil { return err diff --git a/pkg/templates/partials/user_setup.sh.tmpl b/pkg/templates/partials/user_setup.sh.tmpl index de8a0da..d7646e2 100644 --- a/pkg/templates/partials/user_setup.sh.tmpl +++ b/pkg/templates/partials/user_setup.sh.tmpl @@ -20,7 +20,9 @@ setup_user() { fi fi - if command -v chpasswd &> /dev/null; then + if [ -z "${_FIRST_USER_PASS}" ]; then + warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login" + elif command -v chpasswd &> /dev/null; then echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd elif command -v passwd &> /dev/null; then echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}" diff --git a/pkg/templates/testdata/agent.golden.sh b/pkg/templates/testdata/agent.golden.sh index 7cd79e3..01b207b 100644 --- a/pkg/templates/testdata/agent.golden.sh +++ b/pkg/templates/testdata/agent.golden.sh @@ -836,7 +836,9 @@ setup_user() { fi fi - if command -v chpasswd &> /dev/null; then + if [ -z "${_FIRST_USER_PASS}" ]; then + warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login" + elif command -v chpasswd &> /dev/null; then echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd elif command -v passwd &> /dev/null; then echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}" diff --git a/pkg/templates/testdata/docker_server.golden.sh b/pkg/templates/testdata/docker_server.golden.sh index f1248dc..0ff7b56 100644 --- a/pkg/templates/testdata/docker_server.golden.sh +++ b/pkg/templates/testdata/docker_server.golden.sh @@ -939,7 +939,9 @@ setup_user() { fi fi - if command -v chpasswd &> /dev/null; then + if [ -z "${_FIRST_USER_PASS}" ]; then + warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login" + elif command -v chpasswd &> /dev/null; then echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd elif command -v passwd &> /dev/null; then echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}" diff --git a/pkg/templates/testdata/server.golden.sh b/pkg/templates/testdata/server.golden.sh index 09af2cb..c690a7f 100644 --- a/pkg/templates/testdata/server.golden.sh +++ b/pkg/templates/testdata/server.golden.sh @@ -3152,7 +3152,9 @@ setup_user() { fi fi - if command -v chpasswd &> /dev/null; then + if [ -z "${_FIRST_USER_PASS}" ]; then + warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login" + elif command -v chpasswd &> /dev/null; then echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd elif command -v passwd &> /dev/null; then echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}" From 4743efecd15d9e22a1053a1d3d5851f478b77864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85slund?= Date: Fri, 16 Oct 2026 18:05:39 +0000 Subject: [PATCH 2/2] fix(bootstrap): re-runs keep the k3s token and the registry pull secret Re-running a tnctl server script on a device whose k3s has started took k3s down: tnctl generates a new token for every set of scripts, and k3s refuses to start with another one ("bootstrap data already found and encrypted with different token"). The script now takes the passphrase of the existing server/token. Without --docker-secret, teknoir-registry-secret got an empty .dockerconfigjson, which is invalid JSON: k3s rejected the Teknoir manifest in a loop (ApplyManifestFailed). The script now keeps the secret the device already has, or writes a valid empty config ({"auths":{}}). Verified on a fresh Debian 13 VM onboarded to staging with tnctl: first run, two re-runs (rc=0, k3s active, manifest applied, tnctl ssh through the reverse tunnel). Co-Authored-By: Claude Opus 5.5 --- pkg/templates/docker_server.sh.tmpl | 2 +- pkg/templates/k3stoken_test.go | 50 +++++++++++++++ pkg/templates/partials/manifests.yaml.tmpl | 4 +- .../partials/registry_secret.sh.tmpl | 13 ++++ pkg/templates/registrysecret_test.go | 63 +++++++++++++++++++ pkg/templates/server.sh.tmpl | 19 +++++- .../testdata/docker_server.golden.sh | 16 ++++- pkg/templates/testdata/server.golden.sh | 33 +++++++++- 8 files changed, 192 insertions(+), 8 deletions(-) create mode 100644 pkg/templates/k3stoken_test.go create mode 100644 pkg/templates/partials/registry_secret.sh.tmpl create mode 100644 pkg/templates/registrysecret_test.go diff --git a/pkg/templates/docker_server.sh.tmpl b/pkg/templates/docker_server.sh.tmpl index bbd1488..c2d59c8 100644 --- a/pkg/templates/docker_server.sh.tmpl +++ b/pkg/templates/docker_server.sh.tmpl @@ -44,7 +44,7 @@ disable: node-name: ${_DEVICE_ID} EOL -info "Install Teknoir Manifests" +{{template "registry_secret.sh.tmpl" .}}info "Install Teknoir Manifests" $SUDO mkdir -p /var/lib/rancher/k3s/server/manifests $SUDO tee /var/lib/rancher/k3s/server/manifests/teknoir.yaml > /dev/null << EOL {{template "manifests.yaml.tmpl" .}} diff --git a/pkg/templates/k3stoken_test.go b/pkg/templates/k3stoken_test.go new file mode 100644 index 0000000..66a3fff --- /dev/null +++ b/pkg/templates/k3stoken_test.go @@ -0,0 +1,50 @@ +package templates + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// A re-run on a device whose k3s has started before keeps the cluster token of that k3s: tnctl +// generates a new token for every set of scripts, and k3s refuses to start with another token +// than the one its datastore is encrypted with. +func TestReRunKeepsTheExistingK3sToken(t *testing.T) { + i := newInst(t) + i.installed("v1.36.5+k3s1") + srv := filepath.Join(i.dir, "data", "server") + if err := os.MkdirAll(srv, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(srv, "token"), []byte("K10abcdef::server:existingpass\n"), 0o600); err != nil { + t.Fatal(err) + } + out, err := i.runPrologue(t, `echo "EXEC=$INSTALL_K3S_EXEC"; echo "TOKEN=$_K3S_TOKEN"`) + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + if got, want := execOf(t, out), "server --token existingpass --tls-san dev-1.test.cloud --tls-san 10.0.0.10"; got != want { + t.Fatalf("INSTALL_K3S_EXEC = %q, want %q\n%s", got, want, out) + } + if !strings.Contains(out, "TOKEN=existingpass\n") { + t.Fatalf("_K3S_TOKEN not replaced:\n%s", out) + } + if !strings.Contains(out, "keeping its cluster token") || strings.Count(out, "existingpass") != 2 { + t.Fatalf("the script must say it keeps the token without printing it:\n%s", out) + } +} + +func TestFirstRunUsesTheGeneratedK3sToken(t *testing.T) { + i := newInst(t) + out, err := i.runPrologue(t, `echo "EXEC=$INSTALL_K3S_EXEC"`) + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + if got := execOf(t, out); got != baseExec { + t.Fatalf("INSTALL_K3S_EXEC = %q, want %q\n%s", got, baseExec, out) + } + if strings.Contains(out, "keeping its cluster token") { + t.Fatalf("a first run has no token to keep:\n%s", out) + } +} diff --git a/pkg/templates/partials/manifests.yaml.tmpl b/pkg/templates/partials/manifests.yaml.tmpl index 17c19b9..d0eeed9 100644 --- a/pkg/templates/partials/manifests.yaml.tmpl +++ b/pkg/templates/partials/manifests.yaml.tmpl @@ -13,7 +13,7 @@ metadata: namespace: kube-system type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "{{.DockerSecret}}" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 @@ -23,7 +23,7 @@ metadata: namespace: default type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "{{.DockerSecret}}" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 diff --git a/pkg/templates/partials/registry_secret.sh.tmpl b/pkg/templates/partials/registry_secret.sh.tmpl new file mode 100644 index 0000000..2b1a43a --- /dev/null +++ b/pkg/templates/partials/registry_secret.sh.tmpl @@ -0,0 +1,13 @@ +{{define "registry_secret.sh.tmpl"}}# The registry pull secret for teknoir-registry-secret (tnctl bootstrap --docker-secret). Without one, +# a re-run keeps what the device already has, and a first run gets an empty but valid config: an empty +# .dockerconfigjson is invalid JSON and makes k3s reject the whole Teknoir manifest. +_TN_DOCKER_SECRET="{{.DockerSecret}}" +if [ -z "${_TN_DOCKER_SECRET}" ]; then + _TN_DOCKER_SECRET=$($SUDO k3s kubectl -n default get secret teknoir-registry-secret -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null || true) + if [ -n "${_TN_DOCKER_SECRET}" ]; then + info "Keeping the registry pull secret the device already has" + else + _TN_DOCKER_SECRET=eyJhdXRocyI6e319 # {"auths":{}} + fi +fi +{{end}} diff --git a/pkg/templates/registrysecret_test.go b/pkg/templates/registrysecret_test.go new file mode 100644 index 0000000..54aae6b --- /dev/null +++ b/pkg/templates/registrysecret_test.go @@ -0,0 +1,63 @@ +package templates + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// registrySecretBlock renders the registry_secret partial for docker secret ds. +func registrySecretBlock(t *testing.T, ds string) string { + t.Helper() + d := goldenData() + d.DockerSecret = ds + var b strings.Builder + if err := ServerTemplate.ExecuteTemplate(&b, "registry_secret.sh.tmpl", d); err != nil { + t.Fatal(err) + } + return b.String() +} + +// runRegistrySecret runs the block with a fake k3s that prints existing (or fails when empty). +func runRegistrySecret(t *testing.T, ds, existing string) string { + t.Helper() + dir := t.TempDir() + fake := "#!/bin/sh\n[ -n \"" + existing + "\" ] || exit 1\nprintf '%s' '" + existing + "'\n" + if err := os.WriteFile(filepath.Join(dir, "k3s"), []byte(fake), 0o755); err != nil { + t.Fatal(err) + } + script := "set -e\ninfo() { echo \"[INFO] $*\"; }\nSUDO=\n" + registrySecretBlock(t, ds) + "echo \"SECRET=${_TN_DOCKER_SECRET}\"\n" + cmd := exec.Command("sh", "-c", script) + cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH")) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + return string(out) +} + +func TestRegistrySecret(t *testing.T) { + for _, tt := range []struct{ name, ds, existing, want string }{ + {"given", "Z2l2ZW4=", "ZXhpc3Rpbmc=", "SECRET=Z2l2ZW4=\n"}, + {"re-run keeps the device's", "", "ZXhpc3Rpbmc=", "SECRET=ZXhpc3Rpbmc=\n"}, + {"first run gets a valid empty config", "", "", "SECRET=eyJhdXRocyI6e319\n"}, + } { + t.Run(tt.name, func(t *testing.T) { + if out := runRegistrySecret(t, tt.ds, tt.existing); !strings.Contains(out, tt.want) { + t.Fatalf("want %q in:\n%s", tt.want, out) + } + }) + } +} + +func TestManifestsUseTheRegistrySecretVariable(t *testing.T) { + out := renderK3S(t, ServerTemplate, "") + if strings.Count(out, `.dockerconfigjson: "${_TN_DOCKER_SECRET}"`) != 2 { + t.Fatal("both teknoir-registry-secret objects must take ${_TN_DOCKER_SECRET}") + } + if strings.Index(out, "_TN_DOCKER_SECRET=\"") > strings.Index(out, "teknoir.yaml > /dev/null << EOL") { + t.Fatal("_TN_DOCKER_SECRET must be set before the manifests are written") + } +} diff --git a/pkg/templates/server.sh.tmpl b/pkg/templates/server.sh.tmpl index 2dff3f6..ebf6494 100644 --- a/pkg/templates/server.sh.tmpl +++ b/pkg/templates/server.sh.tmpl @@ -69,6 +69,23 @@ fi # END install-k3s-exec tn_ca_resolve_datadir tn_ca_require_consistent_datadir +# BEGIN keep-k3s-token +# A k3s that has started before keeps its cluster token: its datastore is encrypted with it, and +# k3s refuses to start with another one ("bootstrap data already found and encrypted with different +# token"). tnctl generates a new token for every set of scripts, so a re-run takes the passphrase of +# the existing server/token instead. Agents that join later need that token too (K3S_TOKEN). +if $SUDO test -s "${DATA_DIR}/server/token"; then + _tn_existing_token=$($SUDO cat "${DATA_DIR}/server/token") + _tn_existing_token=${_tn_existing_token##*:} + if [ -n "${_tn_existing_token}" ] && [ "${_tn_existing_token}" != "${_K3S_TOKEN}" ]; then + INSTALL_K3S_EXEC=$(printf '%s' "${INSTALL_K3S_EXEC}" | sed "s|--token ${_K3S_TOKEN}|--token ${_tn_existing_token}|") + export INSTALL_K3S_EXEC + _K3S_TOKEN=${_tn_existing_token} + info "k3s has started on this device before: keeping its cluster token (agents joining it need K3S_TOKEN from ${DATA_DIR}/server/token)" + fi + unset _tn_existing_token +fi +# END keep-k3s-token tn_inst_prepare server tn_k3s_ca_setup @@ -116,7 +133,7 @@ tn_inst_finish # https://:6443/static/teknoir/k3s-version. tn_inst_publish_release -info "Install Teknoir Manifests" +{{template "registry_secret.sh.tmpl" .}}info "Install Teknoir Manifests" $SUDO mkdir -p ${DATA_DIR}/server/manifests $SUDO tee ${DATA_DIR}/server/manifests/teknoir.yaml > /dev/null << EOL {{template "manifests.yaml.tmpl" .}} diff --git a/pkg/templates/testdata/docker_server.golden.sh b/pkg/templates/testdata/docker_server.golden.sh index 0ff7b56..1b89a6a 100644 --- a/pkg/templates/testdata/docker_server.golden.sh +++ b/pkg/templates/testdata/docker_server.golden.sh @@ -124,6 +124,18 @@ disable: node-name: ${_DEVICE_ID} EOL +# The registry pull secret for teknoir-registry-secret (tnctl bootstrap --docker-secret). Without one, +# a re-run keeps what the device already has, and a first run gets an empty but valid config: an empty +# .dockerconfigjson is invalid JSON and makes k3s reject the whole Teknoir manifest. +_TN_DOCKER_SECRET="TEST-ONLY-docker-secret" +if [ -z "${_TN_DOCKER_SECRET}" ]; then + _TN_DOCKER_SECRET=$($SUDO k3s kubectl -n default get secret teknoir-registry-secret -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null || true) + if [ -n "${_TN_DOCKER_SECRET}" ]; then + info "Keeping the registry pull secret the device already has" + else + _TN_DOCKER_SECRET=eyJhdXRocyI6e319 # {"auths":{}} + fi +fi info "Install Teknoir Manifests" $SUDO mkdir -p /var/lib/rancher/k3s/server/manifests $SUDO tee /var/lib/rancher/k3s/server/manifests/teknoir.yaml > /dev/null << EOL @@ -142,7 +154,7 @@ metadata: namespace: kube-system type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "TEST-ONLY-docker-secret" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 @@ -152,7 +164,7 @@ metadata: namespace: default type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "TEST-ONLY-docker-secret" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 diff --git a/pkg/templates/testdata/server.golden.sh b/pkg/templates/testdata/server.golden.sh index c690a7f..ea485d5 100644 --- a/pkg/templates/testdata/server.golden.sh +++ b/pkg/templates/testdata/server.golden.sh @@ -1968,6 +1968,23 @@ fi # END install-k3s-exec tn_ca_resolve_datadir tn_ca_require_consistent_datadir +# BEGIN keep-k3s-token +# A k3s that has started before keeps its cluster token: its datastore is encrypted with it, and +# k3s refuses to start with another one ("bootstrap data already found and encrypted with different +# token"). tnctl generates a new token for every set of scripts, so a re-run takes the passphrase of +# the existing server/token instead. Agents that join later need that token too (K3S_TOKEN). +if $SUDO test -s "${DATA_DIR}/server/token"; then + _tn_existing_token=$($SUDO cat "${DATA_DIR}/server/token") + _tn_existing_token=${_tn_existing_token##*:} + if [ -n "${_tn_existing_token}" ] && [ "${_tn_existing_token}" != "${_K3S_TOKEN}" ]; then + INSTALL_K3S_EXEC=$(printf '%s' "${INSTALL_K3S_EXEC}" | sed "s|--token ${_K3S_TOKEN}|--token ${_tn_existing_token}|") + export INSTALL_K3S_EXEC + _K3S_TOKEN=${_tn_existing_token} + info "k3s has started on this device before: keeping its cluster token (agents joining it need K3S_TOKEN from ${DATA_DIR}/server/token)" + fi + unset _tn_existing_token +fi +# END keep-k3s-token tn_inst_prepare server tn_k3s_ca_setup @@ -2015,6 +2032,18 @@ tn_inst_finish # https://:6443/static/teknoir/k3s-version. tn_inst_publish_release +# The registry pull secret for teknoir-registry-secret (tnctl bootstrap --docker-secret). Without one, +# a re-run keeps what the device already has, and a first run gets an empty but valid config: an empty +# .dockerconfigjson is invalid JSON and makes k3s reject the whole Teknoir manifest. +_TN_DOCKER_SECRET="TEST-ONLY-docker-secret" +if [ -z "${_TN_DOCKER_SECRET}" ]; then + _TN_DOCKER_SECRET=$($SUDO k3s kubectl -n default get secret teknoir-registry-secret -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null || true) + if [ -n "${_TN_DOCKER_SECRET}" ]; then + info "Keeping the registry pull secret the device already has" + else + _TN_DOCKER_SECRET=eyJhdXRocyI6e319 # {"auths":{}} + fi +fi info "Install Teknoir Manifests" $SUDO mkdir -p ${DATA_DIR}/server/manifests $SUDO tee ${DATA_DIR}/server/manifests/teknoir.yaml > /dev/null << EOL @@ -2033,7 +2062,7 @@ metadata: namespace: kube-system type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "TEST-ONLY-docker-secret" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 @@ -2043,7 +2072,7 @@ metadata: namespace: default type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "TEST-ONLY-docker-secret" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1