fix(bootstrap): onboarding and re-runs with tnctl (keys, k3s token, pull secret)

cmd/bootstrap.go

@@ -86,6 +86,12 @@ var bootstrapCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
+		// The device-controller generates the key pair after the Device is created, and the
+		// Backstage catalog picks it up on its next device refresh (every few minutes). Scripts
+		// without it install empty keys and fail at the k3s CA step on the device.
+		if rsaPrivate == "" || rsaPublic == "" {
+			return fmt.Errorf("device %s/%s has no key pair in the catalog yet (the platform adds it a few minutes after the device is created); run tnctl bootstrap again shortly", namespace, deviceName)
+		}
 		username, err := decode("username", device.Spec.Settings.Username)
 		if err != nil {
 			return err

pkg/templates/docker_server.sh.tmpl

@@ -44,7 +44,7 @@ disable:
 node-name: ${_DEVICE_ID}
 EOL
 
-info "Install Teknoir Manifests"
+{{template "registry_secret.sh.tmpl" .}}info "Install Teknoir Manifests"
 $SUDO mkdir -p /var/lib/rancher/k3s/server/manifests
 $SUDO tee /var/lib/rancher/k3s/server/manifests/teknoir.yaml > /dev/null << EOL
 {{template "manifests.yaml.tmpl" .}}

pkg/templates/k3stoken_test.go

@@ -0,0 +1,50 @@
+package templates
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// A re-run on a device whose k3s has started before keeps the cluster token of that k3s: tnctl
+// generates a new token for every set of scripts, and k3s refuses to start with another token
+// than the one its datastore is encrypted with.
+func TestReRunKeepsTheExistingK3sToken(t *testing.T) {
+	i := newInst(t)
+	i.installed("v1.36.5+k3s1")
+	srv := filepath.Join(i.dir, "data", "server")
+	if err := os.MkdirAll(srv, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(srv, "token"), []byte("K10abcdef::server:existingpass\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	out, err := i.runPrologue(t, `echo "EXEC=$INSTALL_K3S_EXEC"; echo "TOKEN=$_K3S_TOKEN"`)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	if got, want := execOf(t, out), "server --token existingpass --tls-san dev-1.test.cloud --tls-san 10.0.0.10"; got != want {
+		t.Fatalf("INSTALL_K3S_EXEC = %q, want %q\n%s", got, want, out)
+	}
+	if !strings.Contains(out, "TOKEN=existingpass\n") {
+		t.Fatalf("_K3S_TOKEN not replaced:\n%s", out)
+	}
+	if !strings.Contains(out, "keeping its cluster token") || strings.Count(out, "existingpass") != 2 {
+		t.Fatalf("the script must say it keeps the token without printing it:\n%s", out)
+	}
+}
+
+func TestFirstRunUsesTheGeneratedK3sToken(t *testing.T) {
+	i := newInst(t)
+	out, err := i.runPrologue(t, `echo "EXEC=$INSTALL_K3S_EXEC"`)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	if got := execOf(t, out); got != baseExec {
+		t.Fatalf("INSTALL_K3S_EXEC = %q, want %q\n%s", got, baseExec, out)
+	}
+	if strings.Contains(out, "keeping its cluster token") {
+		t.Fatalf("a first run has no token to keep:\n%s", out)
+	}
+}

pkg/templates/partials/manifests.yaml.tmpl

@@ -13,7 +13,7 @@ metadata:
   namespace: kube-system
 type: kubernetes.io/dockerconfigjson
 data:
-  .dockerconfigjson: "{{.DockerSecret}}"
+  .dockerconfigjson: "${_TN_DOCKER_SECRET}"
 
 ---
 apiVersion: v1
@@ -23,7 +23,7 @@ metadata:
   namespace: default
 type: kubernetes.io/dockerconfigjson
 data:
-  .dockerconfigjson: "{{.DockerSecret}}"
+  .dockerconfigjson: "${_TN_DOCKER_SECRET}"
 
 ---
 apiVersion: v1

pkg/templates/partials/registry_secret.sh.tmpl

@@ -0,0 +1,13 @@
+{{define "registry_secret.sh.tmpl"}}# The registry pull secret for teknoir-registry-secret (tnctl bootstrap --docker-secret). Without one,
+# a re-run keeps what the device already has, and a first run gets an empty but valid config: an empty
+# .dockerconfigjson is invalid JSON and makes k3s reject the whole Teknoir manifest.
+_TN_DOCKER_SECRET="{{.DockerSecret}}"
+if [ -z "${_TN_DOCKER_SECRET}" ]; then
+    _TN_DOCKER_SECRET=$($SUDO k3s kubectl -n default get secret teknoir-registry-secret -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null || true)
+    if [ -n "${_TN_DOCKER_SECRET}" ]; then
+        info "Keeping the registry pull secret the device already has"
+    else
+        _TN_DOCKER_SECRET=eyJhdXRocyI6e319 # {"auths":{}}
+    fi
+fi
+{{end}}

pkg/templates/partials/user_setup.sh.tmpl

@@ -20,7 +20,9 @@ setup_user() {
     fi
   fi
 
-  if command -v chpasswd &> /dev/null; then
+  if [ -z "${_FIRST_USER_PASS}" ]; then
+    warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login"
+  elif command -v chpasswd &> /dev/null; then
     echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd
   elif command -v passwd &> /dev/null; then
     echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}"

pkg/templates/registrysecret_test.go

@@ -0,0 +1,63 @@
+package templates
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// registrySecretBlock renders the registry_secret partial for docker secret ds.
+func registrySecretBlock(t *testing.T, ds string) string {
+	t.Helper()
+	d := goldenData()
+	d.DockerSecret = ds
+	var b strings.Builder
+	if err := ServerTemplate.ExecuteTemplate(&b, "registry_secret.sh.tmpl", d); err != nil {
+		t.Fatal(err)
+	}
+	return b.String()
+}
+
+// runRegistrySecret runs the block with a fake k3s that prints existing (or fails when empty).
+func runRegistrySecret(t *testing.T, ds, existing string) string {
+	t.Helper()
+	dir := t.TempDir()
+	fake := "#!/bin/sh\n[ -n \"" + existing + "\" ] || exit 1\nprintf '%s' '" + existing + "'\n"
+	if err := os.WriteFile(filepath.Join(dir, "k3s"), []byte(fake), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	script := "set -e\ninfo() { echo \"[INFO] $*\"; }\nSUDO=\n" + registrySecretBlock(t, ds) + "echo \"SECRET=${_TN_DOCKER_SECRET}\"\n"
+	cmd := exec.Command("sh", "-c", script)
+	cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH"))
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	return string(out)
+}
+
+func TestRegistrySecret(t *testing.T) {
+	for _, tt := range []struct{ name, ds, existing, want string }{
+		{"given", "Z2l2ZW4=", "ZXhpc3Rpbmc=", "SECRET=Z2l2ZW4=\n"},
+		{"re-run keeps the device's", "", "ZXhpc3Rpbmc=", "SECRET=ZXhpc3Rpbmc=\n"},
+		{"first run gets a valid empty config", "", "", "SECRET=eyJhdXRocyI6e319\n"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			if out := runRegistrySecret(t, tt.ds, tt.existing); !strings.Contains(out, tt.want) {
+				t.Fatalf("want %q in:\n%s", tt.want, out)
+			}
+		})
+	}
+}
+
+func TestManifestsUseTheRegistrySecretVariable(t *testing.T) {
+	out := renderK3S(t, ServerTemplate, "")
+	if strings.Count(out, `.dockerconfigjson: "${_TN_DOCKER_SECRET}"`) != 2 {
+		t.Fatal("both teknoir-registry-secret objects must take ${_TN_DOCKER_SECRET}")
+	}
+	if strings.Index(out, "_TN_DOCKER_SECRET=\"") > strings.Index(out, "teknoir.yaml > /dev/null << EOL") {
+		t.Fatal("_TN_DOCKER_SECRET must be set before the manifests are written")
+	}
+}

pkg/templates/server.sh.tmpl

@@ -69,6 +69,23 @@ fi
 # END install-k3s-exec
 tn_ca_resolve_datadir
 tn_ca_require_consistent_datadir
+# BEGIN keep-k3s-token
+# A k3s that has started before keeps its cluster token: its datastore is encrypted with it, and
+# k3s refuses to start with another one ("bootstrap data already found and encrypted with different
+# token"). tnctl generates a new token for every set of scripts, so a re-run takes the passphrase of
+# the existing server/token instead. Agents that join later need that token too (K3S_TOKEN).
+if $SUDO test -s "${DATA_DIR}/server/token"; then
+    _tn_existing_token=$($SUDO cat "${DATA_DIR}/server/token")
+    _tn_existing_token=${_tn_existing_token##*:}
+    if [ -n "${_tn_existing_token}" ] && [ "${_tn_existing_token}" != "${_K3S_TOKEN}" ]; then
+        INSTALL_K3S_EXEC=$(printf '%s' "${INSTALL_K3S_EXEC}" | sed "s|--token ${_K3S_TOKEN}|--token ${_tn_existing_token}|")
+        export INSTALL_K3S_EXEC
+        _K3S_TOKEN=${_tn_existing_token}
+        info "k3s has started on this device before: keeping its cluster token (agents joining it need K3S_TOKEN from ${DATA_DIR}/server/token)"
+    fi
+    unset _tn_existing_token
+fi
+# END keep-k3s-token
 tn_inst_prepare server
 tn_k3s_ca_setup
 
@@ -116,7 +133,7 @@ tn_inst_finish
 # https://<server>:6443/static/teknoir/k3s-version.
 tn_inst_publish_release
 
-info "Install Teknoir Manifests"
+{{template "registry_secret.sh.tmpl" .}}info "Install Teknoir Manifests"
 $SUDO mkdir -p ${DATA_DIR}/server/manifests
 $SUDO tee ${DATA_DIR}/server/manifests/teknoir.yaml > /dev/null << EOL
 {{template "manifests.yaml.tmpl" .}}

pkg/templates/testdata/agent.golden.sh

@@ -836,7 +836,9 @@ setup_user() {
     fi
   fi
 
-  if command -v chpasswd &> /dev/null; then
+  if [ -z "${_FIRST_USER_PASS}" ]; then
+    warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login"
+  elif command -v chpasswd &> /dev/null; then
     echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd
   elif command -v passwd &> /dev/null; then
     echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}"

pkg/templates/testdata/docker_server.golden.sh

@@ -124,6 +124,18 @@ disable:
 node-name: ${_DEVICE_ID}
 EOL
 
+# The registry pull secret for teknoir-registry-secret (tnctl bootstrap --docker-secret). Without one,
+# a re-run keeps what the device already has, and a first run gets an empty but valid config: an empty
+# .dockerconfigjson is invalid JSON and makes k3s reject the whole Teknoir manifest.
+_TN_DOCKER_SECRET="TEST-ONLY-docker-secret"
+if [ -z "${_TN_DOCKER_SECRET}" ]; then
+    _TN_DOCKER_SECRET=$($SUDO k3s kubectl -n default get secret teknoir-registry-secret -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null || true)
+    if [ -n "${_TN_DOCKER_SECRET}" ]; then
+        info "Keeping the registry pull secret the device already has"
+    else
+        _TN_DOCKER_SECRET=eyJhdXRocyI6e319 # {"auths":{}}
+    fi
+fi
 info "Install Teknoir Manifests"
 $SUDO mkdir -p /var/lib/rancher/k3s/server/manifests
 $SUDO tee /var/lib/rancher/k3s/server/manifests/teknoir.yaml > /dev/null << EOL
@@ -142,7 +154,7 @@ metadata:
   namespace: kube-system
 type: kubernetes.io/dockerconfigjson
 data:
-  .dockerconfigjson: "TEST-ONLY-docker-secret"
+  .dockerconfigjson: "${_TN_DOCKER_SECRET}"
 
 ---
 apiVersion: v1
@@ -152,7 +164,7 @@ metadata:
   namespace: default
 type: kubernetes.io/dockerconfigjson
 data:
-  .dockerconfigjson: "TEST-ONLY-docker-secret"
+  .dockerconfigjson: "${_TN_DOCKER_SECRET}"
 
 ---
 apiVersion: v1
@@ -939,7 +951,9 @@ setup_user() {
     fi
   fi
 
-  if command -v chpasswd &> /dev/null; then
+  if [ -z "${_FIRST_USER_PASS}" ]; then
+    warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login"
+  elif command -v chpasswd &> /dev/null; then
     echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd
   elif command -v passwd &> /dev/null; then
     echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}"

pkg/templates/testdata/server.golden.sh

@@ -1968,6 +1968,23 @@ fi
 # END install-k3s-exec
 tn_ca_resolve_datadir
 tn_ca_require_consistent_datadir
+# BEGIN keep-k3s-token
+# A k3s that has started before keeps its cluster token: its datastore is encrypted with it, and
+# k3s refuses to start with another one ("bootstrap data already found and encrypted with different
+# token"). tnctl generates a new token for every set of scripts, so a re-run takes the passphrase of
+# the existing server/token instead. Agents that join later need that token too (K3S_TOKEN).
+if $SUDO test -s "${DATA_DIR}/server/token"; then
+    _tn_existing_token=$($SUDO cat "${DATA_DIR}/server/token")
+    _tn_existing_token=${_tn_existing_token##*:}
+    if [ -n "${_tn_existing_token}" ] && [ "${_tn_existing_token}" != "${_K3S_TOKEN}" ]; then
+        INSTALL_K3S_EXEC=$(printf '%s' "${INSTALL_K3S_EXEC}" | sed "s|--token ${_K3S_TOKEN}|--token ${_tn_existing_token}|")
+        export INSTALL_K3S_EXEC
+        _K3S_TOKEN=${_tn_existing_token}
+        info "k3s has started on this device before: keeping its cluster token (agents joining it need K3S_TOKEN from ${DATA_DIR}/server/token)"
+    fi
+    unset _tn_existing_token
+fi
+# END keep-k3s-token
 tn_inst_prepare server
 tn_k3s_ca_setup
 
@@ -2015,6 +2032,18 @@ tn_inst_finish
 # https://<server>:6443/static/teknoir/k3s-version.
 tn_inst_publish_release
 
+# The registry pull secret for teknoir-registry-secret (tnctl bootstrap --docker-secret). Without one,
+# a re-run keeps what the device already has, and a first run gets an empty but valid config: an empty
+# .dockerconfigjson is invalid JSON and makes k3s reject the whole Teknoir manifest.
+_TN_DOCKER_SECRET="TEST-ONLY-docker-secret"
+if [ -z "${_TN_DOCKER_SECRET}" ]; then
+    _TN_DOCKER_SECRET=$($SUDO k3s kubectl -n default get secret teknoir-registry-secret -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null || true)
+    if [ -n "${_TN_DOCKER_SECRET}" ]; then
+        info "Keeping the registry pull secret the device already has"
+    else
+        _TN_DOCKER_SECRET=eyJhdXRocyI6e319 # {"auths":{}}
+    fi
+fi
 info "Install Teknoir Manifests"
 $SUDO mkdir -p ${DATA_DIR}/server/manifests
 $SUDO tee ${DATA_DIR}/server/manifests/teknoir.yaml > /dev/null << EOL
@@ -2033,7 +2062,7 @@ metadata:
   namespace: kube-system
 type: kubernetes.io/dockerconfigjson
 data:
-  .dockerconfigjson: "TEST-ONLY-docker-secret"
+  .dockerconfigjson: "${_TN_DOCKER_SECRET}"
 
 ---
 apiVersion: v1
@@ -2043,7 +2072,7 @@ metadata:
   namespace: default
 type: kubernetes.io/dockerconfigjson
 data:
-  .dockerconfigjson: "TEST-ONLY-docker-secret"
+  .dockerconfigjson: "${_TN_DOCKER_SECRET}"
 
 ---
 apiVersion: v1
@@ -3152,7 +3181,9 @@ setup_user() {
     fi
   fi
 
-  if command -v chpasswd &> /dev/null; then
+  if [ -z "${_FIRST_USER_PASS}" ]; then
+    warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login"
+  elif command -v chpasswd &> /dev/null; then
     echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd
   elif command -v passwd &> /dev/null; then
     echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}"