diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index ab7f9a2..570d6c5 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -86,6 +86,12 @@ var bootstrapCmd = &cobra.Command{ if err != nil { return err } + // The device-controller generates the key pair after the Device is created, and the + // Backstage catalog picks it up on its next device refresh (every few minutes). Scripts + // without it install empty keys and fail at the k3s CA step on the device. + if rsaPrivate == "" || rsaPublic == "" { + return fmt.Errorf("device %s/%s has no key pair in the catalog yet (the platform adds it a few minutes after the device is created); run tnctl bootstrap again shortly", namespace, deviceName) + } username, err := decode("username", device.Spec.Settings.Username) if err != nil { return err diff --git a/pkg/templates/docker_server.sh.tmpl b/pkg/templates/docker_server.sh.tmpl index bbd1488..c2d59c8 100644 --- a/pkg/templates/docker_server.sh.tmpl +++ b/pkg/templates/docker_server.sh.tmpl @@ -44,7 +44,7 @@ disable: node-name: ${_DEVICE_ID} EOL -info "Install Teknoir Manifests" +{{template "registry_secret.sh.tmpl" .}}info "Install Teknoir Manifests" $SUDO mkdir -p /var/lib/rancher/k3s/server/manifests $SUDO tee /var/lib/rancher/k3s/server/manifests/teknoir.yaml > /dev/null << EOL {{template "manifests.yaml.tmpl" .}} diff --git a/pkg/templates/k3stoken_test.go b/pkg/templates/k3stoken_test.go new file mode 100644 index 0000000..66a3fff --- /dev/null +++ b/pkg/templates/k3stoken_test.go @@ -0,0 +1,50 @@ +package templates + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// A re-run on a device whose k3s has started before keeps the cluster token of that k3s: tnctl +// generates a new token for every set of scripts, and k3s refuses to start with another token +// than the one its datastore is encrypted with. +func TestReRunKeepsTheExistingK3sToken(t *testing.T) { + i := newInst(t) + i.installed("v1.36.5+k3s1") + srv := filepath.Join(i.dir, "data", "server") + if err := os.MkdirAll(srv, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(srv, "token"), []byte("K10abcdef::server:existingpass\n"), 0o600); err != nil { + t.Fatal(err) + } + out, err := i.runPrologue(t, `echo "EXEC=$INSTALL_K3S_EXEC"; echo "TOKEN=$_K3S_TOKEN"`) + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + if got, want := execOf(t, out), "server --token existingpass --tls-san dev-1.test.cloud --tls-san 10.0.0.10"; got != want { + t.Fatalf("INSTALL_K3S_EXEC = %q, want %q\n%s", got, want, out) + } + if !strings.Contains(out, "TOKEN=existingpass\n") { + t.Fatalf("_K3S_TOKEN not replaced:\n%s", out) + } + if !strings.Contains(out, "keeping its cluster token") || strings.Count(out, "existingpass") != 2 { + t.Fatalf("the script must say it keeps the token without printing it:\n%s", out) + } +} + +func TestFirstRunUsesTheGeneratedK3sToken(t *testing.T) { + i := newInst(t) + out, err := i.runPrologue(t, `echo "EXEC=$INSTALL_K3S_EXEC"`) + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + if got := execOf(t, out); got != baseExec { + t.Fatalf("INSTALL_K3S_EXEC = %q, want %q\n%s", got, baseExec, out) + } + if strings.Contains(out, "keeping its cluster token") { + t.Fatalf("a first run has no token to keep:\n%s", out) + } +} diff --git a/pkg/templates/partials/manifests.yaml.tmpl b/pkg/templates/partials/manifests.yaml.tmpl index 17c19b9..d0eeed9 100644 --- a/pkg/templates/partials/manifests.yaml.tmpl +++ b/pkg/templates/partials/manifests.yaml.tmpl @@ -13,7 +13,7 @@ metadata: namespace: kube-system type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "{{.DockerSecret}}" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 @@ -23,7 +23,7 @@ metadata: namespace: default type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "{{.DockerSecret}}" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 diff --git a/pkg/templates/partials/registry_secret.sh.tmpl b/pkg/templates/partials/registry_secret.sh.tmpl new file mode 100644 index 0000000..2b1a43a --- /dev/null +++ b/pkg/templates/partials/registry_secret.sh.tmpl @@ -0,0 +1,13 @@ +{{define "registry_secret.sh.tmpl"}}# The registry pull secret for teknoir-registry-secret (tnctl bootstrap --docker-secret). Without one, +# a re-run keeps what the device already has, and a first run gets an empty but valid config: an empty +# .dockerconfigjson is invalid JSON and makes k3s reject the whole Teknoir manifest. +_TN_DOCKER_SECRET="{{.DockerSecret}}" +if [ -z "${_TN_DOCKER_SECRET}" ]; then + _TN_DOCKER_SECRET=$($SUDO k3s kubectl -n default get secret teknoir-registry-secret -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null || true) + if [ -n "${_TN_DOCKER_SECRET}" ]; then + info "Keeping the registry pull secret the device already has" + else + _TN_DOCKER_SECRET=eyJhdXRocyI6e319 # {"auths":{}} + fi +fi +{{end}} diff --git a/pkg/templates/partials/user_setup.sh.tmpl b/pkg/templates/partials/user_setup.sh.tmpl index de8a0da..d7646e2 100644 --- a/pkg/templates/partials/user_setup.sh.tmpl +++ b/pkg/templates/partials/user_setup.sh.tmpl @@ -20,7 +20,9 @@ setup_user() { fi fi - if command -v chpasswd &> /dev/null; then + if [ -z "${_FIRST_USER_PASS}" ]; then + warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login" + elif command -v chpasswd &> /dev/null; then echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd elif command -v passwd &> /dev/null; then echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}" diff --git a/pkg/templates/registrysecret_test.go b/pkg/templates/registrysecret_test.go new file mode 100644 index 0000000..54aae6b --- /dev/null +++ b/pkg/templates/registrysecret_test.go @@ -0,0 +1,63 @@ +package templates + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// registrySecretBlock renders the registry_secret partial for docker secret ds. +func registrySecretBlock(t *testing.T, ds string) string { + t.Helper() + d := goldenData() + d.DockerSecret = ds + var b strings.Builder + if err := ServerTemplate.ExecuteTemplate(&b, "registry_secret.sh.tmpl", d); err != nil { + t.Fatal(err) + } + return b.String() +} + +// runRegistrySecret runs the block with a fake k3s that prints existing (or fails when empty). +func runRegistrySecret(t *testing.T, ds, existing string) string { + t.Helper() + dir := t.TempDir() + fake := "#!/bin/sh\n[ -n \"" + existing + "\" ] || exit 1\nprintf '%s' '" + existing + "'\n" + if err := os.WriteFile(filepath.Join(dir, "k3s"), []byte(fake), 0o755); err != nil { + t.Fatal(err) + } + script := "set -e\ninfo() { echo \"[INFO] $*\"; }\nSUDO=\n" + registrySecretBlock(t, ds) + "echo \"SECRET=${_TN_DOCKER_SECRET}\"\n" + cmd := exec.Command("sh", "-c", script) + cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH")) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + return string(out) +} + +func TestRegistrySecret(t *testing.T) { + for _, tt := range []struct{ name, ds, existing, want string }{ + {"given", "Z2l2ZW4=", "ZXhpc3Rpbmc=", "SECRET=Z2l2ZW4=\n"}, + {"re-run keeps the device's", "", "ZXhpc3Rpbmc=", "SECRET=ZXhpc3Rpbmc=\n"}, + {"first run gets a valid empty config", "", "", "SECRET=eyJhdXRocyI6e319\n"}, + } { + t.Run(tt.name, func(t *testing.T) { + if out := runRegistrySecret(t, tt.ds, tt.existing); !strings.Contains(out, tt.want) { + t.Fatalf("want %q in:\n%s", tt.want, out) + } + }) + } +} + +func TestManifestsUseTheRegistrySecretVariable(t *testing.T) { + out := renderK3S(t, ServerTemplate, "") + if strings.Count(out, `.dockerconfigjson: "${_TN_DOCKER_SECRET}"`) != 2 { + t.Fatal("both teknoir-registry-secret objects must take ${_TN_DOCKER_SECRET}") + } + if strings.Index(out, "_TN_DOCKER_SECRET=\"") > strings.Index(out, "teknoir.yaml > /dev/null << EOL") { + t.Fatal("_TN_DOCKER_SECRET must be set before the manifests are written") + } +} diff --git a/pkg/templates/server.sh.tmpl b/pkg/templates/server.sh.tmpl index 2dff3f6..ebf6494 100644 --- a/pkg/templates/server.sh.tmpl +++ b/pkg/templates/server.sh.tmpl @@ -69,6 +69,23 @@ fi # END install-k3s-exec tn_ca_resolve_datadir tn_ca_require_consistent_datadir +# BEGIN keep-k3s-token +# A k3s that has started before keeps its cluster token: its datastore is encrypted with it, and +# k3s refuses to start with another one ("bootstrap data already found and encrypted with different +# token"). tnctl generates a new token for every set of scripts, so a re-run takes the passphrase of +# the existing server/token instead. Agents that join later need that token too (K3S_TOKEN). +if $SUDO test -s "${DATA_DIR}/server/token"; then + _tn_existing_token=$($SUDO cat "${DATA_DIR}/server/token") + _tn_existing_token=${_tn_existing_token##*:} + if [ -n "${_tn_existing_token}" ] && [ "${_tn_existing_token}" != "${_K3S_TOKEN}" ]; then + INSTALL_K3S_EXEC=$(printf '%s' "${INSTALL_K3S_EXEC}" | sed "s|--token ${_K3S_TOKEN}|--token ${_tn_existing_token}|") + export INSTALL_K3S_EXEC + _K3S_TOKEN=${_tn_existing_token} + info "k3s has started on this device before: keeping its cluster token (agents joining it need K3S_TOKEN from ${DATA_DIR}/server/token)" + fi + unset _tn_existing_token +fi +# END keep-k3s-token tn_inst_prepare server tn_k3s_ca_setup @@ -116,7 +133,7 @@ tn_inst_finish # https://:6443/static/teknoir/k3s-version. tn_inst_publish_release -info "Install Teknoir Manifests" +{{template "registry_secret.sh.tmpl" .}}info "Install Teknoir Manifests" $SUDO mkdir -p ${DATA_DIR}/server/manifests $SUDO tee ${DATA_DIR}/server/manifests/teknoir.yaml > /dev/null << EOL {{template "manifests.yaml.tmpl" .}} diff --git a/pkg/templates/testdata/agent.golden.sh b/pkg/templates/testdata/agent.golden.sh index 7cd79e3..01b207b 100644 --- a/pkg/templates/testdata/agent.golden.sh +++ b/pkg/templates/testdata/agent.golden.sh @@ -836,7 +836,9 @@ setup_user() { fi fi - if command -v chpasswd &> /dev/null; then + if [ -z "${_FIRST_USER_PASS}" ]; then + warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login" + elif command -v chpasswd &> /dev/null; then echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd elif command -v passwd &> /dev/null; then echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}" diff --git a/pkg/templates/testdata/docker_server.golden.sh b/pkg/templates/testdata/docker_server.golden.sh index f1248dc..1b89a6a 100644 --- a/pkg/templates/testdata/docker_server.golden.sh +++ b/pkg/templates/testdata/docker_server.golden.sh @@ -124,6 +124,18 @@ disable: node-name: ${_DEVICE_ID} EOL +# The registry pull secret for teknoir-registry-secret (tnctl bootstrap --docker-secret). Without one, +# a re-run keeps what the device already has, and a first run gets an empty but valid config: an empty +# .dockerconfigjson is invalid JSON and makes k3s reject the whole Teknoir manifest. +_TN_DOCKER_SECRET="TEST-ONLY-docker-secret" +if [ -z "${_TN_DOCKER_SECRET}" ]; then + _TN_DOCKER_SECRET=$($SUDO k3s kubectl -n default get secret teknoir-registry-secret -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null || true) + if [ -n "${_TN_DOCKER_SECRET}" ]; then + info "Keeping the registry pull secret the device already has" + else + _TN_DOCKER_SECRET=eyJhdXRocyI6e319 # {"auths":{}} + fi +fi info "Install Teknoir Manifests" $SUDO mkdir -p /var/lib/rancher/k3s/server/manifests $SUDO tee /var/lib/rancher/k3s/server/manifests/teknoir.yaml > /dev/null << EOL @@ -142,7 +154,7 @@ metadata: namespace: kube-system type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "TEST-ONLY-docker-secret" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 @@ -152,7 +164,7 @@ metadata: namespace: default type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "TEST-ONLY-docker-secret" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 @@ -939,7 +951,9 @@ setup_user() { fi fi - if command -v chpasswd &> /dev/null; then + if [ -z "${_FIRST_USER_PASS}" ]; then + warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login" + elif command -v chpasswd &> /dev/null; then echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd elif command -v passwd &> /dev/null; then echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}" diff --git a/pkg/templates/testdata/server.golden.sh b/pkg/templates/testdata/server.golden.sh index 09af2cb..ea485d5 100644 --- a/pkg/templates/testdata/server.golden.sh +++ b/pkg/templates/testdata/server.golden.sh @@ -1968,6 +1968,23 @@ fi # END install-k3s-exec tn_ca_resolve_datadir tn_ca_require_consistent_datadir +# BEGIN keep-k3s-token +# A k3s that has started before keeps its cluster token: its datastore is encrypted with it, and +# k3s refuses to start with another one ("bootstrap data already found and encrypted with different +# token"). tnctl generates a new token for every set of scripts, so a re-run takes the passphrase of +# the existing server/token instead. Agents that join later need that token too (K3S_TOKEN). +if $SUDO test -s "${DATA_DIR}/server/token"; then + _tn_existing_token=$($SUDO cat "${DATA_DIR}/server/token") + _tn_existing_token=${_tn_existing_token##*:} + if [ -n "${_tn_existing_token}" ] && [ "${_tn_existing_token}" != "${_K3S_TOKEN}" ]; then + INSTALL_K3S_EXEC=$(printf '%s' "${INSTALL_K3S_EXEC}" | sed "s|--token ${_K3S_TOKEN}|--token ${_tn_existing_token}|") + export INSTALL_K3S_EXEC + _K3S_TOKEN=${_tn_existing_token} + info "k3s has started on this device before: keeping its cluster token (agents joining it need K3S_TOKEN from ${DATA_DIR}/server/token)" + fi + unset _tn_existing_token +fi +# END keep-k3s-token tn_inst_prepare server tn_k3s_ca_setup @@ -2015,6 +2032,18 @@ tn_inst_finish # https://:6443/static/teknoir/k3s-version. tn_inst_publish_release +# The registry pull secret for teknoir-registry-secret (tnctl bootstrap --docker-secret). Without one, +# a re-run keeps what the device already has, and a first run gets an empty but valid config: an empty +# .dockerconfigjson is invalid JSON and makes k3s reject the whole Teknoir manifest. +_TN_DOCKER_SECRET="TEST-ONLY-docker-secret" +if [ -z "${_TN_DOCKER_SECRET}" ]; then + _TN_DOCKER_SECRET=$($SUDO k3s kubectl -n default get secret teknoir-registry-secret -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null || true) + if [ -n "${_TN_DOCKER_SECRET}" ]; then + info "Keeping the registry pull secret the device already has" + else + _TN_DOCKER_SECRET=eyJhdXRocyI6e319 # {"auths":{}} + fi +fi info "Install Teknoir Manifests" $SUDO mkdir -p ${DATA_DIR}/server/manifests $SUDO tee ${DATA_DIR}/server/manifests/teknoir.yaml > /dev/null << EOL @@ -2033,7 +2062,7 @@ metadata: namespace: kube-system type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "TEST-ONLY-docker-secret" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 @@ -2043,7 +2072,7 @@ metadata: namespace: default type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: "TEST-ONLY-docker-secret" + .dockerconfigjson: "${_TN_DOCKER_SECRET}" --- apiVersion: v1 @@ -3152,7 +3181,9 @@ setup_user() { fi fi - if command -v chpasswd &> /dev/null; then + if [ -z "${_FIRST_USER_PASS}" ]; then + warn "No password for ${_FIRST_USER_NAME} in the platform: key-only login" + elif command -v chpasswd &> /dev/null; then echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd elif command -v passwd &> /dev/null; then echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}"