From 4fec3348f9787ffc760e8f9c523a813e422d0188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85slund?= Date: Fri, 16 Oct 2026 19:12:22 +0000 Subject: [PATCH] tnctl bootstrap: node roles, labels, taints, --server and --tls-san The role of a computer in a robot is a set of k3s node labels, and k3s applies node-label/node-taint only when a node registers, so it has to be chosen at bootstrap. - New flags: --robot, --role, --node-name, --label, --taint (server script), --agent-role/--agent-node-name/--agent-label/--agent-taint (agent script), --server (K3S_URL=https://:6443 in the agent script, env still wins; also a TLS SAN of the server) and --tls-san. One run still produces server and agent scripts with one token. Values are validated (DNS labels, label and taint syntax); the kubernetes.io/ and k8s.io/ prefixes are refused. - New scripts/tn-k3s-node.sh, embedded only when a node flag is given: writes node-name to config.yaml when absent, and node-label+/node-taint+ to the drop-in config.yaml.d/98-teknoir-node.yaml (atomic, rewritten only when it differs, never touches keys the owner set). On a node that has registered already it warns and prints the kubectl label/taint commands. - Without the new flags the scripts are byte-identical (existing goldens). - Tests: validation, rendering, new roles goldens for server and agent, shell tests that run the config writer twice against a temp dir. - docs/commands/bootstrap.md: "Robots with several computers". Co-Authored-By: Claude Opus 5.5 --- .github/workflows/k3s-ca.yml | 2 +- README.md | 1 + cmd/bootstrap.go | 20 +- cmd/bootstrap_node.go | 162 + cmd/bootstrap_test.go | 162 + docs/commands/bootstrap.md | 109 + pkg/templates/agent.sh.tmpl | 24 +- pkg/templates/bootstrap_templates.go | 17 +- pkg/templates/node.go | 292 ++ pkg/templates/node_test.go | 726 ++++ pkg/templates/render.go | 17 + pkg/templates/scripts/tn-k3s-node.sh | 144 + pkg/templates/server.sh.tmpl | 23 +- pkg/templates/testdata/agent.roles.golden.sh | 1055 ++++++ pkg/templates/testdata/server.roles.golden.sh | 3367 +++++++++++++++++ 15 files changed, 6110 insertions(+), 11 deletions(-) create mode 100644 cmd/bootstrap_node.go create mode 100644 pkg/templates/node.go create mode 100644 pkg/templates/node_test.go create mode 100644 pkg/templates/scripts/tn-k3s-node.sh create mode 100644 pkg/templates/testdata/agent.roles.golden.sh create mode 100644 pkg/templates/testdata/server.roles.golden.sh diff --git a/.github/workflows/k3s-ca.yml b/.github/workflows/k3s-ca.yml index d76d628..f8634d9 100644 --- a/.github/workflows/k3s-ca.yml +++ b/.github/workflows/k3s-ca.yml @@ -33,7 +33,7 @@ jobs: cache: true - name: shellcheck run: | - shellcheck -s sh -S warning pkg/templates/scripts/tn-k3s-ca.sh + shellcheck -s sh -S warning pkg/templates/scripts/tn-k3s-ca.sh pkg/templates/scripts/tn-k3s-node.sh - name: go test run: | go test ./test/k3s-ca/... ./cmd/... ./pkg/auth/... diff --git a/README.md b/README.md index 27ac46a..359f1d7 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ Securely connect to your edge devices via Teknoir's proxy infrastructure. ```bash tnctl bootstrap my-device # writes bootstrap_{agent,server,docker_server}_my-device.sh tnctl bootstrap my-device --k3s-version v1.34.11+k3s1 # install another k3s release +tnctl bootstrap my-robot --robot my-robot --role vision --agent-role io --server orin.local # node roles: see docs/commands/bootstrap.md ``` The scripts install a **pinned k3s release** (`v1.36.5+k3s1` by default, the `stable` channel when this diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index ab7f9a2..f0285e9 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -24,6 +24,14 @@ var bootstrapCmd = &cobra.Command{ if err := templates.ValidateK3SVersion(k3sVersion); err != nil { return err } + nodeOpts, err := nodeOptionsFromFlags(cmd) + if err != nil { + return err + } + nodes, err := nodeOpts.resolve() + if err != nil { + return err + } cfg, err := config.Load() if err != nil { return err @@ -128,7 +136,9 @@ var bootstrapCmd = &cobra.Command{ K3SForce: force, } - generate := func(tmpl *template.Template, filename string) error { + serverData, agentData := nodes.apply(data) + + generate := func(tmpl *template.Template, filename string, data templates.TemplateData) error { f, err := os.Create(filename) if err != nil { return err @@ -141,13 +151,13 @@ var bootstrapCmd = &cobra.Command{ serverFile := fmt.Sprintf("bootstrap_server_%s.sh", deviceName) dockerServerFile := fmt.Sprintf("bootstrap_docker_server_%s.sh", deviceName) - if err := generate(templates.AgentTemplate, agentFile); err != nil { + if err := generate(templates.AgentTemplate, agentFile, agentData); err != nil { return err } - if err := generate(templates.ServerTemplate, serverFile); err != nil { + if err := generate(templates.ServerTemplate, serverFile, serverData); err != nil { return err } - if err := generate(templates.DockerServerTemplate, dockerServerFile); err != nil { + if err := generate(templates.DockerServerTemplate, dockerServerFile, data); err != nil { return err } @@ -155,6 +165,7 @@ var bootstrapCmd = &cobra.Command{ fmt.Printf("- %s\n", agentFile) fmt.Printf("- %s\n", serverFile) fmt.Printf("- %s\n", dockerServerFile) + fmt.Print(nodes.describe(agentFile, serverFile)) skipUpload, _ := cmd.Flags().GetBool("skip-upload") if !skipUpload { @@ -181,6 +192,7 @@ func init() { "let the scripts replace an installed k3s with --k3s-version (TN_K3S_UPGRADE=true is the default in the scripts). Without it a re-run keeps the installed k3s; a downgrade is always refused") bootstrapCmd.Flags().Bool("force", false, "with --k3s-upgrade: allow an upgrade over more than one minor version (TN_K3S_FORCE=true is the default in the scripts)") + addNodeFlags(bootstrapCmd) bootstrapCmd.Flags().String("docker-secret", "", "The contents of the .dockerconfigjson") rootCmd.AddCommand(bootstrapCmd) } diff --git a/cmd/bootstrap_node.go b/cmd/bootstrap_node.go new file mode 100644 index 0000000..154d045 --- /dev/null +++ b/cmd/bootstrap_node.go @@ -0,0 +1,162 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" + "github.com/teknoir/cli/pkg/templates" +) + +// The node flags of `tnctl bootstrap`: the role of a computer in a robot, chosen when it is bootstrapped. +// +// A robot is one device in the platform (one set of device keys) that may have several computers: one k3s +// server and any number of k3s agents. A single run produces the server script and the agent script of +// the robot, with one token, so the flags come in two sets: --role, --node-name, --label and --taint +// are for the server script; the --agent-* flags are for the agent script. --robot, --server and +// --tls-san are about the robot as a whole. + +// nodeOptions is what the node flags say, before validation. +type nodeOptions struct { + Robot string + Server string + SANs []string + + Role, NodeName string + Labels, Taints []string + AgentRole string + AgentNodeName string + AgentLabels []string + AgentTaints []string + serverFlagsPresent bool + agentFlagsPresent bool +} + +// bootstrapNodes is the validated result: what each script is told. +type bootstrapNodes struct { + Server templates.NodeConfig + Agent templates.NodeConfig + ServerAddress string + TLSSANs []string +} + +func addNodeFlags(c *cobra.Command) { + f := c.Flags() + f.String("robot", "", "name of the robot this computer belongs to; becomes the label "+templates.LabelRobot+" and, with --role, the node name -") + f.String("role", "", "node role of the SERVER script (a DNS label, like vision or io); becomes the label "+templates.LabelNodeRole+". k3s applies labels at the first registration of a node, so it is chosen here") + f.String("node-name", "", "node name of the server script (default: - with --robot and --role, else teknoir-master)") + f.StringArray("label", nil, "extra node label key=value for the server script (repeatable). The kubernetes.io/ and k8s.io/ prefixes are refused") + f.StringArray("taint", nil, "node taint key[=value]:Effect for the server script (repeatable; Effect is NoSchedule, PreferNoSchedule or NoExecute)") + f.String("agent-role", "", "node role of the AGENT script, for a second computer of the robot (see --role)") + f.String("agent-node-name", "", "node name of the agent script (default: - with --robot and --agent-role, else the host name)") + f.StringArray("agent-label", nil, "extra node label key=value for the agent script (repeatable)") + f.StringArray("agent-taint", nil, "node taint key[=value]:Effect for the agent script (repeatable)") + f.String("server", "", "host name or IP address of the k3s server that the agent script joins; baked in as K3S_URL=https://:6443 (K3S_URL in the environment still wins) and added to the server's TLS SANs") + f.StringArray("tls-san", nil, "extra host name or IP address in the server certificate (--tls-san of k3s; repeatable)") +} + +func nodeOptionsFromFlags(c *cobra.Command) (nodeOptions, error) { + f := c.Flags() + var o nodeOptions + var err error + str := func(name string) string { + if err != nil { + return "" + } + var v string + v, err = f.GetString(name) + return v + } + arr := func(name string) []string { + if err != nil { + return nil + } + var v []string + v, err = f.GetStringArray(name) + return v + } + o.Robot, o.Server = str("robot"), str("server") + o.Role, o.NodeName, o.Labels, o.Taints = str("role"), str("node-name"), arr("label"), arr("taint") + o.AgentRole, o.AgentNodeName, o.AgentLabels, o.AgentTaints = str("agent-role"), str("agent-node-name"), arr("agent-label"), arr("agent-taint") + o.SANs = arr("tls-san") + o.serverFlagsPresent = f.Changed("role") || f.Changed("node-name") || f.Changed("label") || f.Changed("taint") + o.agentFlagsPresent = f.Changed("agent-role") || f.Changed("agent-node-name") || f.Changed("agent-label") || f.Changed("agent-taint") + return o, err +} + +// resolve validates the options and builds the node configuration of the server and the agent script. +func (o nodeOptions) resolve() (bootstrapNodes, error) { + var n bootstrapNodes + if o.Robot != "" && !o.serverFlagsPresent && !o.agentFlagsPresent { + return n, fmt.Errorf("--robot has no effect without --role (or --agent-role): it names the robot the node belongs to") + } + if o.serverFlagsPresent { + n.Server = templates.NodeConfig{Role: o.Role, Robot: o.Robot, Name: o.NodeName, Labels: o.Labels, Taints: o.Taints} + if err := n.Server.Validate(); err != nil { + return n, fmt.Errorf("server node: %w", err) + } + } + if o.agentFlagsPresent { + n.Agent = templates.NodeConfig{Role: o.AgentRole, Robot: o.Robot, Name: o.AgentNodeName, Labels: o.AgentLabels, Taints: o.AgentTaints} + if err := n.Agent.Validate(); err != nil { + return n, fmt.Errorf("agent node: %w", err) + } + } + if s, a := n.Server.EffectiveName(), n.Agent.EffectiveName(); s != "" && s == a { + return n, fmt.Errorf("the server and the agent node would both be named %q: give one of them another --node-name / --agent-node-name, or another role", s) + } + if o.Server != "" { + if err := templates.ValidateHost("--server", o.Server); err != nil { + return n, err + } + n.ServerAddress = strings.Trim(o.Server, "[]") + } + for _, san := range o.SANs { + if err := templates.ValidateHost("--tls-san", san); err != nil { + return n, err + } + } + n.TLSSANs = o.SANs + return n, nil +} + +// describe tells what the scripts were generated with, so that a typo in a role is seen before the script is +// copied to a device. It is empty when no node flag was given. +func (n bootstrapNodes) describe(agentFile, serverFile string) string { + var b strings.Builder + node := func(file, kind string, c templates.NodeConfig) { + if !c.Configured() { + return + } + name := c.EffectiveName() + if name == "" { + name = "(default)" + } + fmt.Fprintf(&b, "%s: %s node %s, labels %s", file, kind, name, strings.Join(c.AllLabels(), ",")) + if len(c.Taints) > 0 { + fmt.Fprintf(&b, ", taints %s", strings.Join(c.Taints, ",")) + } + b.WriteString("\n") + } + node(serverFile, "server", n.Server) + node(agentFile, "agent", n.Agent) + if n.ServerAddress != "" { + fmt.Fprintf(&b, "%s joins %s\n", agentFile, templates.K3SServerURL(n.ServerAddress)) + } + if n.Server.HasLabelsOrTaints() || n.Agent.HasLabelsOrTaints() { + b.WriteString("Note: k3s applies node labels and taints when a node registers for the first time; on a computer that is already part of a cluster the script prints the kubectl commands instead.\n") + } + return b.String() +} + +// apply gives the server and the agent script their own data from the common data of one run: the scripts +// share the device, its keys and the token, but are for different computers of the robot. The docker +// server script keeps the common data (it takes no node roles). +func (n bootstrapNodes) apply(base templates.TemplateData) (server, agent templates.TemplateData) { + server, agent = base, base + server.Node = n.Server + server.TLSSANs = templates.MergeTLSSANs(n.ServerAddress, n.TLSSANs, base.DeviceID+"."+base.Domain, "10.0.0.10") + agent.Node = n.Agent + agent.Server = n.ServerAddress + return server, agent +} diff --git a/cmd/bootstrap_test.go b/cmd/bootstrap_test.go index 9fa39d7..e00cd72 100644 --- a/cmd/bootstrap_test.go +++ b/cmd/bootstrap_test.go @@ -1,8 +1,10 @@ package cmd import ( + "strings" "testing" + "github.com/spf13/cobra" "github.com/teknoir/cli/pkg/templates" ) @@ -28,3 +30,163 @@ func TestBootstrapK3SUpgradeFlags(t *testing.T) { } } } + +func newNodeFlagCmd(t *testing.T, args ...string) nodeOptions { + t.Helper() + c := &cobra.Command{Use: "x", Run: func(*cobra.Command, []string) {}} + addNodeFlags(c) + if err := c.ParseFlags(args); err != nil { + t.Fatal(err) + } + o, err := nodeOptionsFromFlags(c) + if err != nil { + t.Fatal(err) + } + return o +} + +func TestBootstrapNodeFlagsExist(t *testing.T) { + for _, name := range []string{"role", "robot", "node-name", "label", "taint", "server", "tls-san", "agent-role", "agent-node-name", "agent-label", "agent-taint"} { + if bootstrapCmd.Flags().Lookup(name) == nil { + t.Errorf("no --%s flag", name) + } + } +} + +// With none of the node flags nothing is configured: the scripts stay as they were. +func TestBootstrapNodes_NoFlagsNoConfig(t *testing.T) { + n, err := newNodeFlagCmd(t).resolve() + if err != nil { + t.Fatal(err) + } + if n.Server.Configured() || n.Agent.Configured() || n.ServerAddress != "" || len(n.TLSSANs) != 0 { + t.Errorf("%+v", n) + } +} + +func TestBootstrapNodes_Robot(t *testing.T) { + n, err := newNodeFlagCmd(t, + "--robot", "mobile-01", "--role", "vision", "--label", "teknoir.org/gpu=nvidia", "--taint", "nvidia.com/gpu=true:NoSchedule", + "--agent-role", "io", "--server", "orin.local", "--tls-san", "10.0.0.5", "--tls-san", "orin").resolve() + if err != nil { + t.Fatal(err) + } + if n.Server.EffectiveName() != "mobile-01-vision" || n.Agent.EffectiveName() != "mobile-01-io" { + t.Errorf("names %q %q", n.Server.EffectiveName(), n.Agent.EffectiveName()) + } + if got := strings.Join(n.Server.AllLabels(), " "); got != "teknoir.org/node-role=vision teknoir.org/robot=mobile-01 teknoir.org/gpu=nvidia" { + t.Errorf("server labels %s", got) + } + if got := strings.Join(n.Agent.AllLabels(), " "); got != "teknoir.org/node-role=io teknoir.org/robot=mobile-01" { + t.Errorf("agent labels %s", got) + } + if len(n.Agent.Taints) != 0 { + t.Errorf("the agent got the taints of the server: %v", n.Agent.Taints) + } + if n.ServerAddress != "orin.local" || len(n.TLSSANs) != 2 { + t.Errorf("%+v", n) + } +} + +func TestBootstrapNodes_RoleWithoutRobotKeepsDefaultName(t *testing.T) { + n, err := newNodeFlagCmd(t, "--role", "vision").resolve() + if err != nil { + t.Fatal(err) + } + if n.Server.EffectiveName() != "" || n.Agent.Configured() { + t.Errorf("%+v", n) + } +} + +func TestBootstrapNodes_OnlyTheAgentIsConfigured(t *testing.T) { + n, err := newNodeFlagCmd(t, "--robot", "r1", "--agent-role", "io").resolve() + if err != nil { + t.Fatal(err) + } + if n.Server.Configured() || n.Agent.EffectiveName() != "r1-io" { + t.Errorf("%+v", n) + } +} + +func TestBootstrapNodes_Rejects(t *testing.T) { + for name, args := range map[string][]string{ + "robot alone": {"--robot", "r1"}, + "robot without role": {"--robot", "r1", "--label", "a=b"}, + "bad role": {"--role", "Vision"}, + "bad robot": {"--role", "io", "--robot", "r_1"}, + "bad agent role": {"--agent-role", "x y"}, + "reserved label": {"--role", "io", "--label", "kubernetes.io/hostname=x"}, + "reserved agent label": {"--agent-role", "io", "--agent-label", "foo.k8s.io/x=y"}, + "role label": {"--role", "io", "--label", "teknoir.org/node-role=x"}, + "bad taint": {"--role", "io", "--taint", "a=b"}, + "bad agent taint": {"--agent-role", "io", "--agent-taint", "a=b:Nope"}, + "bad server": {"--server", "https://orin"}, + "server with port": {"--server", "orin:6443"}, + "bad san": {"--tls-san", "a b"}, + "same node name": {"--robot", "r1", "--role", "io", "--agent-role", "io"}, + "same explicit node name": {"--node-name", "n1", "--agent-node-name", "n1"}, + "bad node name": {"--node-name", "N_1"}, + } { + if _, err := newNodeFlagCmd(t, args...).resolve(); err == nil { + t.Errorf("%s: %v accepted", name, args) + } + } +} + +func TestBootstrapNodes_ReservedPrefixError(t *testing.T) { + _, err := newNodeFlagCmd(t, "--role", "io", "--label", "kubernetes.io/hostname=x").resolve() + if err == nil || !strings.Contains(err.Error(), "kubernetes.io/ and k8s.io/") || !strings.Contains(err.Error(), "server node") { + t.Errorf("%v", err) + } +} + +func TestBootstrapNodes_Describe(t *testing.T) { + n, err := newNodeFlagCmd(t, "--robot", "m", "--role", "vision", "--agent-role", "io", "--agent-taint", "a=b:NoExecute", "--server", "orin").resolve() + if err != nil { + t.Fatal(err) + } + out := n.describe("agent.sh", "server.sh") + for _, want := range []string{"server.sh: server node m-vision, labels teknoir.org/node-role=vision,teknoir.org/robot=m", "agent.sh: agent node m-io", "taints a=b:NoExecute", "agent.sh joins https://orin:6443"} { + if !strings.Contains(out, want) { + t.Errorf("missing %q in\n%s", want, out) + } + } + if none := (bootstrapNodes{}).describe("a", "s"); none != "" { + t.Errorf("without node flags nothing is printed: %q", none) + } +} + +func TestBootstrapNodes_ApplyKeepsOneToken(t *testing.T) { + n, err := newNodeFlagCmd(t, "--robot", "m", "--role", "vision", "--agent-role", "io", "--server", "orin.local", "--tls-san", "orin.local", "--tls-san", "dev-1.test.cloud", "--tls-san", "10.0.0.5").resolve() + if err != nil { + t.Fatal(err) + } + base := templates.TemplateData{DeviceID: "dev-1", Domain: "test.cloud", K3SToken: "tok"} + s, a := n.apply(base) + if s.K3SToken != "tok" || a.K3SToken != "tok" { + t.Error("the scripts of one run must share the token") + } + // the server address is first, nothing twice, nothing that the script adds itself + if strings.Join(s.TLSSANs, ",") != "orin.local,10.0.0.5" { + t.Errorf("tls-san %v", s.TLSSANs) + } + if s.Server != "" || a.Server != "orin.local" || len(a.TLSSANs) != 0 { + t.Errorf("server %q / %q, agent sans %v", s.Server, a.Server, a.TLSSANs) + } + if s.Node.EffectiveName() != "m-vision" || a.Node.EffectiveName() != "m-io" { + t.Errorf("%q %q", s.Node.EffectiveName(), a.Node.EffectiveName()) + } + var out strings.Builder + if err := templates.ServerTemplate.Execute(&out, s); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "--tls-san 10.0.0.10 --tls-san orin.local --tls-san 10.0.0.5") { + t.Error("the server script lacks the SANs") + } + // no node flags: the data is the common data + z, _ := newNodeFlagCmd(t).resolve() + s, a = z.apply(base) + if s.Node.Configured() || a.Node.Configured() || s.Server != "" || a.Server != "" || len(s.TLSSANs) != 0 { + t.Errorf("%+v %+v", s, a) + } +} diff --git a/docs/commands/bootstrap.md b/docs/commands/bootstrap.md index daa69ba..095bbb1 100644 --- a/docs/commands/bootstrap.md +++ b/docs/commands/bootstrap.md @@ -19,6 +19,14 @@ The device may be given as an argument, via the `--device` global flag, or selec | `--k3s-upgrade` | `false` | Let the scripts replace an installed k3s with `--k3s-version` (sets `TN_K3S_UPGRADE=true` as the script's default). Without it a re-run **keeps** the installed k3s. See [Re-running a script](#re-running-a-script-the-installed-k3s-is-kept). | | `--force` | `false` | Only with `--k3s-upgrade`: allow an upgrade over more than one minor version (`TN_K3S_FORCE=true`). A downgrade is never allowed. | | `--docker-secret` | `""` | Contents of the `.dockerconfigjson` to embed. | +| `--robot` | `""` | Name of the robot (the multi-computer cluster) the computers belong to. Becomes the node label `teknoir.org/robot` and, together with a role, the node name `-`. See [Robots with several computers](#robots-with-several-computers). | +| `--role` | `""` | Node role of the **server** script (a DNS label such as `vision` or `io`). Becomes the node label `teknoir.org/node-role`. | +| `--node-name` | `""` | Node name of the server script. Default: `-` with `--robot` and `--role`, otherwise `teknoir-master` as before. | +| `--label key=value` | none | Extra node label for the server script. Repeatable. | +| `--taint key[=value]:Effect` | none | Node taint for the server script (`NoSchedule`, `PreferNoSchedule` or `NoExecute`). Repeatable. | +| `--agent-role`, `--agent-node-name`, `--agent-label`, `--agent-taint` | none | The same four settings for the **agent** script. The agent node name defaults to `-` with `--robot`, otherwise to the host name of the computer. | +| `--server ` | `""` | Host name or IP address of the k3s server that the agent joins. The agent script gets `K3S_URL=https://:6443` (still overridable with `K3S_URL` in the environment), and the host is added to the server certificate's TLS SANs. | +| `--tls-san ` | none | Extra host name or IP address in the server certificate (k3s `--tls-san`). Repeatable. | Plus all [global flags](index.md#global-flags). @@ -36,6 +44,101 @@ The scripts are rendered from templates and populated with the device's decoded (RSA keypair, username/password, public SSH key), the domain/namespace, the device ID, a freshly generated K3S token, and the optional Docker secret. +## Robots with several computers + +A robot is one device in the platform (one set of device keys) that can have several computers: one k3s +**server** and any number of k3s **agents**. The role of a computer, what it is used for, is a set of k3s +node labels, so the platform can place workloads on it: + +| Role | Labels the script sets | Typical use | +| --- | --- | --- | +| `vision` | `teknoir.org/node-role=vision`, `teknoir.org/robot=`, plus what you add with `--label` (for example `teknoir.org/gpu=nvidia`) | GPU workloads: vision models, NVR | +| `io` | `teknoir.org/node-role=io`, `teknoir.org/robot=`, plus `--label` | cameras, sensors, control | + +**The role has to be chosen when the computer is bootstrapped.** k3s applies `node-label` and `node-taint` +only when a node registers with the cluster for the first time, so the bootstrap script writes them before +the k3s installer starts k3s. The role names are yours to choose (a DNS label); `vision` and `io` are the +ones the platform uses. + +**One `tnctl bootstrap` run per robot** writes a server script and an agent script that share the k3s +token, the device keys and the k3s release policy. The unprefixed flags (`--role`, `--node-name`, +`--label`, `--taint`) are for the server script; the `--agent-*` flags are for the agent script. +`--robot`, `--server` and `--tls-san` are about the robot as a whole. + +Example, `mobile-01`: the Jetson Orin is the k3s server with the role `vision`, a Raspberry Pi joins it later +as an agent with the role `io`: + +```bash +tnctl bootstrap mobile-01 \ + --robot mobile-01 \ + --role vision --label teknoir.org/gpu=nvidia \ + --agent-role io \ + --server orin.local +``` + +``` +Successfully generated bootstrap scripts: +- bootstrap_agent_mobile-01.sh +- bootstrap_server_mobile-01.sh +- bootstrap_docker_server_mobile-01.sh +bootstrap_server_mobile-01.sh: server node mobile-01-vision, labels teknoir.org/node-role=vision,teknoir.org/robot=mobile-01,teknoir.org/gpu=nvidia +bootstrap_agent_mobile-01.sh: agent node mobile-01-io, labels teknoir.org/node-role=io,teknoir.org/robot=mobile-01 +bootstrap_agent_mobile-01.sh joins https://orin.local:6443 +``` + +1. Copy `bootstrap_server_mobile-01.sh` to the Orin and run it as root. The node is called + `mobile-01-vision` and carries the two labels and `teknoir.org/gpu=nvidia`. `orin.local` is added to + the server certificate (`--server` is added to the SANs automatically, `--tls-san` adds more). +2. Later, copy `bootstrap_agent_mobile-01.sh` to the Pi and run it as root. It joins + `https://orin.local:6443` with the shared token (`K3S_URL=... K3S_TOKEN=... sh bootstrap_agent_...sh` + still overrides both), reads the k3s release from the server, and registers as `mobile-01-io` with the + `io` role. + +A robot with a single computer uses the server script only: `tnctl bootstrap vision-station-01 --robot +vision-station-01 --role vision --label teknoir.org/gpu=nvidia`. A sensor unit such as `sensor-unit-01` is the same +with `--role io`. + +Rules for the values: + +- `--role`, `--robot`: DNS labels (lowercase letters, digits and `-`, at most 63 characters). + `--robot` needs a role (the node name is `-`) or an explicit `--node-name`. +- `--node-name`: a DNS name. The server and the agent must not end up with the same node name, so two + computers with the same role need an explicit `--agent-node-name`. +- `--label`: Kubernetes label syntax (`key=value`, an optional DNS prefix and `/` in the key). The + prefixes `kubernetes.io/` and `k8s.io/` (and their subdomains) are refused, and so are + `teknoir.org/node-role` and `teknoir.org/robot`: they come from `--role` and `--robot`. +- `--taint`: `key[=value]:Effect`, same key rules. A tainted node only runs workloads that tolerate the + taint, so use it for a node that must not run anything else. +- `--server`, `--tls-san`: a host name or an IP address, without scheme or port (the port is 6443). + +What the scripts write, and what they never overwrite: + +| File | Content | +| --- | --- | +| `/etc/rancher/k3s/config.yaml` | `node-name`, added only when the file has no `node-name` already. A name that the device owner set is kept, and a different one is reported. The rest of the file is never rewritten. | +| `/etc/rancher/k3s/config.yaml.d/98-teknoir-node.yaml` | `node-label+` and `node-taint+`. The `+` **appends** to the `node-label` and `node-taint` lists of `config.yaml` and the other drop-ins, so labels and taints of the device owner are kept. The file belongs to the script: it is replaced (atomically) when it differs and left alone when it does not, so a re-run changes nothing. | + +The agent script uses the same two files; `--server` only changes the `K3S_URL` it sets. + +!!! warning "Labels are applied when a node registers" + On a computer that is already part of a cluster, k3s ignores a changed `node-label` or `node-taint`. + The script notices that the node has registered (`/agent/client-kubelet.crt` exists) and + that the labels differ, and prints the commands that apply them to the running node. Run them on the + k3s server: + + ```bash + k3s kubectl label node mobile-01-io teknoir.org/node-role=io teknoir.org/robot=mobile-01 --overwrite + k3s kubectl taint node mobile-01-io dedicated=cameras:NoSchedule --overwrite + ``` + + A label that you removed from the script stays on the node until you remove it with + `k3s kubectl label node -`. A node that is renamed registers as a new node; the old one + has to be deleted (`k3s kubectl delete node `). + +Without any of these flags the scripts are exactly what they were before (`teknoir-master` for the +server, the host name for the agent, no labels). The docker server script (`k3s < 1.24`) does not take +node roles. + ## The k3s version The server and agent scripts install a **pinned** k3s release: `INSTALL_K3S_VERSION` is set from @@ -196,6 +299,12 @@ tnctl bootstrap --docker-secret "$(cat dockerconfig.json)" # Generate but do not upload tnctl bootstrap edge-01 --skip-upload + +# A single-computer robot with a GPU: the server script gets the role vision +tnctl bootstrap vision-station-01 --robot vision-station-01 --role vision --label teknoir.org/gpu=nvidia + +# A robot with two computers: server (vision) and agent (io), see "Robots with several computers" +tnctl bootstrap mobile-01 --robot mobile-01 --role vision --agent-role io --server orin.local ``` !!! note diff --git a/pkg/templates/agent.sh.tmpl b/pkg/templates/agent.sh.tmpl index 65db83a..2001e6c 100644 --- a/pkg/templates/agent.sh.tmpl +++ b/pkg/templates/agent.sh.tmpl @@ -23,6 +23,21 @@ # The k3s release policy (a re-run keeps the installed k3s) and the cgroup v1 kubelet flag. {{template "tn-k3s-install.sh" .}} +{{- if .Node.Configured}} + +# The node name, role labels and taints of this node (tnctl bootstrap --agent-role ...). k3s applies the +# labels and taints only when the node registers for the first time, so they are written before the installer runs. +{{template "tn-k3s-node.sh" .}} +{{- if .Node.EffectiveName}} +tn_node_name '{{.Node.EffectiveName}}' +{{- end}} +{{- range .Node.AllLabels}} +tn_node_label '{{.}}' +{{- end}} +{{- range .Node.Taints}} +tn_node_taint '{{.}}' +{{- end}} +{{- end}} {{template "keys.sh.tmpl" .}} @@ -45,7 +60,9 @@ if [ "${OS_BUILD}" ]; then $SUDO sed -i "s#-d /run/systemd#true#g" k3s_installer.sh fi -if [ -z "${K3S_URL}" ]; then +{{if .Server}}# tnctl bootstrap --server: the k3s server this agent joins. K3S_URL in the environment overrides it. +K3S_URL="${K3S_URL:-{{.ServerURL}}}" +{{end}}if [ -z "${K3S_URL}" ]; then DEFAULT_K3S_URL="https://${_DEVICE_ID}.local:6443" read -p "Is ${DEFAULT_K3S_URL} the correct URL to the server/master node? [Y/n] " confirm confirm=${confirm:-"Y"} @@ -71,7 +88,10 @@ fi # (the flag goes in from k3s v1.31, where the kubelet learned it). INSTALL_K3S_EXEC is final above, before # tn_inst_prepare reads it. tn_inst_prepare agent -export INSTALL_K3S_SYMLINK=force +{{if .Node.Configured}}# The node name, role labels and taints go into the k3s configuration before the installer starts k3s +# (a refusal of tn_inst_prepare above has changed nothing yet). +tn_node_apply agent +{{end}}export INSTALL_K3S_SYMLINK=force $SUDO ./k3s_installer.sh # The kubelet flag that had to wait for the installer (an installed k3s older than v1.31 that was upgraded) tn_inst_finish diff --git a/pkg/templates/bootstrap_templates.go b/pkg/templates/bootstrap_templates.go index 9a8a8a1..d94f358 100644 --- a/pkg/templates/bootstrap_templates.go +++ b/pkg/templates/bootstrap_templates.go @@ -52,6 +52,14 @@ type TemplateData struct { K3SUpgrade bool // K3SForce allows that upgrade to skip minor versions (TN_K3S_FORCE=true is the default in the script). K3SForce bool + // Node is the role, robot, name, labels and taints of the node this script bootstraps (`tnctl + // bootstrap --role` and friends). The zero value changes nothing in the script. + Node NodeConfig + // Server is the address the agent script joins (`tnctl bootstrap --server`): K3S_URL=https://:6443, + // still overridable by K3S_URL in the environment. Empty: the script asks, as it always did. + Server string + // TLSSANs are extra --tls-san values of the server script, after the two it always adds. + TLSSANs []string } func (d TemplateData) k3sVersion() string { @@ -103,6 +111,13 @@ var TnK3sCAScript string //go:embed scripts/tn-k3s-install.sh var TnK3sInstallScript string +// TnK3sNodeScript writes the node name, labels and taints of a node (tnctl bootstrap --role) into the k3s +// configuration, before the installer starts k3s. It is embedded in the server and the agent script as the +// template "tn-k3s-node.sh", only when the script was generated with a node role, label, taint or name. +// +//go:embed scripts/tn-k3s-node.sh +var TnK3sNodeScript string + //go:embed scripts/*.sh var scriptsFS embed.FS @@ -115,7 +130,7 @@ var ( func init() { AgentTemplate = template.Must(template.New("agent").Parse(AgentTemplateStr)) template.Must(AgentTemplate.ParseFS(partialsFS, "partials/*.tmpl")) - template.Must(AgentTemplate.ParseFS(scriptsFS, "scripts/tn-k3s-install.sh")) + template.Must(AgentTemplate.ParseFS(scriptsFS, "scripts/tn-k3s-install.sh", "scripts/tn-k3s-node.sh")) ServerTemplate = template.Must(template.New("server").Parse(ServerTemplateStr)) template.Must(ServerTemplate.ParseFS(partialsFS, "partials/*.tmpl")) diff --git a/pkg/templates/node.go b/pkg/templates/node.go new file mode 100644 index 0000000..08b7756 --- /dev/null +++ b/pkg/templates/node.go @@ -0,0 +1,292 @@ +package templates + +import ( + "fmt" + "net" + "regexp" + "strings" +) + +// Node roles. A role is a set of k3s node labels: always teknoir.org/node-role= and, when the +// node belongs to a robot, teknoir.org/robot=, plus any extra labels and taints the robot type +// asks for. k3s applies node-label and node-taint only when a node registers for the first time, so the +// bootstrap scripts write them before the k3s installer runs (see scripts/tn-k3s-node.sh). +const ( + // LabelNodeRole is the label every node with a role carries. + LabelNodeRole = "teknoir.org/node-role" + // LabelRobot is the label of the robot (the multi-node cluster) a node belongs to. + LabelRobot = "teknoir.org/robot" +) + +var ( + dnsLabelRE = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + // hostLabelRE is a label of a host name: the case of a name is not significant, so a user's + // "Orin.local" is let through (and written as typed). + hostLabelRE = regexp.MustCompile(`^[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?$`) + // labelNameRE is the name part of a Kubernetes label key and a label value. + labelNameRE = regexp.MustCompile(`^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$`) + taintRE = regexp.MustCompile(`^([^=:]+)(?:=([^:]*))?:([A-Za-z]+)$`) +) + +// TaintEffects are the effects Kubernetes knows for a taint. +var TaintEffects = []string{"NoSchedule", "PreferNoSchedule", "NoExecute"} + +// NodeConfig is what one node of a robot is told at bootstrap: its role, its robot, its name, and extra +// labels and taints. The zero value means "nothing": the script then behaves as it always did. +type NodeConfig struct { + // Role is the node role (a DNS label), for example vision or io. + Role string + // Robot is the robot the node belongs to (a DNS label). + Robot string + // Name is the node name asked for. Empty: - when both are set, else the default of the + // script (a server: teknoir-master, an agent: the host name). + Name string + // Labels are extra labels, "key=value" (the value may be empty), without the two Teknoir labels. + Labels []string + // Taints are node taints, "key[=value]:Effect". + Taints []string +} + +// Configured reports whether the node asks for anything at all. +func (n NodeConfig) Configured() bool { + return n.Role != "" || n.Robot != "" || n.Name != "" || len(n.Labels) > 0 || len(n.Taints) > 0 +} + +// EffectiveName is the node name the script sets, or "" when it keeps its own default. +func (n NodeConfig) EffectiveName() string { + if n.Name != "" { + return n.Name + } + if n.Robot != "" && n.Role != "" { + return n.Robot + "-" + n.Role + } + return "" +} + +// AllLabels is every label of the node in the order they are written: the role, the robot, then the +// extra labels as given. +func (n NodeConfig) AllLabels() []string { + var l []string + if n.Role != "" { + l = append(l, LabelNodeRole+"="+n.Role) + } + if n.Robot != "" { + l = append(l, LabelRobot+"="+n.Robot) + } + return append(l, n.Labels...) +} + +// HasLabelsOrTaints reports whether there is anything for the labels and taints file. +func (n NodeConfig) HasLabelsOrTaints() bool { + return len(n.AllLabels()) > 0 || len(n.Taints) > 0 +} + +// Validate checks every part of the node configuration. The values end up in shell and YAML, so +// nothing is let through that is not Kubernetes syntax. +func (n NodeConfig) Validate() error { + if n.Role != "" { + if err := validateDNSLabel("role", n.Role); err != nil { + return err + } + } + if n.Robot != "" { + if err := validateDNSLabel("robot", n.Robot); err != nil { + return err + } + } + if n.Name != "" { + if err := validateDNSSubdomain("node name", n.Name); err != nil { + return err + } + } + if n.Robot != "" && n.Role == "" && n.Name == "" { + return fmt.Errorf("--robot %q needs a --role (the node is named -), or an explicit --node-name", n.Robot) + } + seen := map[string]bool{} + if n.Role != "" { + seen[LabelNodeRole] = true + } + if n.Robot != "" { + seen[LabelRobot] = true + } + for _, l := range n.Labels { + key, err := ValidateLabel(l) + if err != nil { + return err + } + switch key { + case LabelNodeRole: + return fmt.Errorf("label %q: %s is set by --role, do not pass it as a label", l, LabelNodeRole) + case LabelRobot: + return fmt.Errorf("label %q: %s is set by --robot, do not pass it as a label", l, LabelRobot) + } + if seen[key] { + return fmt.Errorf("label key %q is given more than once", key) + } + seen[key] = true + } + seenTaint := map[string]bool{} + for _, t := range n.Taints { + if err := ValidateTaint(t); err != nil { + return err + } + if seenTaint[t] { + return fmt.Errorf("taint %q is given more than once", t) + } + seenTaint[t] = true + } + return nil +} + +func validateDNSLabel(what, v string) error { + if len(v) > 63 || !dnsLabelRE.MatchString(v) { + return fmt.Errorf("invalid %s %q: want a DNS label (lowercase letters, digits and '-', starting and ending with a letter or digit, at most 63 characters)", what, v) + } + return nil +} + +func validateDNSSubdomain(what, v string) error { + if len(v) > 253 { + return fmt.Errorf("invalid %s %q: longer than 253 characters", what, v) + } + for _, p := range strings.Split(v, ".") { + if len(p) > 63 || !dnsLabelRE.MatchString(p) { + return fmt.Errorf("invalid %s %q: want a DNS name (lowercase letters, digits, '-' and '.', every part starting and ending with a letter or digit, at most 63 characters)", what, v) + } + } + return nil +} + +// restrictedDomain reports whether a label or taint key prefix belongs to Kubernetes itself +// (kubernetes.io, k8s.io and any subdomain of them). The kubelet refuses to register with such labels, +// except a few it sets itself, and k3s would not start the node. +func restrictedDomain(prefix string) bool { + for _, d := range []string{"kubernetes.io", "k8s.io"} { + if prefix == d || strings.HasSuffix(prefix, "."+d) { + return true + } + } + return false +} + +// validateKey checks a label or taint key (an optional DNS subdomain prefix and '/', then a name). +func validateKey(what, key string) error { + prefix, name := "", key + if i := strings.LastIndex(key, "/"); i >= 0 { + prefix, name = key[:i], key[i+1:] + if err := validateDNSSubdomain(what+" key prefix", prefix); err != nil { + return err + } + if restrictedDomain(prefix) { + return fmt.Errorf("%s key %q: the kubernetes.io/ and k8s.io/ prefixes are reserved for Kubernetes; use your own prefix, like teknoir.org/", what, key) + } + } + if len(name) > 63 || !labelNameRE.MatchString(name) { + return fmt.Errorf("invalid %s key %q: the name part (after any '/') must be 1-63 characters of letters, digits, '-', '_' and '.', starting and ending with a letter or digit", what, key) + } + return nil +} + +// ValidateLabel checks "key=value" (the value may be empty) and returns the key. +func ValidateLabel(l string) (string, error) { + key, val, ok := strings.Cut(l, "=") + if !ok { + return "", fmt.Errorf("invalid label %q: want key=value", l) + } + if err := validateKey("label", key); err != nil { + return "", err + } + if val != "" && (len(val) > 63 || !labelNameRE.MatchString(val)) { + return "", fmt.Errorf("invalid label %q: the value must be at most 63 characters of letters, digits, '-', '_' and '.', starting and ending with a letter or digit", l) + } + return key, nil +} + +// ValidateTaint checks "key[=value]:Effect". +func ValidateTaint(t string) error { + m := taintRE.FindStringSubmatch(t) + if m == nil { + return fmt.Errorf("invalid taint %q: want key[=value]:Effect, with Effect one of %s", t, strings.Join(TaintEffects, ", ")) + } + if err := validateKey("taint", m[1]); err != nil { + return err + } + if v := m[2]; v != "" && (len(v) > 63 || !labelNameRE.MatchString(v)) { + return fmt.Errorf("invalid taint %q: the value must be at most 63 characters of letters, digits, '-', '_' and '.', starting and ending with a letter or digit", t) + } + for _, e := range TaintEffects { + if m[3] == e { + return nil + } + } + return fmt.Errorf("invalid taint %q: the effect must be one of %s", t, strings.Join(TaintEffects, ", ")) +} + +// ValidateHost checks the address of a k3s server: a DNS name, an IPv4 or an IPv6 address, without +// scheme, port or path. +func ValidateHost(what, h string) error { + if h == "" { + return fmt.Errorf("%s must not be empty", what) + } + if strings.ContainsAny(h, "/@ ?#") || strings.Contains(h, "://") { + return fmt.Errorf("invalid %s %q: give a host name or an IP address only, without scheme, port or path", what, h) + } + if ip := net.ParseIP(strings.Trim(h, "[]")); ip != nil { + return nil + } + if strings.Contains(h, ":") { + return fmt.Errorf("invalid %s %q: give a host name or an IP address only (the port is always 6443)", what, h) + } + if len(h) > 253 { + return fmt.Errorf("invalid %s %q: longer than 253 characters", what, h) + } + for _, p := range strings.Split(h, ".") { + if len(p) > 63 || !hostLabelRE.MatchString(p) { + return fmt.Errorf("invalid %s %q: want a host name or an IP address", what, h) + } + } + return nil +} + +// bareHost is h without the brackets of an IPv6 literal. +func bareHost(h string) string { return strings.Trim(h, "[]") } + +// K3SServerURL is the URL an agent joins: https://:6443 (an IPv6 address in brackets). +func K3SServerURL(host string) string { + h := bareHost(host) + if ip := net.ParseIP(h); ip != nil && ip.To4() == nil { + h = "[" + h + "]" + } + return "https://" + h + ":6443" +} + +// ServerURL is K3SServerURL of the Server field, or "" when none is set. +func (d TemplateData) ServerURL() string { + if d.Server == "" { + return "" + } + return K3SServerURL(d.Server) +} + +// MergeTLSSANs returns the --tls-san list of a server script: the server address (--server) first, +// then the explicit ones, without duplicates and without the two names the script always adds. +func MergeTLSSANs(server string, explicit []string, skip ...string) []string { + seen := map[string]bool{} + for _, s := range skip { + seen[strings.ToLower(s)] = true + } + var out []string + add := func(h string) { + h = bareHost(h) + if h == "" || seen[strings.ToLower(h)] { + return + } + seen[strings.ToLower(h)] = true + out = append(out, h) + } + add(server) + for _, h := range explicit { + add(h) + } + return out +} diff --git a/pkg/templates/node_test.go b/pkg/templates/node_test.go new file mode 100644 index 0000000..4636dbf --- /dev/null +++ b/pkg/templates/node_test.go @@ -0,0 +1,726 @@ +package templates + +import ( + "bytes" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +var updateGolden = flag.Bool("update-roles-golden", false, "rewrite the testdata/*.roles.golden.sh files") + +// ---------------------------------------------------------------- validation + +func TestNodeConfigValidate_Accepts(t *testing.T) { + ok := []NodeConfig{ + {}, + {Role: "vision"}, + {Role: "io", Robot: "mobile-01"}, + {Robot: "sensor-unit-01", Name: "sensor-1"}, + {Name: "my.node.example"}, + {Role: "vision", Robot: "vision-station-01", Labels: []string{"teknoir.org/gpu=nvidia", "plain=1", "empty=", "a.b-c/e_f.g=h"}, Taints: []string{"nvidia.com/gpu=true:NoSchedule", "dedicated:NoExecute", "x=y:PreferNoSchedule"}}, + } + for i, n := range ok { + if err := n.Validate(); err != nil { + t.Errorf("case %d %+v: %v", i, n, err) + } + } +} + +func TestNodeConfigValidate_Rejects(t *testing.T) { + long := strings.Repeat("a", 64) + bad := map[string]NodeConfig{ + "role upper case": {Role: "Vision"}, + "role underscore": {Role: "vi_sion"}, + "role slash": {Role: "a/b"}, + "role leading dash": {Role: "-x"}, + "role too long": {Role: long}, + "role quote": {Role: "a'b"}, + "robot dot": {Role: "io", Robot: "mobile.01"}, + "robot without role": {Robot: "mobile-01"}, + "name upper case": {Name: "Orin"}, + "name underscore": {Name: "or_in"}, + "name empty part": {Name: "a..b"}, + "label no equals": {Role: "x", Labels: []string{"foo"}}, + "label empty key": {Labels: []string{"=v"}}, + "label key space": {Labels: []string{"a b=c"}}, + "label value space": {Labels: []string{"a=b c"}}, + "label value too long": {Labels: []string{"a=" + long}}, + "label name too long": {Labels: []string{long + "=v"}}, + "label prefix upper": {Labels: []string{"Teknoir.org/x=1"}}, + "label quote": {Labels: []string{"a=b'c"}}, + "label dollar": {Labels: []string{"a=$b"}}, + "label kubernetes.io": {Labels: []string{"kubernetes.io/foo=bar"}}, + "label sub kubernetes": {Labels: []string{"node-role.kubernetes.io/worker=true"}}, + "label k8s.io": {Labels: []string{"k8s.io/foo=bar"}}, + "label sub k8s.io": {Labels: []string{"beta.k8s.io/foo=bar"}}, + "label role dup": {Role: "io", Labels: []string{LabelNodeRole + "=vision"}}, + "label robot dup": {Role: "io", Robot: "r", Labels: []string{LabelRobot + "=other"}}, + "label key twice": {Labels: []string{"a=1", "a=2"}}, + "taint no effect": {Taints: []string{"a=b"}}, + "taint bad effect": {Taints: []string{"a=b:Never"}}, + "taint lower effect": {Taints: []string{"a=b:noschedule"}}, + "taint empty key": {Taints: []string{"=b:NoSchedule"}}, + "taint kubernetes.io": {Taints: []string{"node.kubernetes.io/foo:NoSchedule"}}, + "taint value quote": {Taints: []string{"a=b'c:NoSchedule"}}, + "taint twice": {Taints: []string{"a=b:NoSchedule", "a=b:NoSchedule"}}, + "taint two colons": {Taints: []string{"a:b:NoSchedule"}}, + "taint value with space": {Taints: []string{"a=b c:NoSchedule"}}, + } + for name, n := range bad { + if err := n.Validate(); err == nil { + t.Errorf("%s: %+v was accepted", name, n) + } + } +} + +func TestNodeConfigValidate_ReservedPrefixErrorIsClear(t *testing.T) { + for _, l := range []string{"kubernetes.io/x=1", "foo.kubernetes.io/x=1", "k8s.io/x=1", "foo.k8s.io/x=1"} { + err := NodeConfig{Labels: []string{l}}.Validate() + if err == nil || !strings.Contains(err.Error(), "kubernetes.io/ and k8s.io/") { + t.Errorf("%s: error %v", l, err) + } + } + // a name that merely ends in the text is not the domain + if err := (NodeConfig{Labels: []string{"mykubernetes.io/x=1", "notk8s.io/y=1"}}).Validate(); err != nil { + t.Errorf("a different domain must be accepted: %v", err) + } +} + +func TestNodeConfigDerived(t *testing.T) { + n := NodeConfig{Role: "vision", Robot: "mobile-01", Labels: []string{"teknoir.org/gpu=nvidia"}} + if n.EffectiveName() != "mobile-01-vision" { + t.Errorf("name %q", n.EffectiveName()) + } + want := []string{"teknoir.org/node-role=vision", "teknoir.org/robot=mobile-01", "teknoir.org/gpu=nvidia"} + if got := n.AllLabels(); strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("labels %v", got) + } + if (NodeConfig{Role: "io"}).EffectiveName() != "" { + t.Error("a role without a robot must keep the default node name") + } + n.Name = "orin" + if n.EffectiveName() != "orin" { + t.Errorf("explicit name %q", n.EffectiveName()) + } + if (NodeConfig{}).Configured() || !(NodeConfig{Taints: []string{"a:NoSchedule"}}).Configured() { + t.Error("Configured") + } +} + +func TestValidateHost(t *testing.T) { + for _, h := range []string{"orin.local", "Orin", "10.0.0.5", "192.168.2.181", "fd00::1", "[fd00::1]", "a-b.c.example.com"} { + if err := ValidateHost("server", h); err != nil { + t.Errorf("%s: %v", h, err) + } + } + for _, h := range []string{"", "https://orin", "orin:6443", "orin/path", "o rin", "-orin", "orin_1", "a..b", "user@orin", "orin;rm", "$(x)", "'x'"} { + if err := ValidateHost("server", h); err == nil { + t.Errorf("%q was accepted", h) + } + } +} + +func TestK3SServerURL(t *testing.T) { + for in, want := range map[string]string{ + "orin.local": "https://orin.local:6443", + "10.0.0.5": "https://10.0.0.5:6443", + "fd00::1": "https://[fd00::1]:6443", + "[fd00::1]": "https://[fd00::1]:6443", + } { + if got := K3SServerURL(in); got != want { + t.Errorf("%s: %s, want %s", in, got, want) + } + } +} + +func TestMergeTLSSANs(t *testing.T) { + got := MergeTLSSANs("orin.local", []string{"10.0.0.5", "Orin.local", "dev-1.test.cloud", "x.example"}, "dev-1.test.cloud", "10.0.0.10") + if strings.Join(got, ",") != "orin.local,10.0.0.5,x.example" { + t.Errorf("%v", got) + } + if got := MergeTLSSANs("", nil); len(got) != 0 { + t.Errorf("%v", got) + } +} + +func TestValidateTemplateData_NodeFields(t *testing.T) { + for name, mut := range map[string]func(*TemplateData){ + "role": func(d *TemplateData) { d.Node.Role = "a$b" }, + "robot": func(d *TemplateData) { d.Node.Robot = "a`b" }, + "name": func(d *TemplateData) { d.Node.Name = "a\"b" }, + "label": func(d *TemplateData) { d.Node.Labels = []string{"a=b\\c"} }, + "taint": func(d *TemplateData) { d.Node.Taints = []string{"a=b\n:NoSchedule"} }, + "server": func(d *TemplateData) { d.Server = "a$(x)" }, + "san": func(d *TemplateData) { d.TLSSANs = []string{"a\"b"} }, + } { + d := goldenData() + mut(&d) + if err := ValidateTemplateData(d); err == nil { + t.Errorf("%s: unsafe value accepted", name) + } + } +} + +// ---------------------------------------------------------------- rendering + +func serverRolesData() TemplateData { + d := goldenData() + d.Node = NodeConfig{Role: "vision", Robot: "mobile-01", Labels: []string{"teknoir.org/gpu=nvidia"}, Taints: []string{"nvidia.com/gpu=true:NoSchedule"}} + d.Server = "orin.example.test" + d.TLSSANs = MergeTLSSANs("orin.example.test", []string{"10.0.0.5"}, "golden-device.example.test", "10.0.0.10") + return d +} + +func agentRolesData() TemplateData { + d := goldenData() + d.Node = NodeConfig{Role: "io", Robot: "mobile-01", Labels: []string{"teknoir.org/io=cameras"}} + d.Server = "orin.example.test" + return d +} + +func TestGolden_Roles(t *testing.T) { + for name, tc := range map[string]struct { + render func() ([]byte, error) + }{ + "server": {func() ([]byte, error) { + var b bytes.Buffer + err := ServerTemplate.Execute(&b, serverRolesData()) + return b.Bytes(), err + }}, + "agent": {func() ([]byte, error) { + var b bytes.Buffer + err := AgentTemplate.Execute(&b, agentRolesData()) + return b.Bytes(), err + }}, + } { + t.Run(name, func(t *testing.T) { + got, err := tc.render() + if err != nil { + t.Fatal(err) + } + path := "testdata/" + name + ".roles.golden.sh" + if *updateGolden { + if err := os.WriteFile(path, got, 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("%s differs from %s (got %d bytes, want %d); go test -run Golden_Roles -update-roles-golden rewrites it", name, path, len(got), len(want)) + } + }) + } +} + +// Without any node flag the scripts do not mention the node library at all (the golden files prove the +// bytes; this proves the reason). +func TestNoNodeFlagsNoNodeCode(t *testing.T) { + for name, out := range map[string]string{"server": renderK3S(t, ServerTemplate, ""), "agent": renderK3S(t, AgentTemplate, "")} { + for _, s := range []string{"tn_node_", "tn-k3s-node", "teknoir.org/node-role", "teknoir.org/robot", "98-teknoir-node"} { + if strings.Contains(out, s) { + t.Errorf("%s script without node flags contains %q", name, s) + } + } + } + if !strings.Contains(renderK3S(t, ServerTemplate, ""), `node-name: teknoir-master`) { + t.Error("the server default node name is gone") + } +} + +func TestRolesRendering_Server(t *testing.T) { + var b bytes.Buffer + if err := ServerTemplate.Execute(&b, serverRolesData()); err != nil { + t.Fatal(err) + } + out := b.String() + for _, want := range []string{ + "tn_node_name 'mobile-01-vision'\n", + "tn_node_label 'teknoir.org/node-role=vision'\n", + "tn_node_label 'teknoir.org/robot=mobile-01'\n", + "tn_node_label 'teknoir.org/gpu=nvidia'\n", + "tn_node_taint 'nvidia.com/gpu=true:NoSchedule'\n", + "tn_node_apply server\n", + "--tls-san 10.0.0.10 --tls-san orin.example.test --tls-san 10.0.0.5${INSTALL_K3S_EXEC", + } { + if !strings.Contains(out, want) { + t.Errorf("server script lacks %q", want) + } + } + // the role name replaces the default, it is not written next to it + if strings.Contains(out, `"node-name: teknoir-master"`) { + t.Error("the default node name is still written") + } + // the labels are written before the k3s installer runs + if a, i := strings.Index(out, "tn_node_apply server"), strings.Index(out, "$SUDO ./k3s_installer.sh"); a < 0 || i < a { + t.Errorf("tn_node_apply (%d) must come before the installer (%d)", a, i) + } +} + +func TestRolesRendering_ServerKeepsDefaultNameWithoutRobot(t *testing.T) { + d := goldenData() + d.Node = NodeConfig{Role: "vision"} + var b bytes.Buffer + if err := ServerTemplate.Execute(&b, d); err != nil { + t.Fatal(err) + } + out := b.String() + if !strings.Contains(out, `"node-name: teknoir-master"`) || strings.Contains(out, "tn_node_name '") { + t.Error("a role without a robot must keep the default server node name") + } + if !strings.Contains(out, "tn_node_apply server") || !strings.Contains(out, "tn_node_label 'teknoir.org/node-role=vision'") { + t.Error("the role label is missing") + } +} + +func TestRolesRendering_Agent(t *testing.T) { + var b bytes.Buffer + if err := AgentTemplate.Execute(&b, agentRolesData()); err != nil { + t.Fatal(err) + } + out := b.String() + for _, want := range []string{ + "tn_node_name 'mobile-01-io'\n", + "tn_node_label 'teknoir.org/node-role=io'\n", + "tn_node_label 'teknoir.org/io=cameras'\n", + `K3S_URL="${K3S_URL:-https://orin.example.test:6443}"`, + "tn_node_apply agent\n", + } { + if !strings.Contains(out, want) { + t.Errorf("agent script lacks %q", want) + } + } + if strings.Contains(out, "tn_node_taint '") { + t.Error("no taint was asked for") + } + // K3S_URL is set before the question, and the labels are written after the release check and + // before the installer + u, q := strings.Index(out, `K3S_URL="${K3S_URL:-`), strings.Index(out, `if [ -z "${K3S_URL}" ]`) + p, a, i := strings.Index(out, "\ntn_inst_prepare agent\n"), strings.Index(out, "tn_node_apply agent"), strings.Index(out, "$SUDO ./k3s_installer.sh") + if !(u > 0 && u < q && p > 0 && p < a && a < i) { + t.Errorf("order: url %d question %d prepare %d apply %d installer %d", u, q, p, a, i) + } +} + +// --server on its own: an agent without a role still joins the right server +func TestRolesRendering_AgentServerOnly(t *testing.T) { + d := goldenData() + d.Server = "fd00::7" + var b bytes.Buffer + if err := AgentTemplate.Execute(&b, d); err != nil { + t.Fatal(err) + } + out := b.String() + if !strings.Contains(out, `K3S_URL="${K3S_URL:-https://[fd00::7]:6443}"`) || strings.Contains(out, "tn_node_") { + t.Errorf("unexpected agent script") + } +} + +// the rendered K3S_URL line: the environment wins +func TestAgentServerIsOverridableByEnvironment(t *testing.T) { + needTool(t, "sh") + for env, want := range map[string]string{"": "https://orin.example.test:6443", "K3S_URL=https://other:6443": "https://other:6443"} { + cmd := exec.Command("sh", "-c", `K3S_URL="${K3S_URL:-https://orin.example.test:6443}"; echo "$K3S_URL"`) + cmd.Env = []string{"PATH=" + os.Getenv("PATH")} + if env != "" { + cmd.Env = append(cmd.Env, env) + } + out, err := cmd.Output() + if err != nil || strings.TrimSpace(string(out)) != want { + t.Errorf("%q: %q %v", env, out, err) + } + } + // and the line is the one in the template + var b bytes.Buffer + if err := AgentTemplate.Execute(&b, agentRolesData()); err != nil { + t.Fatal(err) + } + if !strings.Contains(b.String(), `K3S_URL="${K3S_URL:-https://orin.example.test:6443}"`) { + t.Error("the template line differs from the one tested") + } +} + +// ---------------------------------------------------------------- the shell library + +func TestNodeLibraryIsPlainShell(t *testing.T) { + needTool(t, "sh") + b, err := os.ReadFile("scripts/tn-k3s-node.sh") + if err != nil { + t.Fatal(err) + } + if bytes.Contains(b, []byte("{{")) { + t.Fatal("the library is embedded as a template: it must not contain '{{'") + } + if out, err := exec.Command("sh", "-n", "scripts/tn-k3s-node.sh").CombinedOutput(); err != nil { + t.Fatalf("sh -n: %v\n%s", err, out) + } + if string(b) != TnK3sNodeScript { + t.Error("embedded TnK3sNodeScript differs from the file") + } + for _, bad := range []string{"\ntrap ", "set +o noglob", "set -x"} { + if strings.Contains(string(b), bad) { + t.Errorf("library must not contain %q", bad) + } + } +} + +type nodeEnv struct { + t *testing.T + dir string + cfg string +} + +func newNodeEnv(t *testing.T) *nodeEnv { + t.Helper() + needTool(t, "sh") + d := t.TempDir() + return &nodeEnv{t: t, dir: d, cfg: filepath.Join(d, "rancher-k3s")} +} + +// run sources both libraries under the options of the bootstrap scripts and runs script. awk is the +// first of awks (default: the system's). +func (n *nodeEnv) run(script string, env ...string) (string, error) { + n.t.Helper() + full := "set -e; set -o noglob; umask 027; . scripts/tn-k3s-install.sh; . scripts/tn-k3s-node.sh\n" + script + cmd := exec.Command("sh", "-c", full) + for _, e := range os.Environ() { + if strings.HasPrefix(e, "TN_NODE_") || strings.HasPrefix(e, "TN_K3S_") || strings.HasPrefix(e, "DATA_DIR") { + continue + } + cmd.Env = append(cmd.Env, e) + } + cmd.Env = append(cmd.Env, "TN_K3S_CONFIG_DIR="+n.cfg, "DATA_DIR="+filepath.Join(n.dir, "data")) + cmd.Env = append(cmd.Env, env...) + out, err := cmd.CombinedOutput() + return string(out), err +} + +func (n *nodeEnv) file(rel string) string { + b, err := os.ReadFile(filepath.Join(n.cfg, rel)) + if err != nil { + return "<" + err.Error() + ">" + } + return string(b) +} + +const visionNode = `tn_node_name 'mobile-01-vision' +tn_node_label 'teknoir.org/node-role=vision' +tn_node_label 'teknoir.org/robot=mobile-01' +tn_node_label 'teknoir.org/gpu=nvidia' +tn_node_taint 'nvidia.com/gpu=true:NoSchedule' +tn_node_apply server +` + +const wantDropin = `# Written by the Teknoir bootstrap script (tnctl bootstrap --role ...). Do not edit: a re-run rewrites it. +# "+" appends to the node-label and node-taint lists of config.yaml and the other drop-ins. +# k3s applies them only when the node registers for the first time. +node-label+: + - "teknoir.org/node-role=vision" + - "teknoir.org/robot=mobile-01" + - "teknoir.org/gpu=nvidia" +node-taint+: + - "nvidia.com/gpu=true:NoSchedule" +` + +// The config writer, run twice on an empty directory: the second run changes nothing. +func TestNodeWriter_IdempotentOnEmptyDir(t *testing.T) { + n := newNodeEnv(t) + out, err := n.run(visionNode) + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + if got := n.file("config.yaml"); got != "node-name: mobile-01-vision\n" { + t.Errorf("config.yaml:\n%s", got) + } + if got := n.file("config.yaml.d/98-teknoir-node.yaml"); got != wantDropin { + t.Errorf("drop-in:\n%s", got) + } + if strings.Contains(out, "registered") { + t.Errorf("a new node must not get the registered warning:\n%s", out) + } + before := snapshot(t, n.cfg) + + out2, err := n.run(visionNode) + if err != nil { + t.Fatalf("second run: %v\n%s", err, out2) + } + if after := snapshot(t, n.cfg); after != before { + t.Errorf("the second run changed the configuration:\nbefore:\n%s\nafter:\n%s", before, after) + } + if !strings.Contains(out2, "already in place") || !strings.Contains(out2, "already sets 'node-name'") { + t.Errorf("second run output:\n%s", out2) + } + if strings.Contains(out2, "[WARN]") { + t.Errorf("an idempotent re-run must not warn:\n%s", out2) + } + if left, _ := filepath.Glob(filepath.Join(n.cfg, "config.yaml.d", ".*")); len(left) != 0 { + t.Errorf("temporary files left: %v", left) + } +} + +func snapshot(t *testing.T, dir string) string { + t.Helper() + var b strings.Builder + err := filepath.Walk(dir, func(p string, fi os.FileInfo, err error) error { + if err != nil || fi.IsDir() { + return err + } + c, err := os.ReadFile(p) + rel, _ := filepath.Rel(dir, p) + fmt.Fprintf(&b, "== %s (%s, %d)\n%s", rel, fi.Mode(), fi.Size(), c) + return err + }) + if err != nil { + t.Fatal(err) + } + return b.String() +} + +// Keys the device owner set are never clobbered: the user's config.yaml is not rewritten, its +// node-label stays, ours are appended by the drop-in; a different node-name is kept and reported. +func TestNodeWriter_KeepsTheOwnersKeys(t *testing.T) { + n := newNodeEnv(t) + user := "kubelet-arg:\n - max-pods=77\nnode-label:\n - owner=1\nnode-name: owners-name # mine\ndisable:\n - traefik" + write(t, filepath.Join(n.cfg, "config.yaml"), user) // no trailing newline on purpose + out, err := n.run(visionNode) + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + if got := n.file("config.yaml"); got != user { + t.Errorf("config.yaml was changed:\n%s", got) + } + if got := n.file("config.yaml.d/98-teknoir-node.yaml"); got != wantDropin { + t.Errorf("drop-in:\n%s", got) + } + if !strings.Contains(out, "sets node-name 'owners-name', not 'mobile-01-vision'") { + t.Errorf("the different node-name is not reported:\n%s", out) + } +} + +// The user's config.yaml lacks node-name and does not end in a newline: the name is appended on a new line. +func TestNodeWriter_AppendsNodeNameToExistingConfig(t *testing.T) { + n := newNodeEnv(t) + write(t, filepath.Join(n.cfg, "config.yaml"), "disable:\n - traefik") + if out, err := n.run(visionNode); err != nil { + t.Fatalf("%v\n%s", err, out) + } + if got := n.file("config.yaml"); got != "disable:\n - traefik\nnode-name: mobile-01-vision\n" { + t.Errorf("config.yaml:\n%q", got) + } +} + +// Same name set by the owner: kept silently (no warning). +func TestNodeWriter_SameNameNoWarning(t *testing.T) { + n := newNodeEnv(t) + write(t, filepath.Join(n.cfg, "config.yaml"), "node-name: \"mobile-01-vision\"\n") + out, err := n.run(visionNode) + if err != nil || strings.Contains(out, "[WARN]") { + t.Fatalf("%v\n%s", err, out) + } +} + +// Labels and taints only: no node-name line is written (the script keeps its own default). +func TestNodeWriter_NoNameNoNodeName(t *testing.T) { + n := newNodeEnv(t) + out, err := n.run("tn_node_label 'teknoir.org/node-role=io'\ntn_node_apply agent") + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + if _, err := os.Stat(filepath.Join(n.cfg, "config.yaml")); err == nil { + t.Errorf("config.yaml was created: %s", n.file("config.yaml")) + } + if got := n.file("config.yaml.d/98-teknoir-node.yaml"); !strings.Contains(got, ` - "teknoir.org/node-role=io"`) || strings.Contains(got, "node-taint+") { + t.Errorf("drop-in:\n%s", got) + } +} + +// Only a name (--node-name without labels): config.yaml gets it, no drop-in. +func TestNodeWriter_NameOnly(t *testing.T) { + n := newNodeEnv(t) + if out, err := n.run("tn_node_name pi-1\ntn_node_apply agent"); err != nil { + t.Fatalf("%v\n%s", err, out) + } + if n.file("config.yaml") != "node-name: pi-1\n" { + t.Errorf("config.yaml:\n%s", n.file("config.yaml")) + } + if _, err := os.Stat(filepath.Join(n.cfg, "config.yaml.d")); err == nil { + t.Error("a drop-in directory was created for nothing") + } +} + +// A changed role on a re-run rewrites our file and leaves everything else alone. +func TestNodeWriter_ChangedRoleRewritesOurFileOnly(t *testing.T) { + n := newNodeEnv(t) + write(t, filepath.Join(n.cfg, "config.yaml.d", "50-owner.yaml"), "node-label+:\n - mine=1\n") + if out, err := n.run(visionNode); err != nil { + t.Fatalf("%v\n%s", err, out) + } + other := strings.ReplaceAll(visionNode, "vision", "io") + other = strings.ReplaceAll(other, "tn_node_label 'teknoir.org/gpu=nvidia'\n", "") + out, err := n.run(other) + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + got := n.file("config.yaml.d/98-teknoir-node.yaml") + if !strings.Contains(got, `"teknoir.org/node-role=io"`) || strings.Contains(got, "vision") || strings.Contains(got, "teknoir.org/gpu") { + t.Errorf("drop-in not rewritten:\n%s", got) + } + if n.file("config.yaml.d/50-owner.yaml") != "node-label+:\n - mine=1\n" { + t.Error("the owner's drop-in was touched") + } + // the name stays the one that was registered first + if !strings.Contains(out, "sets node-name 'mobile-01-vision', not 'mobile-01-io'") { + t.Errorf("output:\n%s", out) + } +} + +// The labels do not reach a node that has registered: the script warns, with the kubectl commands. +func TestNodeWriter_WarnsWhenAlreadyRegistered(t *testing.T) { + n := newNodeEnv(t) + write(t, filepath.Join(n.dir, "data", "agent", "client-kubelet.crt"), "cert") + out, err := n.run(visionNode) + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + for _, want := range []string{ + "registered with the cluster already", + "k3s kubectl label node mobile-01-vision teknoir.org/node-role=vision teknoir.org/robot=mobile-01 teknoir.org/gpu=nvidia --overwrite", + "k3s kubectl taint node mobile-01-vision nvidia.com/gpu=true:NoSchedule --overwrite", + } { + if !strings.Contains(out, want) { + t.Errorf("output lacks %q:\n%s", want, out) + } + } + // once the file is in place the labels are what was asked for: no further warning + out2, err := n.run(visionNode) + if err != nil || strings.Contains(out2, "[WARN]") { + t.Fatalf("second run: %v\n%s", err, out2) + } + // a change of the labels warns again + out3, err := n.run(strings.Replace(visionNode, "nvidia'", "amd'", 1)) + if err != nil || !strings.Contains(out3, "registered with the cluster already") { + t.Fatalf("third run: %v\n%s", err, out3) + } +} + +func TestNodeWriter_RegisteredOverride(t *testing.T) { + n := newNodeEnv(t) + out, err := n.run(visionNode, "TN_NODE_REGISTERED=true") + if err != nil || !strings.Contains(out, "registered with the cluster already") { + t.Fatalf("%v\n%s", err, out) + } + n2 := newNodeEnv(t) + write(t, filepath.Join(n2.dir, "data", "agent", "client-kubelet.crt"), "cert") + out, err = n2.run(visionNode, "TN_NODE_REGISTERED=false") + if err != nil || strings.Contains(out, "registered with the cluster already") { + t.Fatalf("%v\n%s", err, out) + } +} + +// The warning names the node as it is called in config.yaml, or by host name when nothing sets it. +func TestNodeWriter_RegisteredNodeNameFallbacks(t *testing.T) { + n := newNodeEnv(t) + write(t, filepath.Join(n.cfg, "config.yaml"), "node-name: owners-name\n") + out, err := n.run("tn_node_label 'a=b'\ntn_node_apply server", "TN_NODE_REGISTERED=true") + if err != nil || !strings.Contains(out, "label node owners-name a=b --overwrite") { + t.Fatalf("%v\n%s", err, out) + } + n = newNodeEnv(t) + out, err = n.run("tn_node_label 'a=b'\ntn_node_apply agent", "TN_NODE_REGISTERED=true") + host, _ := os.Hostname() + if err != nil || !strings.Contains(out, "label node "+strings.ToLower(host)+" a=b --overwrite") { + t.Fatalf("%v\n%s", err, out) + } +} + +// The YAML reader of the library works with every awk that sh scripts meet. +func TestNodeWriter_YamlValueWithAllAwks(t *testing.T) { + needTool(t, "sh") + yaml := "a: 1\nnode-name: 'quoted-name' # comment\nnode-name2: no\r\n" + for _, awk := range []string{"awk", "gawk", "mawk", "busybox"} { + if _, err := exec.LookPath(awk); err != nil { + continue + } + t.Run(awk, func(t *testing.T) { + dir := t.TempDir() + write(t, filepath.Join(dir, "c.yaml"), yaml) + bin := filepath.Join(dir, "bin") + target, _ := exec.LookPath(awk) + if awk == "busybox" { + write(t, filepath.Join(bin, "awk"), "#!/bin/sh\nexec "+target+" awk \"$@\"\n") + } else { + write(t, filepath.Join(bin, "awk"), "#!/bin/sh\nexec "+target+" \"$@\"\n") + } + if err := os.Chmod(filepath.Join(bin, "awk"), 0o755); err != nil { + t.Fatal(err) + } + cmd := exec.Command("sh", "-c", ". scripts/tn-k3s-install.sh; . scripts/tn-k3s-node.sh; tn_node_yaml_value "+filepath.Join(dir, "c.yaml")+" node-name") + cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH")) + out, err := cmd.CombinedOutput() + if err != nil || strings.TrimSpace(string(out)) != "quoted-name" { + t.Errorf("%q %v", out, err) + } + }) + } +} + +// The whole rendered scripts, with the node part run for real: from the first line of the libraries to the +// call of tn_node_apply, in a temporary configuration directory. +func TestRenderedNodeBlock_RunsAndIsIdempotent(t *testing.T) { + needTool(t, "sh") + for name, d := range map[string]struct { + data TemplateData + role string + want string + }{ + "server": {serverRolesData(), "server", "mobile-01-vision"}, + "agent": {agentRolesData(), "agent", "mobile-01-io"}, + } { + t.Run(name, func(t *testing.T) { + var b bytes.Buffer + tpl := ServerTemplate + if name == "agent" { + tpl = AgentTemplate + } + if err := tpl.Execute(&b, d.data); err != nil { + t.Fatal(err) + } + out := b.String() + // the calls the template renders, between the library and the keys/k3s_version section + start := strings.Index(out, "\ntn_node_name ") + end := strings.Index(out, "tn_node_label 'teknoir.org/robot=") + if start < 0 || end < 0 { + t.Fatal("no node calls in the script") + } + lines := out[start:] + var calls []string + for _, l := range strings.Split(lines, "\n") { + if strings.HasPrefix(l, "tn_node_name ") || strings.HasPrefix(l, "tn_node_label ") || strings.HasPrefix(l, "tn_node_taint ") { + calls = append(calls, l) + } + } + n := newNodeEnv(t) + script := strings.Join(calls, "\n") + "\ntn_node_apply " + d.role + "\n" + if o, err := n.run(script); err != nil { + t.Fatalf("%v\n%s", err, o) + } + first := snapshot(t, n.cfg) + if o, err := n.run(script); err != nil { + t.Fatalf("%v\n%s", err, o) + } + if snapshot(t, n.cfg) != first { + t.Error("not idempotent") + } + if !strings.Contains(first, "node-name: "+d.want+"\n") || !strings.Contains(first, `"teknoir.org/robot=mobile-01"`) { + t.Errorf("unexpected configuration:\n%s", first) + } + }) + } +} diff --git a/pkg/templates/render.go b/pkg/templates/render.go index 8ab21e8..f046aa8 100644 --- a/pkg/templates/render.go +++ b/pkg/templates/render.go @@ -32,6 +32,10 @@ type field struct { // newline (or carriage return) is rejected in every field except RSAPrivate // and RSAPublic, which are multi-line PEM material. // +// The node configuration (role, robot, labels, taints, server address) is checked for the same +// characters here; its syntax is checked by NodeConfig.Validate and ValidateHost, which `tnctl +// bootstrap` calls before it asks the platform for the device. +// // Errors name the field and the offending character class, never the value. func ValidateTemplateData(d TemplateData) error { fields := []field{ @@ -45,6 +49,19 @@ func ValidateTemplateData(d TemplateData) error { {"PublicSSHKey", d.PublicSSHKey, false}, {"K3SToken", d.K3SToken, false}, {"DockerSecret", d.DockerSecret, false}, + {"Node.Role", d.Node.Role, false}, + {"Node.Robot", d.Node.Robot, false}, + {"Node.Name", d.Node.Name, false}, + {"Server", d.Server, false}, + } + for _, l := range d.Node.Labels { + fields = append(fields, field{"Node.Labels", l, false}) + } + for _, t := range d.Node.Taints { + fields = append(fields, field{"Node.Taints", t, false}) + } + for _, s := range d.TLSSANs { + fields = append(fields, field{"TLSSANs", s, false}) } for _, f := range fields { if strings.ContainsAny(f.value, unsafeChars) { diff --git a/pkg/templates/scripts/tn-k3s-node.sh b/pkg/templates/scripts/tn-k3s-node.sh new file mode 100644 index 0000000..5efda89 --- /dev/null +++ b/pkg/templates/scripts/tn-k3s-node.sh @@ -0,0 +1,144 @@ +# tn-k3s-node.sh: the node name, role labels and taints of a k3s node (tnctl bootstrap --role ...). +# +# Embedded in the server and the agent bootstrap script (only when the script was generated with a node +# role, label, taint or name), and usable stand-alone from a test. Usage: +# +# . tn-k3s-install.sh; . tn-k3s-node.sh +# tn_node_name NAME (the node name; optional) +# tn_node_label key=value (once per label; the role and robot labels come first) +# tn_node_taint key[=value]:Effect (once per taint) +# tn_node_apply server|agent (writes the k3s configuration; once, before the k3s installer) +# +# It needs tn_inst_config_ensure, tn_inst_yaml_has_key and tn_inst_info/warn from tn-k3s-install.sh. +# POSIX sh. It runs under `set -e` and `set -o noglob` and does not change shell options. +# +# What it writes (TN_K3S_CONFIG_DIR, /etc/rancher/k3s by default): +# config.yaml node-name, added only when config.yaml has no node-name already: a +# name the device owner set is kept (a different one is reported) +# config.yaml.d/98-teknoir-node.yaml node-label+ and node-taint+: the labels and taints of the role. +# "+" APPENDS to the node-label and node-taint lists of config.yaml and +# the other drop-ins, so the entries of the device owner are kept. The +# file is ours: it is rewritten (atomically) when it differs and left +# alone when it does not, so a re-run changes nothing. +# +# k3s applies node-label and node-taint ONLY when a node registers for the first time. When the node has +# registered already and the file changes (or is new), the script says so and prints the kubectl commands +# that apply the labels and taints to the running node; run them on the k3s server. +# +# Environment: +# TN_K3S_CONFIG_DIR /etc/rancher/k3s +# DATA_DIR the k3s data dir (/var/lib/rancher/k3s): /agent/client-kubelet.crt exists +# once the kubelet of this node has registered +# TN_NODE_REGISTERED true / false: do not look at the data dir, take this as the answer (tests) + +TN_K3S_NODE_LIB_VERSION=1 +export TN_K3S_NODE_LIB_VERSION + +TN_NODE_NAME= +TN_NODE_LABELS= +TN_NODE_TAINTS= +TN_NODE_DROPIN=98-teknoir-node.yaml + +tn_node_name() { TN_NODE_NAME=$1; } +tn_node_label() { TN_NODE_LABELS="${TN_NODE_LABELS:+$TN_NODE_LABELS +}$1"; } +tn_node_taint() { TN_NODE_TAINTS="${TN_NODE_TAINTS:+$TN_NODE_TAINTS +}$1"; } + +# tn_node_yaml_value FILE KEY: the value of the top-level scalar KEY (quotes and a trailing comment removed) +tn_node_yaml_value() { + [ -r "$1" ] || return 0 + awk -v k="$2" ' + { sub(/\r$/, "") } + $0 ~ ("^" k "[ \t]*:") { + v = $0 + sub("^" k "[ \t]*:[ \t]*", "", v) + sub(/[ \t]+#.*$/, "", v) + gsub(/^["\047]|["\047]$/, "", v) + print v + exit + }' "$1" +} + +# tn_node_registered: has the kubelet of this node a client certificate, i.e. has the node registered +tn_node_registered() { + case ${TN_NODE_REGISTERED:-} in + true) return 0 ;; + false) return 1 ;; + esac + [ -s "${DATA_DIR:-/var/lib/rancher/k3s}/agent/client-kubelet.crt" ] +} + +# tn_node_dropin_text: the content of the labels and taints file +tn_node_dropin_text() { + printf '%s\n' \ + '# Written by the Teknoir bootstrap script (tnctl bootstrap --role ...). Do not edit: a re-run rewrites it.' \ + '# "+" appends to the node-label and node-taint lists of config.yaml and the other drop-ins.' \ + '# k3s applies them only when the node registers for the first time.' + if [ -n "$TN_NODE_LABELS" ]; then + echo 'node-label+:' + printf '%s\n' "$TN_NODE_LABELS" | while IFS= read -r _tn_l; do printf ' - "%s"\n' "$_tn_l"; done + fi + if [ -n "$TN_NODE_TAINTS" ]; then + echo 'node-taint+:' + printf '%s\n' "$TN_NODE_TAINTS" | while IFS= read -r _tn_t; do printf ' - "%s"\n' "$_tn_t"; done + fi +} + +# tn_node_kubectl_help NODE: how to apply the labels and taints to a node that has registered already +tn_node_kubectl_help() { + _tn_kl= + if [ -n "$TN_NODE_LABELS" ]; then + _tn_kl=$(printf '%s\n' "$TN_NODE_LABELS" | tr '\n' ' ') + tn_inst_warn " k3s kubectl label node $1 ${_tn_kl}--overwrite" + fi + if [ -n "$TN_NODE_TAINTS" ]; then + _tn_kl=$(printf '%s\n' "$TN_NODE_TAINTS" | tr '\n' ' ') + tn_inst_warn " k3s kubectl taint node $1 ${_tn_kl}--overwrite" + fi +} + +# tn_node_apply ROLE: write the node name, and the labels and taints, into the k3s configuration +tn_node_apply() { + _tn_nrole=${1:-server} + _tn_ncfg=${TN_K3S_CONFIG_DIR:-/etc/rancher/k3s} + _tn_nyaml=$_tn_ncfg/config.yaml + mkdir -p "$_tn_ncfg" || tn_inst_fatal "cannot create $_tn_ncfg" + + # the name of the node: a node-name in config.yaml wins (it is the owner's), a different one is reported + if [ -n "$TN_NODE_NAME" ]; then + if tn_inst_yaml_has_key "$_tn_nyaml" node-name; then + _tn_have=$(tn_node_yaml_value "$_tn_nyaml" node-name) + if [ "$_tn_have" != "$TN_NODE_NAME" ]; then + tn_inst_warn "$_tn_nyaml sets node-name '$_tn_have', not '$TN_NODE_NAME' as asked for this $_tn_nrole: kept. The node keeps its name; edit the file to rename it (k3s registers a renamed node as a new one)" + fi + fi + tn_inst_config_ensure "$_tn_nyaml" node-name "node-name: $TN_NODE_NAME" + fi + + [ -n "$TN_NODE_LABELS$TN_NODE_TAINTS" ] || return 0 + + _tn_nfile=$_tn_ncfg/config.yaml.d/$TN_NODE_DROPIN + _tn_nwant=$(tn_node_dropin_text) + if [ -f "$_tn_nfile" ] && [ "$(cat "$_tn_nfile")" = "$_tn_nwant" ]; then + tn_inst_info "node labels and taints are already in place ($_tn_nfile)" + return 0 + fi + # atomically, and under a name that k3s does not read (it does not end in .yaml) + _tn_ntmp=$_tn_ncfg/config.yaml.d/.$TN_NODE_DROPIN.tmp.$$ + if mkdir -p "$_tn_ncfg/config.yaml.d" && printf '%s\n' "$_tn_nwant" > "$_tn_ntmp" && mv -f "$_tn_ntmp" "$_tn_nfile"; then + : + else + rm -f "$_tn_ntmp" 2>/dev/null || true + tn_inst_fatal "cannot write $_tn_nfile" + fi + tn_inst_info "Wrote the node labels and taints of this $_tn_nrole to $_tn_nfile" + + if tn_node_registered; then + _tn_nname=$(tn_node_yaml_value "$_tn_nyaml" node-name) + _tn_nname=${_tn_nname:-${TN_NODE_NAME:-$(hostname | tr 'A-Z' 'a-z')}} + tn_inst_warn "This node has registered with the cluster already, and k3s applies node-label and node-taint only when a node registers for the first time: the labels and taints above will NOT show up on it by themselves." + tn_inst_warn "Apply them to the running node from the k3s server (this changes the Node object only; the file above keeps them if the node ever registers again):" + tn_node_kubectl_help "$_tn_nname" + fi +} diff --git a/pkg/templates/server.sh.tmpl b/pkg/templates/server.sh.tmpl index 2dff3f6..1a75704 100644 --- a/pkg/templates/server.sh.tmpl +++ b/pkg/templates/server.sh.tmpl @@ -48,6 +48,21 @@ TN_K3S_CA_NO_MAIN=1 {{template "tn-k3s-ca.sh" .}} # The k3s release policy (a re-run keeps the installed k3s) and the cgroup v1 kubelet flag. {{template "tn-k3s-install.sh" .}} +{{- if .Node.Configured}} + +# The node name, role labels and taints of this node (tnctl bootstrap --role ...). k3s applies the labels +# and taints only when the node registers for the first time, so they are written before the installer runs. +{{template "tn-k3s-node.sh" .}} +{{- if .Node.EffectiveName}} +tn_node_name '{{.Node.EffectiveName}}' +{{- end}} +{{- range .Node.AllLabels}} +tn_node_label '{{.}}' +{{- end}} +{{- range .Node.Taints}} +tn_node_taint '{{.}}' +{{- end}} +{{- end}} {{template "k3s_version.sh.tmpl" .}} # Order: (0) the arguments of the k3s installer, (1) the k3s release policy and the cgroup v1 kubelet @@ -61,7 +76,7 @@ TN_K3S_CA_NO_MAIN=1 # it, because a k3s datastore there decides whether a k3s whose version cannot be read is installed; a # data dir that disagrees with config.yaml is refused here, before (1) changes anything. # BEGIN install-k3s-exec -export INSTALL_K3S_EXEC="server --token ${_K3S_TOKEN} --tls-san ${_DEVICE_ID}.${_DOMAIN} --tls-san 10.0.0.10${INSTALL_K3S_EXEC:+ ${INSTALL_K3S_EXEC}}" +export INSTALL_K3S_EXEC="server --token ${_K3S_TOKEN} --tls-san ${_DEVICE_ID}.${_DOMAIN} --tls-san 10.0.0.10{{range .TLSSANs}} --tls-san {{.}}{{end}}${INSTALL_K3S_EXEC:+ ${INSTALL_K3S_EXEC}}" if [ "${USE_DOCKER}" = true ]; then info "Use docker container-runtime for K3s" export INSTALL_K3S_EXEC="${INSTALL_K3S_EXEC} --docker" @@ -79,8 +94,10 @@ $SUDO mkdir -p ${K3S_CONFIG_PATH} # like a kubelet-arg): the Teknoir keys are added when they are missing. tn_inst_config_ensure ${K3S_CONFIG_PATH}/config.yaml disable "disable: - traefik" -tn_inst_config_ensure ${K3S_CONFIG_PATH}/config.yaml node-name "node-name: teknoir-master" -# The CA above was made in ${DATA_DIR}, the data dir that k3s uses on this device. The file just +{{if .Node.EffectiveName}}tn_node_apply server +{{else}}tn_inst_config_ensure ${K3S_CONFIG_PATH}/config.yaml node-name "node-name: teknoir-master" +{{if .Node.Configured}}tn_node_apply server +{{end}}{{end}}# The CA above was made in ${DATA_DIR}, the data dir that k3s uses on this device. The file just # written and the unit that the installer writes (it has no --data-dir) must keep it there. if [ "${DATA_DIR}" != "/var/lib/rancher/k3s" ]; then tn_inst_config_ensure ${K3S_CONFIG_PATH}/config.yaml data-dir "data-dir: ${DATA_DIR}" diff --git a/pkg/templates/testdata/agent.roles.golden.sh b/pkg/templates/testdata/agent.roles.golden.sh new file mode 100644 index 0000000..9a3654d --- /dev/null +++ b/pkg/templates/testdata/agent.roles.golden.sh @@ -0,0 +1,1055 @@ +#!/bin/sh +set -e +set -o noglob +#set -x +umask 027 + +# --- helper functions --- +info() { echo '[INFO] ' "$@"; } +warn() { echo '[WARN] ' "$@" >&2; } +fatal() { echo '[ERROR] ' "$@" >&2; exit 1; } + +# Exit if not running as root +[ $(id -u) -eq 0 ] || fatal 'You must run this script as root' + +DOWNLOADER= +# --- download --- +download() { + [ $# -eq 2 ] || fatal 'download needs exactly 2 arguments' + + case $DOWNLOADER in + curl) + if [ "${OS_BUILD}" ] || [ "${INSECURE}" ]; then + info "Running installation without verifying ssl certs on URLs" + $SUDO curl --insecure -o $1 -sSfL $2 + else + $SUDO curl -o $1 -sSfL $2 + fi + ;; + wget) + $SUDO wget -qO $1 $2 + ;; + *) + fatal "Incorrect executable '$DOWNLOADER'" + ;; + esac + + # Abort if download command failed + [ $? -eq 0 ] || fatal 'Download failed' +} + +# --- verify existence of network downloader executable --- +verify_downloader() { + # Return failure if it doesn't exist or is no executable + [ -x "$(command -v $1)" ] || return 1 + + # Set verified executable as our downloader program and return success + DOWNLOADER=$1 + return 0 +} + +verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' + + +# This is the drop in device k3s agent(worker) bootstrap script for the Teknoir platform. +# It should work for most arm32/arm64/amd64 devices running Debian or Ubuntu. + +# Environment variables: +# K3S_URL=https://:6443 (default: generated at master node creation time) +# K3S_TOKEN="32 char string" (default: generated at master node creation time) +# CREATE_USER=false (default: ask, create a user for the device without asking) +# INSTALL_K3S_VERSION=v1.xx.y+k3sN (default: the release of the SERVER, read from https:///static/teknoir/k3s-version: an +# agent never installs a k3s newer than its server. A release you ask for (here, or +# tnctl bootstrap --k3s-version) is installed if it is not newer than the server's, and +# refused if it is. If the server cannot be reached the script stops, unless the +# release was asked for or TN_K3S_FORCE=true) +# TN_K3S_UPGRADE=true (default: false; a re-run KEEPS the installed k3s version. true lets the pin replace +# it: never a downgrade, never more than one minor version. Upgrade the server first) +# TN_K3S_FORCE=true (default: false; allows an upgrade over more than one minor version, an agent that is +# newer than its server, and an install when the release of the server is unknown) +# TN_K3S_CGROUPV1=true|false (default: detected; on cgroup v1 with k3s >= v1.31 the kubelet gets +# fail-cgroupv1=false, see /etc/rancher/k3s/config.yaml.d/99-teknoir-cgroupv1.yaml) + +_DOMAIN="example.test" +_DEVICE_ID="golden-device" +_NAMESPACE="golden-ns" +_FIRST_USER_NAME="golden-user" +_FIRST_USER_PASS="TEST-ONLY-password" +_FIRST_USER_KEY="ssh-ed25519 TEST-ONLY-ssh-public golden@example.test" +_RSA_PUBLIC="TEST-ONLY-rsa-public +TEST-ONLY-rsa-public-line2" +_RSA_PRIVATE="TEST-ONLY-rsa-private +TEST-ONLY-rsa-private-line2" +_K3S_TOKEN="TEST-ONLY-k3s-token" + + +# The k3s release policy (a re-run keeps the installed k3s) and the cgroup v1 kubelet flag. +#!/bin/sh +# tn-k3s-install.sh - what a Teknoir bootstrap script decides before it runs the k3s installer: +# which k3s release (a re-run keeps what is installed), and whether the kubelet must be told that +# cgroup v1 is acceptable. Used by the server and the agent script (sourced), next to +# tn-k3s-ca.sh (the server only). +# +# . tn-k3s-install.sh; tn_inst_prepare server|agent (once, right before the installer) +# tn_inst_finish (once, right after the installer) +# tn_inst_config_ensure FILE KEY TEXT (config.yaml without clobbering it) +# +# POSIX sh. It runs under `set -e` and `set -o noglob` (as the bootstrap scripts do) and does not +# change shell options, traps or the working directory of the caller. Sourcing defines functions only. +# +# ---------------------------------------------------------------- version policy +# A bootstrap script is meant to be re-run (it repairs the CA, refreshes the manifests). It must not +# upgrade k3s as a side effect: the pin in the script (INSTALL_K3S_VERSION, `tnctl bootstrap +# --k3s-version`) is the release for a NEW device. +# - k3s is not installed: the pin is installed (a SERVER; an agent: see below). +# - the version cannot be read although a k3s binary or DATA_DIR/server/db exists: REFUSED (the +# installer would install or replace k3s blind), unless TN_K3S_FORCE=true. +# - k3s is installed: its version is KEPT. The exact installed release is handed to the +# installer (which then only rewrites the unit and the symlinks), and +# a different pin is reported and ignored. +# - TN_K3S_UPGRADE=true (tnctl bootstrap --k3s-upgrade) lets the pin replace the installed +# release. A pin below the installed release is refused (k3s does not +# downgrade: the datastore has been migrated), and so is a jump of more +# than one minor version (v1.34 -> v1.36) unless TN_K3S_FORCE=true +# (tnctl bootstrap --force): Kubernetes supports one minor at a time. +# A channel is checked too: a minor (v1.35) by its major.minor against the +# installed one (lower or other major: refused; more than +1: needs force); +# stable/latest/testing is resolved at update.k3s.io like the installer does +# and then checked as an exact release. A channel that cannot be resolved +# is refused unless TN_K3S_FORCE=true (then it goes on unchecked). +# +# An AGENT follows its server. A kubelet must never be newer than the apiserver it joins, so a new +# agent (and an agent that TN_K3S_UPGRADE=true replaces) does not install the pin of the script: it +# reads the release of the server and installs exactly that one. Where the release comes from: k3s +# does NOT answer /version to an anonymous caller (its apiserver runs with anonymous-auth=false: 401, +# verified with v1.30, v1.34 and v1.36), and the join token is only good for the supervisor +# endpoints. So the Teknoir SERVER script publishes the release it runs (tn_inst_publish_release) as +# /server/static/teknoir/k3s-version, which k3s serves without authentication at +# https:///static/teknoir/k3s-version +# and the agent reads that file (then, for a server that allows anonymous access, /version). The +# pin only counts when the person who runs the script asked for it (`tnctl bootstrap --k3s-version`, or +# INSTALL_K3S_VERSION in the environment; TN_K3S_PIN_SOURCE=flag|env, the template sets it): then it is +# installed if it is not newer than the server (older gets a warning), and refused if it is. When the +# server's release cannot be read (down, unreachable, bootstrapped by a script that does not publish it +# yet): an explicit exact pin is installed with a warning, and otherwise the script REFUSES, unless +# TN_K3S_FORCE=true. An OS image build (OS_BUILD) cannot reach the server and installs the pin. The +# cgroup logic below is keyed on the release that is installed. +# +# ---------------------------------------------------------------- cgroup v1 +# Kubernetes >= 1.35: the kubelet refuses to start on a cgroup v1 host (failCgroupV1 defaults to +# true) and k3s does not change that. Jetson L4T r32/r35 and Ubuntu 18.04/20.04 are cgroup v1. When +# the host is on cgroup v1 (stat -fc %T /sys/fs/cgroup is not cgroup2fs), or cannot be told (an OS +# build runs elsewhere than the device), and the k3s that will run is >= v1.31 (the kubelet knows +# the flag since 1.31; it only matters from 1.35, but a later upgrade, by hand or by the k3s +# installer alone, must not stop the kubelet), the script writes (atomically, only when it differs) +# /etc/rancher/k3s/config.yaml.d/99-teknoir-cgroupv1.yaml: kubelet-arg+: ["fail-cgroupv1=false"] +# `kubelet-arg+` APPENDS to the kubelet-arg of config.yaml and the other files (verified with +# rancher/k3s v1.36.5, a kubelet-arg already in config.yaml is kept); the flag is harmless on a +# cgroup v2 host. A fail-cgroupv1 that the device sets itself is respected. cgroup v1 is +# deprecated upstream: move the device to cgroup v2 (kernel command line +# systemd.unified_cgroup_hierarchy=1, or a newer OS release). +# +# Order of effects. Everything before the installer (tn_inst_prepare, the CA step, the download) can +# be interrupted by a restart of k3s (the CA step restarts it) or by a reboot of the device, and the +# running k3s then reads the configuration again. So tn_inst_prepare only makes changes that are safe +# for the k3s that is running NOW, and never takes a protection away before its replacement is in place: +# - the drop-in is REMOVED early only when the running k3s does not need it: the host is on cgroup v2; +# the release that will run is older than v1.31 (an older kubelet fails on the unknown flag); or the +# device sets fail-cgroupv1 in a PERSISTENT config file that k3s already reads (config.yaml or another +# config.yaml.d/*.yaml). A fail-cgroupv1 only in INSTALL_K3S_EXEC reaches the unit when the installer +# has run, so until then the drop-in stays (or is written): the same value twice is harmless, and +# the command line wins when they differ. +# - the drop-in is WRITTEN early, except when the k3s that is installed now is older than v1.31 (an +# upgrade): its kubelet would fail on the flag if it restarted before the installer replaced it. +# That write waits for tn_inst_finish, after the installer. An existing drop-in is removed at once in +# that case, because the old kubelet cannot take it. +# - but the installer starts the NEW release as soon as it has rewritten the unit, and a release >= v1.35 +# refuses to start on cgroup v1 without the flag (install.sh's `systemctl restart` fails, and under +# `set -e` the script stops before tn_inst_finish: a k3s that crash-loops). So when the write is held +# back and the release the installer will start is not KNOWN to be older than v1.35 (a channel such as +# stable or latest, or an exact v1.35 or newer, also with TN_K3S_FORCE=true), the flag is appended to +# INSTALL_K3S_EXEC (--kubelet-arg=fail-cgroupv1=false, once, and exported): it is then in the NEW unit +# from its first start and the old, running kubelet never sees it. An exact v1.31 - v1.34 does not need +# it on its first start (it accepts it, it only ignores it on cgroup v1), so it gets none. tn_inst_finish +# still writes the drop-in afterwards, which is what keeps later runs and upgrades safe. +# +# What counts as "the device sets fail-cgroupv1 itself": an uncommented line that sets it to false in +# config.yaml, config.yaml.d/*.yaml or *.yml (not our own file), or in INSTALL_K3S_EXEC. A comment does +# not count, and neither does fail-cgroupv1=true: then the drop-in is written (and kept) as usual, and +# it wins, because kubelet-arg+ comes after. A device that sets fail-cgroupv1=true on a cgroup v1 host +# on purpose is not supported (the kubelet of v1.35 and newer cannot start there anyway); delete the +# drop-in and set cgroup v2 instead. +# +# Environment (all optional): +# INSTALL_K3S_VERSION, INSTALL_K3S_CHANNEL the pin of the script (read, and set to what the installer must get) +# TN_K3S_UPGRADE true: the pin may replace an installed k3s +# TN_K3S_FORCE true: allow an upgrade across more than one minor version; an agent newer than its +# server, or without knowing the version of its server; a k3s whose version cannot be read +# TN_K3S_PIN_SOURCE who chose INSTALL_K3S_VERSION: flag / env (the person who runs the script) or +# default (the template); the template sets it, an agent follows the server unless it is flag/env +# K3S_URL the server of an agent (https://host:6443) +# TN_K3S_SERVER_VERSION_TRIES, TN_K3S_SERVER_VERSION_WAIT tries (3) and seconds between them (3) to reach the +# server; a server that answers without a release is not asked again +# DATA_DIR the k3s data dir (/var/lib/rancher/k3s), where a datastore says that k3s is there +# TN_K3S_CGROUPV1 true / false: do not detect, take this as the answer +# TN_CGROUP_FSTYPE the file system type of /sys/fs/cgroup, instead of `stat` (tests) +# K3S_BIN the k3s command used to read the installed version (k3s, then /usr/local/bin/k3s) +# TN_K3S_CONFIG_DIR /etc/rancher/k3s +# OS_BUILD set by the OS image builds: the host is not the device, cgroup cannot be detected + +TN_K3S_INSTALL_LIB_VERSION=3 +export TN_K3S_INSTALL_LIB_VERSION + +# ----------------------------------------------------------------- messages +# Own functions, in the format of header.sh.tmpl: a caller's info/warn/fatal may not exist, and `info` +# can be the GNU info reader. +tn_inst_info() { echo '[INFO] ' "$@"; } +tn_inst_warn() { echo '[WARN] ' "$@" >&2; } +tn_inst_fatal() { echo '[ERROR] ' "$@" >&2; exit 1; } + +# tn_inst_state_note: how a refusal describes the device. This library has changed nothing when it +# refuses; the CA step (tn-k3s-ca.sh) may have, earlier in the same run, and then says exactly what. +tn_inst_state_note() { + if [ -n "${TN_K3S_CA_CHANGED:-}" ]; then + printf 'The k3s release was not changed, but earlier in this run: %s' "$TN_K3S_CA_CHANGED" + else + printf 'Nothing was changed' + fi +} + +# ----------------------------------------------------------------- versions + +# tn_inst_parse VERSION: sets TN_INST_MAJ TN_INST_MIN TN_INST_PAT TN_INST_REV from vMAJ.MIN.PAT[+k3sREV] +# (REV 0 when there is none). Fails for anything else (stable, latest, a commit). +tn_inst_parse() { + TN_INST_MAJ=; TN_INST_MIN=; TN_INST_PAT=; TN_INST_REV= + _tn_pv=$(printf '%s\n' "$1" | sed -n 's/^v\([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\)\(+k3s\([0-9][0-9]*\)\)\{0,1\}$/\1 \2 \3 \5/p') + [ -n "$_tn_pv" ] || return 1 + IFS=' ' read -r TN_INST_MAJ TN_INST_MIN TN_INST_PAT TN_INST_REV < $i + 0) { print 1; exit } } print 0 }') +} + +# the k3s command that reads the installed version +tn_inst_k3s_cmd() { + if [ -n "${K3S_BIN:-}" ]; then echo "$K3S_BIN" + elif command -v k3s >/dev/null 2>&1; then echo k3s + elif [ -x /usr/local/bin/k3s ]; then echo /usr/local/bin/k3s + else echo k3s; fi +} + +# tn_inst_installed_version: sets TN_INST_INSTALLED to the release of the installed k3s +# (v1.34.11+k3s1) or to nothing when there is none. +tn_inst_installed_version() { + TN_INST_INSTALLED=$($(tn_inst_k3s_cmd) --version 2>/dev/null | sed -n '1s/^.*version \(v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\(+k3s[0-9][0-9]*\)\{0,1\}\).*$/\1/p') || TN_INST_INSTALLED= +} + +# tn_inst_k3s_present: 0 if there is a k3s binary or a server datastore, whatever its version is. +# Sets TN_INST_PRESENT to what was found. +tn_inst_k3s_present() { + TN_INST_PRESENT= + _tn_first= + for _tn_w in $(tn_inst_k3s_cmd); do _tn_first=$_tn_w; break; done + if [ -n "$_tn_first" ] && _tn_found=$(command -v "$_tn_first" 2>/dev/null) && [ -n "$_tn_found" ]; then + TN_INST_PRESENT="the k3s command $_tn_found"; return 0 + fi + if [ -d "${DATA_DIR:-/var/lib/rancher/k3s}/server/db" ]; then + TN_INST_PRESENT="the datastore ${DATA_DIR:-/var/lib/rancher/k3s}/server/db"; return 0 + fi + return 1 +} + +# tn_inst_http_get URL: the body on stdout. Returns 0 for 2xx, 1 when the server ANSWERED with an error +# (404: no such file, 401: no anonymous access), 2 when it could not be reached (or there is neither curl +# nor wget). No certificate check: the answer decides which release to install and nothing else (the +# server is the one this device joins with its token anyway). +tn_inst_http_get() { + _tn_hc=0 + if command -v curl >/dev/null 2>&1; then + curl -fsk --max-time 10 "$1" || _tn_hc=$? + case $_tn_hc in 0) return 0 ;; 22) return 1 ;; *) return 2 ;; esac + elif command -v wget >/dev/null 2>&1; then + wget -qO- --no-check-certificate -T 10 "$1" || _tn_hc=$? + case $_tn_hc in 0) return 0 ;; 8) return 1 ;; *) return 2 ;; esac + fi + return 2 +} + +# tn_inst_server_version URL: sets TN_INST_SERVER to the k3s release the server at URL runs +# (v1.34.11+k3s1) and TN_INST_SERVER_SRC to where it was read, or leaves them empty (and sets +# TN_INST_SERVER_WHY to unreachable or no-release). Fails when it cannot tell. Two places, in this order: +# /static/teknoir/k3s-version the file that the Teknoir server script publishes (anonymous) +# /version gitVersion, for a server that allows anonymous access (k3s does not by default) +tn_inst_server_version() { + TN_INST_SERVER=; TN_INST_SERVER_SRC=; TN_INST_SERVER_WHY=unreachable + _tn_su=${1%/} + _tn_sv_max=${TN_K3S_SERVER_VERSION_TRIES:-3} + _tn_sv_try=0 + while [ "$_tn_sv_try" -lt "$_tn_sv_max" ]; do + _tn_sv_try=$((_tn_sv_try + 1)) + _tn_unreach=0 + _tn_rc=0 + _tn_body=$(tn_inst_http_get "$_tn_su/static/teknoir/k3s-version" 2>/dev/null) || _tn_rc=$? + if [ "$_tn_rc" = 0 ]; then + TN_INST_SERVER=$(printf '%s' "$_tn_body" | tr -d ' \t\r\n' | sed -n 's/^\(v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*+k3s[0-9][0-9]*\)$/\1/p') + if [ -n "$TN_INST_SERVER" ]; then TN_INST_SERVER_SRC=$_tn_su/static/teknoir/k3s-version; return 0; fi + elif [ "$_tn_rc" = 2 ]; then + _tn_unreach=1 + fi + if [ "$_tn_unreach" = 0 ]; then + _tn_rc=0 + _tn_body=$(tn_inst_http_get "$_tn_su/version" 2>/dev/null) || _tn_rc=$? + if [ "$_tn_rc" = 0 ]; then + # {"major":"1","minor":"34","gitVersion":"v1.34.11+k3s1",...}, over several lines or one + TN_INST_SERVER=$(printf '%s\n' "$_tn_body" | tr -d '\n' | sed -n 's/^.*"gitVersion"[[:space:]]*:[[:space:]]*"\(v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*+k3s[0-9][0-9]*\)".*$/\1/p') + if [ -n "$TN_INST_SERVER" ]; then TN_INST_SERVER_SRC=$_tn_su/version; return 0; fi + elif [ "$_tn_rc" = 2 ]; then + _tn_unreach=1 + fi + fi + # the server answered, but has no release to give: asking again does not help + if [ "$_tn_unreach" = 0 ]; then TN_INST_SERVER_WHY=no-release; return 1; fi + if [ "$_tn_sv_try" -lt "$_tn_sv_max" ]; then sleep "${TN_K3S_SERVER_VERSION_WAIT:-3}"; fi + done + return 1 +} + +# tn_inst_publish_release: a SERVER script calls it after the k3s installer. Publishes the installed +# release as /server/static/teknoir/k3s-version; k3s serves server/static without +# authentication (https://:6443/static/teknoir/k3s-version), and an agent script reads it to +# install the release of its server. Written atomically; unchanged when it is right; it never fails the +# script. An out-of-band upgrade of k3s leaves it saying the older release, which is the safe direction +# (an agent may be older than its server, never newer): a re-run of the server script updates it. +tn_inst_publish_release() { + tn_inst_installed_version + if [ -z "$TN_INST_INSTALLED" ]; then + tn_inst_warn "cannot read the release of the installed k3s: it is not published, so agents cannot follow this server (they need INSTALL_K3S_VERSION)" + return 0 + fi + _tn_pd=${DATA_DIR:-/var/lib/rancher/k3s}/server/static/teknoir + _tn_pf=$_tn_pd/k3s-version + if [ -f "$_tn_pf" ] && [ "$(cat "$_tn_pf" 2>/dev/null)" = "$TN_INST_INSTALLED" ]; then + tn_inst_info "the k3s release $TN_INST_INSTALLED is already published for the agents ($_tn_pf)" + return 0 + fi + _tn_pt=$_tn_pd/.k3s-version.tmp.$$ + if mkdir -p "$_tn_pd" && printf '%s\n' "$TN_INST_INSTALLED" > "$_tn_pt" && mv -f "$_tn_pt" "$_tn_pf"; then + tn_inst_info "Published the k3s release $TN_INST_INSTALLED for the agents: $_tn_pf (served at /static/teknoir/k3s-version)" + else + rm -f "$_tn_pt" 2>/dev/null || true + tn_inst_warn "cannot publish the k3s release to $_tn_pf: agents cannot follow this server (they need INSTALL_K3S_VERSION)" + fi + return 0 +} + +# ----------------------------------------------------------------- the release to install + +# tn_inst_agent_target: for an agent that is to be installed (or replaced): which release. Sets and +# exports INSTALL_K3S_VERSION (and unsets INSTALL_K3S_CHANNEL) or exits through tn_inst_fatal. +tn_inst_agent_target() { + _tn_pin=${INSTALL_K3S_VERSION:-} + _tn_exp=0 + case ${TN_K3S_PIN_SOURCE:-default} in flag|env) _tn_exp=1 ;; esac + if [ -n "${OS_BUILD:-}" ]; then + tn_inst_warn "OS build: the k3s server is not reachable from here, so the release of the agent cannot follow it. ${_tn_pin:-The release the installer picks} is installed: the server must run this release or a newer one" + return 0 + fi + TN_INST_SERVER= + if [ -z "${K3S_URL:-}" ]; then + tn_inst_warn "K3S_URL is not set: the k3s release of the server cannot be read" + else + tn_inst_info "Reading the k3s release of the server at ${K3S_URL%/}" + tn_inst_server_version "$K3S_URL" || true + fi + if [ -n "$TN_INST_SERVER" ]; then + tn_inst_info "the k3s server at $K3S_URL runs $TN_INST_SERVER (read from $TN_INST_SERVER_SRC)" + if [ "$_tn_exp" = 1 ] && [ -n "$_tn_pin" ] && tn_inst_cmp "$_tn_pin" "$TN_INST_SERVER"; then + case $TN_INST_CMP in + 0) tn_inst_info "the requested release $_tn_pin is the release of the server" ;; + -1) tn_inst_warn "the requested release $_tn_pin is OLDER than the release of the server ($TN_INST_SERVER): it is installed as asked. An agent should run the release of its server (Kubernetes supports a kubelet up to three minor versions older than the apiserver)" ;; + *) if [ "${TN_K3S_FORCE:-}" = true ]; then + tn_inst_warn "the requested release $_tn_pin is NEWER than the release of the server ($TN_INST_SERVER); TN_K3S_FORCE=true: it is installed anyway. A kubelet newer than its apiserver is not supported" + else + tn_inst_fatal "refusing to install k3s $_tn_pin on this agent: the server at $K3S_URL runs the older $TN_INST_SERVER, and a kubelet must not be newer than the apiserver it joins. Without --k3s-version (INSTALL_K3S_VERSION) the agent gets the release of its server, $TN_INST_SERVER. Or upgrade the server first, or set TN_K3S_FORCE=true. $(tn_inst_state_note)" + fi ;; + esac + return 0 + fi + if [ "$_tn_exp" = 1 ]; then + tn_inst_warn "the requested k3s release (${_tn_pin:-${INSTALL_K3S_CHANNEL:-stable}}) is not an exact release, so it cannot be compared with the server: the release of the server is installed instead" + elif [ -n "$_tn_pin" ] && [ "$_tn_pin" != "$TN_INST_SERVER" ]; then + tn_inst_info "the script's default $_tn_pin is not used: an agent runs the release of its server" + fi + INSTALL_K3S_VERSION=$TN_INST_SERVER + export INSTALL_K3S_VERSION + unset INSTALL_K3S_CHANNEL + tn_inst_info "the agent gets the release of its server: $TN_INST_SERVER" + return 0 + fi + # the server cannot be asked + if [ "$_tn_exp" = 1 ] && [ -n "$_tn_pin" ] && tn_inst_parse "$_tn_pin"; then + tn_inst_warn "cannot read the k3s release of the server at ${K3S_URL:-(K3S_URL is not set)}: the requested $_tn_pin is installed as asked. The server must run this release or a newer one" + return 0 + fi + if [ "${TN_K3S_FORCE:-}" = true ]; then + tn_inst_warn "cannot read the k3s release of the server at ${K3S_URL:-(K3S_URL is not set)}; TN_K3S_FORCE=true: ${_tn_pin:-the release the installer picks} is installed anyway. If the server runs an older release, this agent will not work" + return 0 + fi + case ${TN_INST_SERVER_WHY:-unreachable} in + no-release) _tn_why="The server answered, but does not publish its release: it was bootstrapped by a script that does not yet (run the server script again: it keeps k3s as it is and publishes its release), and k3s does not allow an anonymous /version." ;; + *) _tn_why="Is the server up and reachable from this device? Start it and run this script again." ;; + esac + tn_inst_fatal "cannot read the k3s release of the server at ${K3S_URL:-(K3S_URL is not set)} (${K3S_URL:-https://:6443}/static/teknoir/k3s-version): an agent must not run a newer k3s than its server, so this script does not guess. $_tn_why Or say which release the server runs: INSTALL_K3S_VERSION=v1.xx.y+k3sN (tnctl bootstrap --k3s-version), or set TN_K3S_FORCE=true to install ${_tn_pin:-the release the installer picks} anyway. $(tn_inst_state_note)" +} + +# tn_inst_normalize_pin: the k3s installer takes INSTALL_K3S_VERSION as an EXACT release (it downloads that +# tag, so "stable" would be a 404) and INSTALL_K3S_CHANNEL as a channel. A channel name given as the version +# (stable, latest, testing, or a minor such as v1.34) is moved to INSTALL_K3S_CHANNEL and the version is +# unset, so that every later check sees what the installer will see. An exact release is not touched. +tn_inst_normalize_pin() { + [ -n "${INSTALL_K3S_VERSION:-}" ] || return 0 + case $INSTALL_K3S_VERSION in + stable|latest|testing) ;; + v[0-9]*) printf '%s\n' "$INSTALL_K3S_VERSION" | grep -Eq '^v[0-9]+\.[0-9]+$' || return 0 ;; + *) return 0 ;; + esac + if [ -n "${INSTALL_K3S_CHANNEL:-}" ] && [ "$INSTALL_K3S_CHANNEL" != "$INSTALL_K3S_VERSION" ]; then + tn_inst_fatal "conflicting k3s requests: INSTALL_K3S_VERSION=$INSTALL_K3S_VERSION is a channel and INSTALL_K3S_CHANNEL=$INSTALL_K3S_CHANNEL is another. Set only one of them. $(tn_inst_state_note)" + fi + INSTALL_K3S_CHANNEL=$INSTALL_K3S_VERSION + export INSTALL_K3S_CHANNEL + unset INSTALL_K3S_VERSION + tn_inst_info "$INSTALL_K3S_CHANNEL is a k3s channel, not a release: the installer gets INSTALL_K3S_CHANNEL=$INSTALL_K3S_CHANNEL" +} + +# tn_inst_plan: applies the policy at the top. Reads INSTALL_K3S_VERSION / INSTALL_K3S_CHANNEL, +# sets and exports them for the installer. Sets TN_INST_INSTALLED, TN_INST_ACTION +# (install|same|keep|upgrade) and TN_INST_EFFECTIVE (the exact release that will run, or nothing +# when only a channel is known). Exits through tn_inst_fatal when it refuses. +tn_inst_plan() { + tn_inst_normalize_pin + tn_inst_installed_version + if [ -z "$TN_INST_INSTALLED" ] && tn_inst_k3s_present; then + if [ "${TN_K3S_FORCE:-}" = true ]; then + tn_inst_warn "$TN_INST_PRESENT exists, but its k3s version cannot be read; TN_K3S_FORCE=true: going on as if k3s were not installed. The installer may replace it with ${INSTALL_K3S_VERSION:-${INSTALL_K3S_CHANNEL:-the release the installer picks}}, without the upgrade and downgrade checks" + else + tn_inst_fatal "$TN_INST_PRESENT exists, but its k3s version cannot be read ('$(tn_inst_k3s_cmd) --version' gave no release), so this script cannot tell whether the installer would upgrade it, downgrade it or break its datastore, and does not install anything. Look at '$(tn_inst_k3s_cmd) --version' and at the datastore; repair or remove k3s on purpose, or set TN_K3S_FORCE=true to install anyway. $(tn_inst_state_note)" + fi + fi + if [ "${TN_INST_ROLE:-server}" = agent ] && { [ -z "$TN_INST_INSTALLED" ] || [ "${TN_K3S_UPGRADE:-}" = true ]; }; then + tn_inst_agent_target + fi + _tn_t=${INSTALL_K3S_VERSION:-} + _tn_want=${_tn_t:-${INSTALL_K3S_CHANNEL:-the default channel of the k3s installer (stable)}} + TN_INST_ACTION=install + TN_INST_EFFECTIVE= + if [ -z "$TN_INST_INSTALLED" ]; then + TN_INST_EFFECTIVE=$_tn_t + if [ -n "$_tn_t" ]; then tn_inst_info "k3s is not installed: installing $_tn_t"; else tn_inst_info "k3s is not installed: installing $_tn_want"; fi + return 0 + fi + tn_inst_info "k3s $TN_INST_INSTALLED is installed on this device" + + if [ "${TN_K3S_UPGRADE:-}" != true ]; then + # keep: an exact release is not "newer", it is "this one" + TN_INST_EFFECTIVE=$TN_INST_INSTALLED + if [ "$_tn_t" = "$TN_INST_INSTALLED" ]; then + TN_INST_ACTION=same + tn_inst_info "k3s $TN_INST_INSTALLED is the requested release: it is installed again, nothing is upgraded" + return 0 + fi + TN_INST_ACTION=keep + INSTALL_K3S_VERSION=$TN_INST_INSTALLED + export INSTALL_K3S_VERSION + unset INSTALL_K3S_CHANNEL + if tn_inst_cmp "$_tn_t" "$TN_INST_INSTALLED" && [ "$TN_INST_CMP" = -1 ]; then + tn_inst_warn "the requested k3s release $_tn_t is OLDER than the installed $TN_INST_INSTALLED: k3s cannot be downgraded, $TN_INST_INSTALLED is kept" + else + tn_inst_warn "k3s $TN_INST_INSTALLED is kept; the requested $_tn_want is NOT installed. A re-run of this script repairs the device, it does not upgrade k3s. To upgrade, run it with TN_K3S_UPGRADE=true (tnctl bootstrap --k3s-upgrade), one minor version at a time" + fi + return 0 + fi + + # TN_K3S_UPGRADE=true + if [ -z "$_tn_t" ]; then + # a channel: the installer picks the release, so find out which, or compare the minor + _tn_ch=${INSTALL_K3S_CHANNEL:-stable} + if printf '%s\n' "$_tn_ch" | grep -Eq '^v[0-9]+\.[0-9]+$'; then + tn_inst_parse "${_tn_ch}.0"; _tn_cmaj=$TN_INST_MAJ; _tn_cmin=$TN_INST_MIN + tn_inst_parse "$TN_INST_INSTALLED"; _tn_imaj=$TN_INST_MAJ; _tn_imin=$TN_INST_MIN + if [ "$_tn_cmaj" != "$_tn_imaj" ] || [ "$_tn_cmin" -lt "$_tn_imin" ]; then + tn_inst_fatal "refusing to downgrade k3s from $TN_INST_INSTALLED to the channel $_tn_ch (v$_tn_cmaj.$_tn_cmin is not v$_tn_imaj.$_tn_imin or a later minor of the same major): k3s cannot go back (the datastore has been migrated by the newer release). $(tn_inst_state_note). Use a channel of v$_tn_imaj.$_tn_imin or $_tn_imaj.$((_tn_imin + 1)), or restore the device from a backup" + fi + if [ $((_tn_cmin - _tn_imin)) -gt 1 ]; then + if [ "${TN_K3S_FORCE:-}" = true ]; then + tn_inst_warn "upgrading k3s from $TN_INST_INSTALLED to the channel $_tn_ch skips minor versions (TN_K3S_FORCE=true). Kubernetes supports one minor version at a time: this can leave the cluster in a state that does not start. Have a backup of the datastore" + else + tn_inst_fatal "refusing to upgrade k3s from $TN_INST_INSTALLED to the channel $_tn_ch: that is more than one minor version (v$_tn_imaj.$_tn_imin -> v$_tn_cmaj.$_tn_cmin). Upgrade one minor at a time (channel v$_tn_imaj.$((_tn_imin + 1)) first), or set TN_K3S_FORCE=true (tnctl bootstrap --force) if you know what you are doing. $(tn_inst_state_note)" + fi + fi + TN_INST_ACTION=upgrade + tn_inst_info "upgrading k3s from $TN_INST_INSTALLED to the latest release of the channel $_tn_ch (TN_K3S_UPGRADE=true). A server must be upgraded before its agents" + return 0 + fi + # stable, latest, testing: resolve like the installer does (the last path segment of the redirect) + TN_INST_RESOLVED=$(curl -w '%{url_effective}' -L -s -S -o /dev/null "https://update.k3s.io/v1-release/channels/$_tn_ch" 2>/dev/null | sed 's|^.*/||; s|%2[Bb]|+|') || TN_INST_RESOLVED= + if tn_inst_parse "$TN_INST_RESOLVED" 2>/dev/null; then + _tn_t=$TN_INST_RESOLVED + tn_inst_info "the k3s channel $_tn_ch resolves to $_tn_t: it is pinned, so that the installer gets what was checked" + INSTALL_K3S_VERSION=$_tn_t + export INSTALL_K3S_VERSION + unset INSTALL_K3S_CHANNEL + elif [ "${TN_K3S_FORCE:-}" = true ]; then + TN_INST_ACTION=upgrade + tn_inst_warn "TN_K3S_UPGRADE=true with the channel $_tn_ch, which cannot be resolved from here (update.k3s.io): the k3s installer picks the release, so a downgrade or a jump over several minor versions cannot be checked. TN_K3S_FORCE=true: going on without those checks. Pin an exact release (INSTALL_K3S_VERSION=v1.xx.y+k3sN) to get them" + return 0 + else + tn_inst_fatal "refusing to upgrade k3s from $TN_INST_INSTALLED to the channel $_tn_ch: the channel cannot be resolved to a release from here (update.k3s.io is not reachable), so a downgrade or a jump over several minor versions cannot be checked. Pin an exact release (INSTALL_K3S_VERSION=v1.xx.y+k3sN, tnctl bootstrap --k3s-version), or set TN_K3S_FORCE=true to go on without the checks. $(tn_inst_state_note)" + fi + fi + if ! tn_inst_cmp "$_tn_t" "$TN_INST_INSTALLED"; then + tn_inst_fatal "cannot compare the requested k3s release '$_tn_t' with the installed $TN_INST_INSTALLED. Pin an exact release (INSTALL_K3S_VERSION=v1.xx.y+k3sN). $(tn_inst_state_note)" + fi + TN_INST_EFFECTIVE=$_tn_t + case $TN_INST_CMP in + 0) TN_INST_ACTION=same; tn_inst_info "k3s $TN_INST_INSTALLED is the requested release: nothing to upgrade"; return 0 ;; + -1) tn_inst_fatal "refusing to downgrade k3s from $TN_INST_INSTALLED to $_tn_t: k3s cannot go back (the datastore has been migrated by the newer release). $(tn_inst_state_note). Pin $TN_INST_INSTALLED or a newer release, or restore the device from a backup" ;; + esac + tn_inst_parse "$TN_INST_INSTALLED"; _tn_imaj=$TN_INST_MAJ; _tn_imin=$TN_INST_MIN + tn_inst_parse "$_tn_t" + if [ "$TN_INST_MAJ" != "$_tn_imaj" ] || [ $((TN_INST_MIN - _tn_imin)) -gt 1 ]; then + if [ "${TN_K3S_FORCE:-}" = true ]; then + tn_inst_warn "upgrading k3s from $TN_INST_INSTALLED to $_tn_t skips minor versions (TN_K3S_FORCE=true). Kubernetes supports one minor version at a time: this can leave the cluster in a state that does not start. Have a backup of the datastore" + else + tn_inst_fatal "refusing to upgrade k3s from $TN_INST_INSTALLED to $_tn_t: that is more than one minor version (v$_tn_imaj.$_tn_imin -> v$TN_INST_MAJ.$TN_INST_MIN). Upgrade one minor at a time (to v$_tn_imaj.$((_tn_imin + 1)).x first), or set TN_K3S_FORCE=true (tnctl bootstrap --force) if you know what you are doing. $(tn_inst_state_note)" + fi + fi + TN_INST_ACTION=upgrade + tn_inst_info "upgrading k3s from $TN_INST_INSTALLED to $_tn_t (TN_K3S_UPGRADE=true). A server must be upgraded before its agents" +} + +# ----------------------------------------------------------------- cgroup v1 + +TN_INST_CGROUP_FILE=99-teknoir-cgroupv1.yaml + +# tn_inst_cgroup_state: prints v1 (cgroup v1 or hybrid), v2 or unknown +tn_inst_cgroup_state() { + case ${TN_K3S_CGROUPV1:-} in true) echo v1; return 0 ;; false) echo v2; return 0 ;; esac + # an OS image is built on another machine than the one it will run on + if [ -n "${OS_BUILD:-}" ]; then echo unknown; return 0; fi + _tn_cg=${TN_CGROUP_FSTYPE:-} + [ -n "$_tn_cg" ] || _tn_cg=$(stat -fc %T /sys/fs/cgroup 2>/dev/null) || _tn_cg= + case $_tn_cg in + cgroup2fs) echo v2 ;; + '') echo unknown ;; + *) echo v1 ;; # tmpfs: cgroup v1, or the hybrid layout + esac +} + +# tn_inst_cgroup_flag_in_text: reads TEXT on stdin; 0 if an uncommented part of it sets fail-cgroupv1 to +# false (kubelet-arg: ["fail-cgroupv1=false"], - fail-cgroupv1=false, --kubelet-arg=fail-cgroupv1=false). +# A whole-line or trailing YAML comment is dropped first; fail-cgroupv1=true does not match. +tn_inst_cgroup_flag_in_text() { + sed 's/^[[:space:]]*#.*$//; s/[[:space:]]#.*$//' | grep -Eiq 'fail-cgroupv1[[:space:]]*=[[:space:]]*false([^A-Za-z0-9_]|$)' +} + +# tn_inst_cgroup_configured_elsewhere: 0 if the device sets fail-cgroupv1=false itself (not in our file). +# Sets TN_INST_CGROUP_WHERE to where, and TN_INST_CGROUP_PERSISTENT to 1 when that is a config file that +# the running k3s reads (config.yaml, config.yaml.d/*.yaml and *.yml), and to 0 when it is only INSTALL_K3S_EXEC +# (which reaches the unit when the installer has run). Files win over INSTALL_K3S_EXEC. A comment, and +# fail-cgroupv1=true, do not count. +tn_inst_cgroup_configured_elsewhere() { + _tn_cd=${TN_K3S_CONFIG_DIR:-/etc/rancher/k3s} + TN_INST_CGROUP_WHERE=; TN_INST_CGROUP_PERSISTENT=0 + for _tn_cf in "$_tn_cd/config.yaml" $(find "$_tn_cd/config.yaml.d" -maxdepth 1 -type f \( -iname '*.yaml' -o -iname '*.yml' \) 2>/dev/null | LC_ALL=C sort); do + [ -r "$_tn_cf" ] || continue + [ "$_tn_cf" != "$_tn_cd/config.yaml.d/$TN_INST_CGROUP_FILE" ] || continue + if tn_inst_cgroup_flag_in_text < "$_tn_cf"; then TN_INST_CGROUP_WHERE=$_tn_cf; TN_INST_CGROUP_PERSISTENT=1; return 0; fi + done + if printf '%s\n' "${INSTALL_K3S_EXEC:-}" | tn_inst_cgroup_flag_in_text; then TN_INST_CGROUP_WHERE=INSTALL_K3S_EXEC; return 0; fi + return 1 +} + +tn_inst_cgroup_remove() { # FILE WHY + [ -f "$1" ] || return 0 + rm -f "$1" || { tn_inst_warn "cannot remove $1"; return 0; } + tn_inst_info "Removed $1 ($2)" +} + +# tn_inst_cgroup: writes or removes the drop-in, see the top. Uses TN_INST_EFFECTIVE (the release +# that will run; nothing = a channel, taken as new enough: stable and latest are >= v1.35; a channel +# named vMAJ.MIN is read as that minor). The flag goes in from v1.31, where the kubelet learned it (an +# older kubelet would not start with an unknown argument). +tn_inst_cgroup() { + _tn_cdir=${TN_K3S_CONFIG_DIR:-/etc/rancher/k3s}/config.yaml.d + _tn_cfile=$_tn_cdir/$TN_INST_CGROUP_FILE + TN_INST_CGROUP_DEFERRED= + _tn_cst=$(tn_inst_cgroup_state) + if [ "$_tn_cst" = v2 ]; then + tn_inst_cgroup_remove "$_tn_cfile" "this host uses cgroup v2" + return 0 + fi + _tn_eff=${TN_INST_EFFECTIVE:-} + if [ -z "$_tn_eff" ]; then + case ${INSTALL_K3S_CHANNEL:-} in v[0-9]*.[0-9]*) tn_inst_parse "${INSTALL_K3S_CHANNEL}.0" && _tn_eff=${INSTALL_K3S_CHANNEL}.0 ;; esac + fi + _tn_flagok=1 _tn_must=1 + if [ -n "$_tn_eff" ] && tn_inst_cmp "$_tn_eff" v1.31.0 && [ "$TN_INST_CMP" = -1 ]; then _tn_flagok=0; fi + if [ -n "$_tn_eff" ] && tn_inst_cmp "$_tn_eff" v1.35.0 && [ "$TN_INST_CMP" = -1 ]; then _tn_must=0; fi + if [ "$_tn_cst" = v1 ]; then + tn_inst_warn "this host uses cgroup v1 (or the hybrid layout). cgroup v1 is deprecated: Kubernetes >= 1.35 refuses to start the kubelet on it. Move the device to cgroup v2 (kernel command line systemd.unified_cgroup_hierarchy=1, or a newer OS release)" + else + tn_inst_info "cannot tell whether the device uses cgroup v1 (OS build, or no /sys/fs/cgroup): the kubelet flag is set to be safe, it is harmless on cgroup v2" + fi + if [ "$_tn_flagok" = 0 ]; then + tn_inst_info "k3s $_tn_eff is older than v1.31 (its kubelet has no fail-cgroupv1 flag) and starts on cgroup v1 without help" + tn_inst_cgroup_remove "$_tn_cfile" "k3s is older than v1.31" + return 0 + fi + if [ "$_tn_must" = 0 ]; then + tn_inst_info "k3s $_tn_eff starts on cgroup v1 without help; the kubelet flag is set anyway, so that an upgrade to v1.35 or newer (also one made by hand) does not stop the kubelet" + fi + if tn_inst_cgroup_configured_elsewhere; then + if [ "$TN_INST_CGROUP_PERSISTENT" = 1 ]; then + tn_inst_info "fail-cgroupv1=false is set by the device itself ($TN_INST_CGROUP_WHERE): left as it is" + tn_inst_cgroup_remove "$_tn_cfile" "the device sets fail-cgroupv1=false itself, in a file that k3s reads" + return 0 + fi + # only on the command line of the installer: k3s does not read it until the unit is rewritten, so the + # drop-in must keep protecting a k3s that restarts before that. The same value twice does no harm. + tn_inst_info "fail-cgroupv1=false is set by the device itself ($TN_INST_CGROUP_WHERE): left as it is. The drop-in $_tn_cfile stays until the installer has put it in the unit; the same value twice is harmless, and the one on the command line wins" + fi + # an installed k3s older than v1.31 (about to be upgraded) cannot start with the flag + if [ -n "${TN_INST_INSTALLED:-}" ] && tn_inst_cmp "$TN_INST_INSTALLED" v1.31.0 && [ "$TN_INST_CMP" = -1 ]; then + TN_INST_CGROUP_DEFERRED=1 + tn_inst_info "the installed k3s $TN_INST_INSTALLED is older than v1.31: the kubelet flag is set after the installer has upgraded it, not before" + tn_inst_cgroup_remove "$_tn_cfile" "the installed k3s is older than v1.31 and cannot take the flag until it is upgraded" + # The installer starts the new release as soon as it has written the unit. From v1.35 that kubelet + # does not start on cgroup v1 without the flag, and tn_inst_finish would never run (the installer's + # restart fails under set -e). So the unit gets the flag from its first start, through INSTALL_K3S_EXEC: + # the old kubelet, which is running now, never reads it. Not needed for an exact v1.31 - v1.34. + if [ "$_tn_must" = 1 ]; then + if printf '%s\n' "${INSTALL_K3S_EXEC:-}" | tn_inst_cgroup_flag_in_text; then + tn_inst_info "INSTALL_K3S_EXEC already has fail-cgroupv1=false: the new unit carries it from its first start" + else + INSTALL_K3S_EXEC="${INSTALL_K3S_EXEC:+$INSTALL_K3S_EXEC }--kubelet-arg=fail-cgroupv1=false" + export INSTALL_K3S_EXEC + tn_inst_info "k3s ${_tn_eff:-from a channel} may need the flag on its first start: --kubelet-arg=fail-cgroupv1=false is added to INSTALL_K3S_EXEC, so that the new unit has it. The old kubelet does not read it" + fi + fi + return 0 + fi + tn_inst_cgroup_write +} + +# tn_inst_cgroup_write: the drop-in, written atomically and only when it differs +tn_inst_cgroup_write() { + _tn_cdir=${TN_K3S_CONFIG_DIR:-/etc/rancher/k3s}/config.yaml.d + _tn_cfile=$_tn_cdir/$TN_INST_CGROUP_FILE + _tn_want=$(printf '%s\n' \ + '# Written by the Teknoir bootstrap script. Kubernetes >= 1.35 refuses to start the kubelet on cgroup v1' \ + '# unless this is false. cgroup v1 is deprecated: move the device to cgroup v2 and delete this file.' \ + '# "kubelet-arg+" appends to the kubelet-arg of config.yaml; harmless on cgroup v2.' \ + 'kubelet-arg+:' \ + ' - "fail-cgroupv1=false"') + if [ -f "$_tn_cfile" ] && [ "$(cat "$_tn_cfile")" = "$_tn_want" ]; then + tn_inst_info "kubelet-arg fail-cgroupv1=false is already in place ($_tn_cfile)" + return 0 + fi + # atomically: k3s may be restarting in a loop right now and must never read half a file. The temporary + # name does not end in .yaml, so k3s does not read it either. + _tn_tmp=$_tn_cdir/.$TN_INST_CGROUP_FILE.tmp.$$ + if mkdir -p "$_tn_cdir" && find "$_tn_cdir" -maxdepth 1 -name ".$TN_INST_CGROUP_FILE.tmp.*" -exec rm -f {} + && + printf '%s\n' "$_tn_want" > "$_tn_tmp" && mv -f "$_tn_tmp" "$_tn_cfile"; then + : + else + rm -f "$_tn_tmp" 2>/dev/null || true + tn_inst_fatal "cannot write $_tn_cfile: k3s >= v1.35 would not start on this cgroup v1 host" + fi + tn_inst_info "Set the kubelet argument fail-cgroupv1=false in $_tn_cfile (k3s ${TN_INST_EFFECTIVE:-from a channel} on cgroup v1)" +} + +# ----------------------------------------------------------------- config.yaml + +# tn_inst_yaml_has_key FILE KEY: is there a top-level KEY (or KEY+) in FILE +tn_inst_yaml_has_key() { + [ -r "$1" ] || return 1 + awk -v k="$2" '{ sub(/\r$/, "") } $0 ~ ("^" k "\\+?[ \t]*:") { f = 1 } END { exit f ? 0 : 1 }' "$1" +} + +# tn_inst_config_ensure FILE KEY TEXT: appends TEXT (the YAML of the key, may be several lines) to +# FILE unless FILE has the top-level KEY already. A config.yaml that the device owner has edited is +# never rewritten: a re-run of a bootstrap script used to replace it, and lost every key it did not know. +tn_inst_config_ensure() { + if tn_inst_yaml_has_key "$1" "$2"; then + tn_inst_info "$1 already sets '$2': kept" + return 0 + fi + mkdir -p "$(dirname "$1")" || return 1 + if [ -s "$1" ] && [ -n "$(tail -c 1 "$1")" ]; then echo >> "$1" || return 1; fi + printf '%s\n' "$3" >> "$1" || return 1 + tn_inst_info "$1: added '$2'" +} + +# ----------------------------------------------------------------- the entry point + +# tn_inst_prepare ROLE: the release policy, then cgroup v1. Call it after INSTALL_K3S_VERSION / +# INSTALL_K3S_CHANNEL (and INSTALL_K3S_EXEC, if the device sets fail-cgroupv1 there) are set and +# before the k3s installer. A SERVER script calls it BEFORE tn_k3s_ca_setup: the kubelet flag must be in +# place before the CA step restarts k3s, or a k3s that a cgroup v1 host made exit would not be up for +# rotate-ca (it restarts itself every few seconds and reads the flag then). The CA step must in turn +# run before the installer. An agent has no CA step. +tn_inst_prepare() { + TN_INST_ROLE=${1:-server} + tn_inst_info "Checking the k3s release and the cgroup version ($TN_INST_ROLE)" + tn_inst_plan + tn_inst_cgroup +} + +# tn_inst_finish: call it right after the k3s installer succeeded. Writes the kubelet flag that +# tn_inst_prepare held back because the k3s that was installed then could not take it (see "Order of +# effects"). Nothing to do otherwise. +tn_inst_finish() { + if [ -n "${TN_INST_CGROUP_DEFERRED:-}" ]; then + TN_INST_CGROUP_DEFERRED= + tn_inst_info "The installer has run: setting the kubelet flag that was held back" + tn_inst_cgroup_write + fi +} + + +# The node name, role labels and taints of this node (tnctl bootstrap --agent-role ...). k3s applies the +# labels and taints only when the node registers for the first time, so they are written before the installer runs. +# tn-k3s-node.sh: the node name, role labels and taints of a k3s node (tnctl bootstrap --role ...). +# +# Embedded in the server and the agent bootstrap script (only when the script was generated with a node +# role, label, taint or name), and usable stand-alone from a test. Usage: +# +# . tn-k3s-install.sh; . tn-k3s-node.sh +# tn_node_name NAME (the node name; optional) +# tn_node_label key=value (once per label; the role and robot labels come first) +# tn_node_taint key[=value]:Effect (once per taint) +# tn_node_apply server|agent (writes the k3s configuration; once, before the k3s installer) +# +# It needs tn_inst_config_ensure, tn_inst_yaml_has_key and tn_inst_info/warn from tn-k3s-install.sh. +# POSIX sh. It runs under `set -e` and `set -o noglob` and does not change shell options. +# +# What it writes (TN_K3S_CONFIG_DIR, /etc/rancher/k3s by default): +# config.yaml node-name, added only when config.yaml has no node-name already: a +# name the device owner set is kept (a different one is reported) +# config.yaml.d/98-teknoir-node.yaml node-label+ and node-taint+: the labels and taints of the role. +# "+" APPENDS to the node-label and node-taint lists of config.yaml and +# the other drop-ins, so the entries of the device owner are kept. The +# file is ours: it is rewritten (atomically) when it differs and left +# alone when it does not, so a re-run changes nothing. +# +# k3s applies node-label and node-taint ONLY when a node registers for the first time. When the node has +# registered already and the file changes (or is new), the script says so and prints the kubectl commands +# that apply the labels and taints to the running node; run them on the k3s server. +# +# Environment: +# TN_K3S_CONFIG_DIR /etc/rancher/k3s +# DATA_DIR the k3s data dir (/var/lib/rancher/k3s): /agent/client-kubelet.crt exists +# once the kubelet of this node has registered +# TN_NODE_REGISTERED true / false: do not look at the data dir, take this as the answer (tests) + +TN_K3S_NODE_LIB_VERSION=1 +export TN_K3S_NODE_LIB_VERSION + +TN_NODE_NAME= +TN_NODE_LABELS= +TN_NODE_TAINTS= +TN_NODE_DROPIN=98-teknoir-node.yaml + +tn_node_name() { TN_NODE_NAME=$1; } +tn_node_label() { TN_NODE_LABELS="${TN_NODE_LABELS:+$TN_NODE_LABELS +}$1"; } +tn_node_taint() { TN_NODE_TAINTS="${TN_NODE_TAINTS:+$TN_NODE_TAINTS +}$1"; } + +# tn_node_yaml_value FILE KEY: the value of the top-level scalar KEY (quotes and a trailing comment removed) +tn_node_yaml_value() { + [ -r "$1" ] || return 0 + awk -v k="$2" ' + { sub(/\r$/, "") } + $0 ~ ("^" k "[ \t]*:") { + v = $0 + sub("^" k "[ \t]*:[ \t]*", "", v) + sub(/[ \t]+#.*$/, "", v) + gsub(/^["\047]|["\047]$/, "", v) + print v + exit + }' "$1" +} + +# tn_node_registered: has the kubelet of this node a client certificate, i.e. has the node registered +tn_node_registered() { + case ${TN_NODE_REGISTERED:-} in + true) return 0 ;; + false) return 1 ;; + esac + [ -s "${DATA_DIR:-/var/lib/rancher/k3s}/agent/client-kubelet.crt" ] +} + +# tn_node_dropin_text: the content of the labels and taints file +tn_node_dropin_text() { + printf '%s\n' \ + '# Written by the Teknoir bootstrap script (tnctl bootstrap --role ...). Do not edit: a re-run rewrites it.' \ + '# "+" appends to the node-label and node-taint lists of config.yaml and the other drop-ins.' \ + '# k3s applies them only when the node registers for the first time.' + if [ -n "$TN_NODE_LABELS" ]; then + echo 'node-label+:' + printf '%s\n' "$TN_NODE_LABELS" | while IFS= read -r _tn_l; do printf ' - "%s"\n' "$_tn_l"; done + fi + if [ -n "$TN_NODE_TAINTS" ]; then + echo 'node-taint+:' + printf '%s\n' "$TN_NODE_TAINTS" | while IFS= read -r _tn_t; do printf ' - "%s"\n' "$_tn_t"; done + fi +} + +# tn_node_kubectl_help NODE: how to apply the labels and taints to a node that has registered already +tn_node_kubectl_help() { + _tn_kl= + if [ -n "$TN_NODE_LABELS" ]; then + _tn_kl=$(printf '%s\n' "$TN_NODE_LABELS" | tr '\n' ' ') + tn_inst_warn " k3s kubectl label node $1 ${_tn_kl}--overwrite" + fi + if [ -n "$TN_NODE_TAINTS" ]; then + _tn_kl=$(printf '%s\n' "$TN_NODE_TAINTS" | tr '\n' ' ') + tn_inst_warn " k3s kubectl taint node $1 ${_tn_kl}--overwrite" + fi +} + +# tn_node_apply ROLE: write the node name, and the labels and taints, into the k3s configuration +tn_node_apply() { + _tn_nrole=${1:-server} + _tn_ncfg=${TN_K3S_CONFIG_DIR:-/etc/rancher/k3s} + _tn_nyaml=$_tn_ncfg/config.yaml + mkdir -p "$_tn_ncfg" || tn_inst_fatal "cannot create $_tn_ncfg" + + # the name of the node: a node-name in config.yaml wins (it is the owner's), a different one is reported + if [ -n "$TN_NODE_NAME" ]; then + if tn_inst_yaml_has_key "$_tn_nyaml" node-name; then + _tn_have=$(tn_node_yaml_value "$_tn_nyaml" node-name) + if [ "$_tn_have" != "$TN_NODE_NAME" ]; then + tn_inst_warn "$_tn_nyaml sets node-name '$_tn_have', not '$TN_NODE_NAME' as asked for this $_tn_nrole: kept. The node keeps its name; edit the file to rename it (k3s registers a renamed node as a new one)" + fi + fi + tn_inst_config_ensure "$_tn_nyaml" node-name "node-name: $TN_NODE_NAME" + fi + + [ -n "$TN_NODE_LABELS$TN_NODE_TAINTS" ] || return 0 + + _tn_nfile=$_tn_ncfg/config.yaml.d/$TN_NODE_DROPIN + _tn_nwant=$(tn_node_dropin_text) + if [ -f "$_tn_nfile" ] && [ "$(cat "$_tn_nfile")" = "$_tn_nwant" ]; then + tn_inst_info "node labels and taints are already in place ($_tn_nfile)" + return 0 + fi + # atomically, and under a name that k3s does not read (it does not end in .yaml) + _tn_ntmp=$_tn_ncfg/config.yaml.d/.$TN_NODE_DROPIN.tmp.$$ + if mkdir -p "$_tn_ncfg/config.yaml.d" && printf '%s\n' "$_tn_nwant" > "$_tn_ntmp" && mv -f "$_tn_ntmp" "$_tn_nfile"; then + : + else + rm -f "$_tn_ntmp" 2>/dev/null || true + tn_inst_fatal "cannot write $_tn_nfile" + fi + tn_inst_info "Wrote the node labels and taints of this $_tn_nrole to $_tn_nfile" + + if tn_node_registered; then + _tn_nname=$(tn_node_yaml_value "$_tn_nyaml" node-name) + _tn_nname=${_tn_nname:-${TN_NODE_NAME:-$(hostname | tr 'A-Z' 'a-z')}} + tn_inst_warn "This node has registered with the cluster already, and k3s applies node-label and node-taint only when a node registers for the first time: the labels and taints above will NOT show up on it by themselves." + tn_inst_warn "Apply them to the running node from the k3s server (this changes the Node object only; the file above keeps them if the node ever registers again):" + tn_node_kubectl_help "$_tn_nname" + fi +} + +tn_node_name 'mobile-01-io' +tn_node_label 'teknoir.org/node-role=io' +tn_node_label 'teknoir.org/robot=mobile-01' +tn_node_label 'teknoir.org/io=cameras' + +info "Install device specific keys" +export CONFIG_PATH=/etc/teknoir +$SUDO mkdir -p ${CONFIG_PATH} +$SUDO mkdir -p /toe_conf && $SUDO rm -rf /toe_conf && $SUDO ln -s ${CONFIG_PATH}/ /toe_conf # For backward compatibility +download ${CONFIG_PATH}/roots.pem https://pki.goog/roots.pem +$SUDO chmod 440 ${CONFIG_PATH}/roots.pem +$SUDO chown 65532:root ${CONFIG_PATH}/roots.pem +$SUDO tee ${CONFIG_PATH}/rsa_private.pem > /dev/null << EOL +${_RSA_PRIVATE} +EOL +$SUDO chmod 440 ${CONFIG_PATH}/rsa_private.pem +$SUDO chown 65532:root ${CONFIG_PATH}/rsa_private.pem +$SUDO tee ${CONFIG_PATH}/rsa_public.pem > /dev/null << EOL +${_RSA_PUBLIC} +EOL +$SUDO chmod 444 ${CONFIG_PATH}/rsa_public.pem +$SUDO chown 65532:root ${CONFIG_PATH}/rsa_public.pem +$SUDO chgrp 65532 /etc/teknoir 2>/dev/null || true + + +$SUDO cp ${CONFIG_PATH}/rsa_public.pem /usr/local/share/ca-certificates/${_DEVICE_ID}-${_DOMAIN}.crt +$SUDO update-ca-certificates + +info "Install Rancher K3s Agent" +download k3s_installer.sh https://get.k3s.io +$SUDO chmod +x k3s_installer.sh + +if [ "${OS_BUILD}" ] || [ "${INSECURE}" ]; then + info "Running installation without verifying ssl certs on URLs" + $SUDO sed -i "s#curl -w#curl --insecure -w#g" k3s_installer.sh + $SUDO sed -i "s#curl -o#curl --insecure -o#g" k3s_installer.sh +fi + +if [ "${OS_BUILD}" ]; then + info "OS Build specifics" + export INSTALL_K3S_SKIP_START=true + $SUDO sed -i "s#-d /run/systemd#true#g" k3s_installer.sh +fi + +# tnctl bootstrap --server: the k3s server this agent joins. K3S_URL in the environment overrides it. +K3S_URL="${K3S_URL:-https://orin.example.test:6443}" +if [ -z "${K3S_URL}" ]; then + DEFAULT_K3S_URL="https://${_DEVICE_ID}.local:6443" + read -p "Is ${DEFAULT_K3S_URL} the correct URL to the server/master node? [Y/n] " confirm + confirm=${confirm:-"Y"} + if echo "${confirm}" | grep -iq "^y$"; then + export K3S_URL=${DEFAULT_K3S_URL} + else + read -p "Please enter the K3S_URL: " K3S_URL + export K3S_URL=${K3S_URL} + fi +fi +export K3S_URL=${K3S_URL} +export K3S_TOKEN="${K3S_TOKEN:-"${_K3S_TOKEN}"}" + +export INSTALL_K3S_EXEC="agent" +if [ "${USE_DOCKER}" = true ]; then + info "Use docker container-runtime for K3s" + export INSTALL_K3S_EXEC="${INSTALL_K3S_EXEC} --docker" +fi + +# Who chose the k3s release: the person who runs this script (INSTALL_K3S_VERSION / INSTALL_K3S_CHANNEL in the +# environment, or `tnctl bootstrap --k3s-version`) or the default of this script. An agent installs the +# release of its server unless it was asked for one (it is then refused if it is newer than the server). +if [ -n "${INSTALL_K3S_VERSION:-}${INSTALL_K3S_CHANNEL:-}" ]; then TN_K3S_PIN_SOURCE=env; else TN_K3S_PIN_SOURCE=default; fi +export TN_K3S_PIN_SOURCE +# k3s is pinned to v1.36.5+k3s1 (tnctl bootstrap --k3s-version). Override when running the script: +# INSTALL_K3S_VERSION=v1.xx.y+k3sN sh bootstrap_*.sh (an agent must run the version of its server) +# This is the release for a NEW device: a re-run on a device with k3s installed keeps the installed version +# (TN_K3S_UPGRADE=true, tnctl bootstrap --k3s-upgrade, replaces it; never a downgrade). +export INSTALL_K3S_VERSION="${INSTALL_K3S_VERSION:-v1.36.5+k3s1}" + +# A new agent gets the k3s release of its server; a re-run keeps the installed k3s (unless +# TN_K3S_UPGRADE=true) and sets fail-cgroupv1=false on cgroup v1 hosts, which Kubernetes >= 1.35 would refuse +# (the flag goes in from k3s v1.31, where the kubelet learned it). INSTALL_K3S_EXEC is final above, before +# tn_inst_prepare reads it. +tn_inst_prepare agent +# The node name, role labels and taints go into the k3s configuration before the installer starts k3s +# (a refusal of tn_inst_prepare above has changed nothing yet). +tn_node_apply agent +export INSTALL_K3S_SYMLINK=force +$SUDO ./k3s_installer.sh +# The kubelet flag that had to wait for the installer (an installed k3s older than v1.31 that was upgraded) +tn_inst_finish + +info "K3s agent node joined successfully!" + +warn "To enable tunneling, the user ${_FIRST_USER_NAME} has to be created." + +on_sudo() { + $SUDO sh -- "$@" +} + +setup_user() { + $SUDO mkdir -p "/home/${_FIRST_USER_NAME}" + if ! id -u "${_FIRST_USER_NAME}" >/dev/null 2>&1; then + if command -v useradd >/dev/null 2>&1 && command -v groupadd >/dev/null 2>&1; then + if ! getent group "${_FIRST_USER_NAME}" >/dev/null 2>&1; then + $SUDO groupadd "${_FIRST_USER_NAME}" + fi + $SUDO useradd \ + --gid "${_FIRST_USER_NAME}" \ + --groups users \ + --home-dir "/home/${_FIRST_USER_NAME}" \ + --shell /bin/bash \ + "${_FIRST_USER_NAME}" + else + $SUDO adduser --disabled-password --gecos "" "${_FIRST_USER_NAME}" + fi + fi + + if command -v chpasswd &> /dev/null; then + echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd + elif command -v passwd &> /dev/null; then + echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}" + elif command -v usermod &> /dev/null && command -v openssl &> /dev/null; then + usermod --password $(openssl passwd -1 "${_FIRST_USER_PASS}") "${_FIRST_USER_NAME}" + else + warn "Could not set password for user ${_FIRST_USER_NAME}" + info "Skipping tunneling setup" + exit + fi + + on_sudo << EOF +mkdir -p "/home/${_FIRST_USER_NAME}/.ssh" +cp "${CONFIG_PATH}/rsa_private.pem" "/home/${_FIRST_USER_NAME}/.ssh/id_rsa" +ssh-keygen -y -f "${CONFIG_PATH}/rsa_private.pem" > "/home/${_FIRST_USER_NAME}/.ssh/id_rsa.pub" +cat "/home/${_FIRST_USER_NAME}/.ssh/id_rsa.pub" > "/home/${_FIRST_USER_NAME}/.ssh/authorized_keys" +if [ -z "\${_FIRST_USER_KEY+x}" ] || [ "\${_FIRST_USER_KEY}" = "" ]; then + echo "[INFO] _FIRST_USER_KEY is not set, will continue with only password auth"; +else + echo "\${_FIRST_USER_KEY}" >> "/home/${_FIRST_USER_NAME}/.ssh/authorized_keys" +fi +chmod 600 /home/${_FIRST_USER_NAME}/.ssh/* +chmod 744 /home/${_FIRST_USER_NAME}/.ssh +chown -R "${_FIRST_USER_NAME}:${_FIRST_USER_NAME}" "/home/${_FIRST_USER_NAME}" + +# Add root user to users and sudo groups +if [ -e /etc/group ]; then + sed -i "/^users:/s/\(.*\)/\1,${_FIRST_USER_NAME}/;s/:,/:/" /etc/group + sed -i "/^sudo:/s/\(.*\)/\1,${_FIRST_USER_NAME}/;s/:,/:/" /etc/group +fi +EOF +} + + +CREATE_USER=${CREATE_USER:-ask} +if [ "${OS_BUILD}" ] || [ "${CREATE_USER}" = true ]; then + setup_user +else + if id -u "${_FIRST_USER_NAME}" >/dev/null 2>&1; then + warn "The user ${_FIRST_USER_NAME} already exist! This operation will change the password and OVERWRITE ssh keys for this users." + fi + warn "Do you want to add \"${_FIRST_USER_NAME}\" as a user and enable tunneling? [yY]" + read REPLY + + case ${REPLY} in + [Yy]* ) + setup_user + ;; + * ) + info "Skipping tunneling setup" + ;; + esac +fi + + +cd .. +info "Device bootstrapped successfully!" + + diff --git a/pkg/templates/testdata/server.roles.golden.sh b/pkg/templates/testdata/server.roles.golden.sh new file mode 100644 index 0000000..0565798 --- /dev/null +++ b/pkg/templates/testdata/server.roles.golden.sh @@ -0,0 +1,3367 @@ +#!/bin/sh +set -e +set -o noglob +#set -x +umask 027 + +# --- helper functions --- +info() { echo '[INFO] ' "$@"; } +warn() { echo '[WARN] ' "$@" >&2; } +fatal() { echo '[ERROR] ' "$@" >&2; exit 1; } + +# Exit if not running as root +[ $(id -u) -eq 0 ] || fatal 'You must run this script as root' + +DOWNLOADER= +# --- download --- +download() { + [ $# -eq 2 ] || fatal 'download needs exactly 2 arguments' + + case $DOWNLOADER in + curl) + if [ "${OS_BUILD}" ] || [ "${INSECURE}" ]; then + info "Running installation without verifying ssl certs on URLs" + $SUDO curl --insecure -o $1 -sSfL $2 + else + $SUDO curl -o $1 -sSfL $2 + fi + ;; + wget) + $SUDO wget -qO $1 $2 + ;; + *) + fatal "Incorrect executable '$DOWNLOADER'" + ;; + esac + + # Abort if download command failed + [ $? -eq 0 ] || fatal 'Download failed' +} + +# --- verify existence of network downloader executable --- +verify_downloader() { + # Return failure if it doesn't exist or is no executable + [ -x "$(command -v $1)" ] || return 1 + + # Set verified executable as our downloader program and return success + DOWNLOADER=$1 + return 0 +} + +verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' + + +# This is the drop in device bootstrap script for the Teknoir platform. +# It should work for most arm32/arm64/amd64 devices running Debian or Ubuntu. + +# Usage: +# TBD +# +# Environment variables: +# OS_BUILD=true (default: false, for OS builds) +# INSECURE=true (default: false, for OS builds, or if you want to run without ssl cert verification) +# INSTALL_WATCHDOG=true (default: false, enable or disable the teknoir watchdog service) +# USE_DOCKER=true (default: false, enable the use of docker as container runtime for k3sm inly works for k3s < 1.24) +# CREATE_USER=false (default: ask, create a user for the device without asking) +# TN_K3S_CA_AUTO_REMEDIATE=false (default: true; a re-run on a device bootstrapped with the old +# root CA repairs it before k3s is touched. false stops with advice instead) +# TN_K3S_UPGRADE=true (default: false; a re-run KEEPS the installed k3s version. true lets the pin +# below replace it: never a downgrade, never more than one minor version) +# TN_K3S_FORCE=true (default: false; allows an upgrade over more than one minor version) +# TN_K3S_CGROUPV1=true|false (default: detected; on cgroup v1 with k3s >= v1.31 the kubelet gets +# fail-cgroupv1=false, see /etc/rancher/k3s/config.yaml.d/99-teknoir-cgroupv1.yaml) + + +TMP=$(mktemp -dt) +# --- cleanup/error trap --- +error() { + local code="$1" + echo "[ERROR] Error: exiting with status ${code}" + exit "${code}" +} +cleanup() { + rm -rf "$TMP" + [ $? -eq 0 ] && exit + error "$?" +} +trap 'cleanup' EXIT + +cd "$TMP" +info "Teknoir bootstrapping...${TMP}" + +_DOMAIN="example.test" +_DEVICE_ID="golden-device" +_NAMESPACE="golden-ns" +_FIRST_USER_NAME="golden-user" +_FIRST_USER_PASS="TEST-ONLY-password" +_FIRST_USER_KEY="ssh-ed25519 TEST-ONLY-ssh-public golden@example.test" +_RSA_PUBLIC="TEST-ONLY-rsa-public +TEST-ONLY-rsa-public-line2" +_RSA_PRIVATE="TEST-ONLY-rsa-private +TEST-ONLY-rsa-private-line2" +_K3S_TOKEN="TEST-ONLY-k3s-token" + + +info "Install device specific keys" +export CONFIG_PATH=/etc/teknoir +$SUDO mkdir -p ${CONFIG_PATH} +$SUDO mkdir -p /toe_conf && $SUDO rm -rf /toe_conf && $SUDO ln -s ${CONFIG_PATH}/ /toe_conf # For backward compatibility +download ${CONFIG_PATH}/roots.pem https://pki.goog/roots.pem +$SUDO chmod 440 ${CONFIG_PATH}/roots.pem +$SUDO chown 65532:root ${CONFIG_PATH}/roots.pem +$SUDO tee ${CONFIG_PATH}/rsa_private.pem > /dev/null << EOL +${_RSA_PRIVATE} +EOL +$SUDO chmod 440 ${CONFIG_PATH}/rsa_private.pem +$SUDO chown 65532:root ${CONFIG_PATH}/rsa_private.pem +$SUDO tee ${CONFIG_PATH}/rsa_public.pem > /dev/null << EOL +${_RSA_PUBLIC} +EOL +$SUDO chmod 444 ${CONFIG_PATH}/rsa_public.pem +$SUDO chown 65532:root ${CONFIG_PATH}/rsa_public.pem +$SUDO chgrp 65532 /etc/teknoir 2>/dev/null || true + +# The k3s CA of the device: the device certificate becomes the root CA (re-issued with +# keyCertSign, which k3s >= v1.34.12/v1.35.9/v1.36.5/v1.37.1 requires). The library is also +# available stand-alone as pkg/templates/scripts/tn-k3s-ca.sh (check / remediate). +TN_K3S_CA_NO_MAIN=1 +#!/bin/sh +# tn-k3s-ca.sh - the custom k3s CA of a Teknoir device: setup, check, remediate. +# +# Background. A Teknoir device uses its own certificate (Device CR settings, +# /etc/teknoir/rsa_public.pem + rsa_private.pem) as the ROOT of the k3s cluster +# CA and issues the intermediate and the five leaf CAs below it. k3s v1.34.12, +# v1.35.9, v1.36.5, v1.37.1 and later (k3s-io/k3s#14699, "Improve CA hash +# validation") drop every certificate without the keyCertSign key usage from the +# CA bundle before they build the trust pool. The device certificates of the +# platform never had keyCertSign, so such a k3s refuses to start +# ("CA cert validation failed ... x509: certificate signed by unknown authority"). +# +# The fix is to give the root the missing key usage. The root is re-issued from +# the SAME key and subject (and subject key identifier): a new self-signed root +# with CA:TRUE and keyCertSign, valid 20 years. Everything signed below the old +# root stays valid (same key, same subject), and the device files are never +# modified. +# +# Three ways to use this file: +# +# . tn-k3s-ca.sh; tn_k3s_ca_setup bootstrap scripts (sourced): creates the +# PKI of a new device, leaves an +# initialised one alone, repairs an old one +# sh tn-k3s-ca.sh check is the root of this device acceptable to k3s? +# sh tn-k3s-ca.sh remediate repair a device bootstrapped with the old root +# +# POSIX sh. It runs under `set -e` and `set -o noglob` (as the bootstrap scripts +# do) and does not change shell options, traps or the working directory of the +# caller. Sourcing it defines functions only. +# +# Environment (all optional when run as a program): +# CONFIG_PATH directory with rsa_public.pem and rsa_private.pem (/etc/teknoir) +# DATA_DIR k3s data dir. When it is not set (and no --data-dir is given) it is resolved the way +# k3s resolves it, see tn_ca_resolve_datadir: an explicit DATA_DIR/--data-dir, then +# --data-dir/-d in the ExecStart of the k3s systemd unit, then the last data-dir in +# /etc/rancher/k3s/config.yaml.d/*.yaml, then data-dir in /etc/rancher/k3s/config.yaml, +# then K3S_DATA_DIR of the unit's environment, then /var/lib/rancher/k3s +# PRODUCT k3s +# _DEVICE_ID, _DOMAIN used in the CN of the generated CAs (taken from the +# existing CAs when not set) +# K3S_BIN k3s command used by `rotate-ca` (k3s) +# TN_K3S_RESTART_CMD restarts k3s after a remediation (systemctl restart k3s) +# TN_K3S_SERVICE_TIMEOUT seconds that a stop, start or restart of k3s may take (600: a Raspberry Pi or a +# Jetson with a large datastore takes minutes); then the unit is watched for TN_K3S_READY_GRACE +# more seconds (120) and only after that reported as failed instead of hanging (the k3s unit +# waits forever for a k3s that cannot start) +# TN_K3S_READY_GRACE, TN_K3S_READY_POLL seconds to keep polling the unit for 'active' after a restart that +# timed out (120), and the interval between polls (5) +# TN_K3S_ROTATE_TRIES, TN_K3S_ROTATE_WAIT `k3s certificate rotate-ca` needs a k3s server that answers. When the +# k3s unit is starting (or just restarted) it is tried up to this many times (6), this many +# seconds apart (10). Without a k3s unit it is tried once +# TN_K3S_STOP_CMD, TN_K3S_START_CMD stop/start k3s around the replacement of server/tls when the +# PKI is regenerated (systemctl stop|start k3s). Without TN_K3S_STOP_CMD k3s is only +# stopped if the systemd unit is active, and then started again +# TN_K3S_CONFIG_DIR /etc/rancher/k3s (config.yaml and config.yaml.d) +# TN_K3S_UNIT_FILE a unit file to read instead of `systemctl cat k3s.service` (tests) +# TN_K3S_CA_AUTO_REMEDIATE true (default) / false: a re-run of a bootstrap script +# repairs an initialised data dir itself; false stops with advice +# +# check exit codes: +# 0 ok (the root is acceptable to k3s, or this is not a server: no k3s server unit and no +# server/tls/server-ca.crt) +# 2 needs-rotation: initialised data dir, root unusable => `remediate` +# 3 needs-regeneration: data dir never initialised, PKI unusable => `remediate` +# (or re-run the bootstrap script) +# 4 foreign: the root is unusable or expires soon but we do not hold its key (not a Teknoir CA); +# or no-server-data: a k3s server unit exists but the data dir that was resolved has no +# server/tls/server-ca.crt (k3s has not started yet, or its data dir is elsewhere: pass --data-dir) +# 1 error +# +# remediate exit codes: 0 done, 1 failed (nothing changed, except the saved backup), 2 usage, +# 4 foreign/no server data, 5 the CA WAS changed but a later step failed (finalize or the k3s restart): +# the datastore holds the new CA, do not roll back; the message says how to continue. +# +# Rollback limits. The backups (server/tls-backup-tn-.tar, the newest 3 are kept) hold server/tls +# only. `k3s certificate rotate-ca` also writes the new CA into the datastore of k3s, which this tool +# does not back up: once it has succeeded, restoring the tar alone does not go back (the files and +# the datastore would disagree). A real rollback needs a snapshot of the datastore taken before +# (stop k3s, copy DATA_DIR/server/db). The staged files are not committed before rotate-ca +# succeeded, and a failing rotate-ca leaves the device as it was. + +TN_K3S_CA_LIB_VERSION=1 +export TN_K3S_CA_LIB_VERSION + +# ----------------------------------------------------------------- messages +# The library prints with its own functions, in the format of header.sh.tmpl. It must not use a +# caller's info/warn/fatal (there may be none: a stand-alone run) and must not trust `info` +# to be a function either: on Debian/Ubuntu it is the GNU info reader. +tn_ca_info() { echo '[INFO] ' "$@"; } +tn_ca_warn() { echo '[WARN] ' "$@" >&2; } +tn_ca_fatal() { echo '[ERROR] ' "$@" >&2; exit 1; } + +# ----------------------------------------------------------------- small helpers + +# Select the openssl binary (3.x if installed side by side, as on some distros). +tn_ca_openssl() { + if command -v openssl-3 >/dev/null 2>&1; then + OPENSSL=openssl-3 + elif command -v openssl >/dev/null 2>&1; then + OPENSSL=openssl + else + tn_ca_warn "openssl not found" + return 1 + fi + if ! "$OPENSSL" ecparam -name prime256v1 -genkey -noout -out /dev/null >/dev/null 2>&1; then + tn_ca_warn "openssl missing Elliptic Curve (ecparam) support" + return 1 + fi + return 0 +} + +tn_ca_ts() { date +%Y%m%dT%H%M%S; } + +# tn_ca_changed TEXT: records, in TN_K3S_CA_CHANGED (exported), what this run has changed on the device +# for good. A later refusal (the k3s release policy, a failing step) says it instead of claiming that +# nothing was changed. +tn_ca_changed() { + if [ -n "${TN_K3S_CA_CHANGED:-}" ]; then TN_K3S_CA_CHANGED="$TN_K3S_CA_CHANGED; $1"; else TN_K3S_CA_CHANGED=$1; fi + export TN_K3S_CA_CHANGED +} + +# tn_ca_split PEMFILE DIR: write every certificate of PEMFILE to DIR/c.pem, N from 0. +# Echoes the number of certificates. +tn_ca_split() { + mkdir -p "$2" || return 1 + awk -v d="$2" ' + /-----BEGIN CERTIFICATE-----/ { f = sprintf("%s/c%d.pem", d, n); n++ } + f { print > f } + /-----END CERTIFICATE-----/ { if (f != "") close(f); f = "" } + END { print n + 0 }' "$1" +} + +# tn_ca_fp CERT: sha256 fingerprint +tn_ca_fp() { "$OPENSSL" x509 -in "$1" -noout -fingerprint -sha256 | cut -d= -f2; } + +# tn_ca_pubhash_cert CERT / tn_ca_pubhash_key KEY: sha256 of the public key (SPKI) +tn_ca_pubhash_cert() { "$OPENSSL" x509 -in "$1" -noout -pubkey 2>/dev/null | sha256sum | cut -d' ' -f1; } +tn_ca_pubhash_key() { "$OPENSSL" pkey -in "$1" -pubout 2>/dev/null | sha256sum | cut -d' ' -f1; } + +# tn_ca_ski CERT: subject key identifier as printed by openssl, or empty +tn_ca_ski() { + "$OPENSSL" x509 -in "$1" -noout -text 2>/dev/null | + awk '/X509v3 Subject Key Identifier/ { getline; gsub(/[ \t]/, ""); print; exit }' +} + +# tn_ca_has_certsign CERT +tn_ca_has_certsign() { "$OPENSSL" x509 -in "$1" -noout -text 2>/dev/null | grep -q 'Certificate Sign'; } + +# tn_ca_dir_ok DIR: a data dir whose server/token exists was started by k3s once +tn_ca_initialised() { [ -s "$1/server/token" ]; } + +# tn_ca_lastcert BUNDLE OUTFILE: write the last (root) certificate of BUNDLE +tn_ca_lastcert() { + _tn_w=$(mktemp -d) || return 1 + _tn_n=$(tn_ca_split "$1" "$_tn_w") + if [ "${_tn_n:-0}" -lt 1 ]; then rm -rf "$_tn_w"; return 1; fi + cp "$_tn_w/c$((_tn_n - 1)).pem" "$2" + rm -rf "$_tn_w" +} + +# Bundles k3s reads (leaf first, root last), relative to server/tls. +TN_CA_BUNDLES="server-ca.crt client-ca.crt request-header-ca.crt etcd/peer-ca.crt etcd/server-ca.crt intermediate-ca.crt root-ca.crt" + +# ----------------------------------------------------------------- the k3s data dir + +# The data dir of a k3s server, resolved the way k3s resolves it (verified against rancher/k3s +# v1.36.5: a command line flag beats config.yaml.d/*.yaml, which beats config.yaml). First match wins: +# 1. an explicit DATA_DIR (the environment, or --data-dir of this tool) +# 2. --data-dir DIR, --data-dir=DIR, -d DIR or -d=DIR in the ExecStart of the k3s systemd unit +# 3. the last data-dir in /etc/rancher/k3s/config.yaml.d/*.yaml (lexical order) +# 4. data-dir in /etc/rancher/k3s/config.yaml +# 5. K3S_DATA_DIR in the environment of the unit (Environment=, EnvironmentFile=) +# 6. /var/lib/rancher/k3s +# Sets and exports DATA_DIR and TN_CA_DATADIR_SRC (where it came from). Reading only; no globbing +# (the bootstrap scripts run with noglob). A data-dir with white space in it is not supported. + +tn_ca_unit_name() { echo "${PRODUCT:-k3s}.service"; } + +# tn_ca_unit_text: the text of the k3s server unit (merged with drop-ins when systemd tells), or fail +tn_ca_unit_text() { + _tn_un=$(tn_ca_unit_name) + if [ -n "${TN_K3S_UNIT_FILE:-}" ]; then + [ -r "$TN_K3S_UNIT_FILE" ] && cat "$TN_K3S_UNIT_FILE" + return + fi + if command -v systemctl >/dev/null 2>&1 && systemctl cat "$_tn_un" 2>/dev/null; then return 0; fi + for _tn_uf in "/etc/systemd/system/$_tn_un" "/lib/systemd/system/$_tn_un" "/usr/lib/systemd/system/$_tn_un"; do + if [ -r "$_tn_uf" ]; then cat "$_tn_uf"; return 0; fi + done + return 1 +} + +# tn_ca_unit_exists: is there a k3s SERVER unit (k3s.service; an agent has k3s-agent.service) +tn_ca_unit_exists() { tn_ca_unit_text >/dev/null 2>&1; } + +# tn_ca_yaml_datadir FILE: the value of the last top-level `data-dir:` key, or nothing +tn_ca_yaml_datadir() { + [ -r "$1" ] || return 0 + awk ' + { sub(/\r$/, "") } + /^data-dir[ \t]*:/ { + v = $0; sub(/^data-dir[ \t]*:[ \t]*/, "", v) + q = substr(v, 1, 1) + if (q == "\"" || q == "\047") { v = substr(v, 2); i = index(v, q); if (i > 0) v = substr(v, 1, i - 1) } + else { sub(/[ \t]+#.*$/, "", v); sub(/^#.*$/, "", v); sub(/[ \t]+$/, "", v) } + val = v + } + END { if (val != "") print val }' "$1" +} + +# tn_ca_unit_datadir: --data-dir/-d of the last ExecStart= of the unit, or nothing +tn_ca_unit_datadir() { + tn_ca_unit_text 2>/dev/null | awk ' + cont { line = $0; if (line ~ /\\[ \t]*$/) { sub(/\\[ \t]*$/, "", line); buf = buf " " line } else { buf = buf " " line; cont = 0; last = buf; buf = "" } ; next } + /^ExecStart=/ { + line = $0; sub(/^ExecStart=/, "", line) + if (line ~ /\\[ \t]*$/) { sub(/\\[ \t]*$/, "", line); buf = line; cont = 1 } else { last = line } + next } + END { + if (cont) last = buf + gsub(/["\047]/, "", last); sub(/^[ \t]*[-@+!:]+/, "", last) + n = split(last, t, /[ \t]+/); val = "" + for (i = 1; i <= n; i++) { + if ((t[i] == "--data-dir" || t[i] == "-d") && i < n) val = t[i + 1] + else if (t[i] ~ /^--data-dir=/) val = substr(t[i], 12) + else if (t[i] ~ /^-d=/) val = substr(t[i], 4) + } + if (val != "") print val }' +} + +# tn_ca_unit_env_datadir: K3S_DATA_DIR from Environment=/EnvironmentFile= of the unit +tn_ca_unit_env_datadir() { + _tn_ut=$(tn_ca_unit_text 2>/dev/null) || return 0 + _tn_ev=$(printf '%s\n' "$_tn_ut" | sed -n 's/^Environment=.*K3S_DATA_DIR=\([^ "]*\).*/\1/p' | tail -n 1) + if [ -z "$_tn_ev" ]; then + for _tn_ef in $(printf '%s\n' "$_tn_ut" | sed -n 's/^EnvironmentFile=-\{0,1\}//p'); do + [ -r "$_tn_ef" ] || continue + _tn_v=$(sed -n 's/^K3S_DATA_DIR=//p' "$_tn_ef" | tail -n 1 | sed "s/^['\"]//; s/['\"]\$//") + [ -z "$_tn_v" ] || _tn_ev=$_tn_v + done + fi + [ -z "$_tn_ev" ] || echo "$_tn_ev" +} + +# tn_ca_config_dropins DIR: the drop-ins k3s reads from DIR, in the order it reads them (the last one wins): +# *.yaml and *.yml, the suffix without regard to case (k3s: HasSuffixI in pkg/configfilearg), by name. +tn_ca_config_dropins() { + find "$1" -maxdepth 1 -type f \( -iname '*.yaml' -o -iname '*.yml' \) 2>/dev/null | LC_ALL=C sort +} + +tn_ca_resolve_datadir() { + _tn_pr=${PRODUCT:-k3s} + if [ -n "${DATA_DIR:-}" ]; then + # our own earlier resolution keeps its source; anything else was given by the caller + if [ "${TN_CA_DATADIR_RES:-}" != "$DATA_DIR" ]; then TN_CA_DATADIR_SRC="--data-dir / DATA_DIR"; fi + export DATA_DIR TN_CA_DATADIR_SRC; return 0 + fi + _tn_cd=${TN_K3S_CONFIG_DIR:-/etc/rancher/$_tn_pr} + _tn_v=$(tn_ca_unit_datadir) + if [ -n "$_tn_v" ]; then DATA_DIR=$_tn_v; TN_CA_DATADIR_SRC="ExecStart of $(tn_ca_unit_name)"; fi + if [ -z "$_tn_v" ] && [ -d "$_tn_cd/config.yaml.d" ]; then + for _tn_f in $(tn_ca_config_dropins "$_tn_cd/config.yaml.d"); do + _tn_x=$(tn_ca_yaml_datadir "$_tn_f") + if [ -n "$_tn_x" ]; then _tn_v=$_tn_x; TN_CA_DATADIR_SRC=$_tn_f; fi + done + [ -z "$_tn_v" ] || DATA_DIR=$_tn_v + fi + if [ -z "$_tn_v" ]; then + _tn_v=$(tn_ca_yaml_datadir "$_tn_cd/config.yaml") + if [ -n "$_tn_v" ]; then DATA_DIR=$_tn_v; TN_CA_DATADIR_SRC="$_tn_cd/config.yaml"; fi + fi + if [ -z "$_tn_v" ]; then + _tn_v=$(tn_ca_unit_env_datadir) + if [ -n "$_tn_v" ]; then DATA_DIR=$_tn_v; TN_CA_DATADIR_SRC="K3S_DATA_DIR of $(tn_ca_unit_name)"; fi + fi + if [ -z "$_tn_v" ]; then DATA_DIR=/var/lib/rancher/$_tn_pr; TN_CA_DATADIR_SRC=default; fi + TN_CA_DATADIR_RES=$DATA_DIR + export DATA_DIR TN_CA_DATADIR_SRC TN_CA_DATADIR_RES +} + +# tn_ca_datadir_consistent: 0 if DATA_DIR is the data dir that the config files of k3s set, or they set +# none. Sets TN_CA_CFG_DATADIR / TN_CA_CFG_DATADIR_SRC to the config file's value and file otherwise. +# The k3s installer writes a new unit WITHOUT --data-dir and K3S_DATA_DIR: from then on k3s takes its data +# dir from the config files. A DATA_DIR that came from anywhere else (the ExecStart of the old unit, +# K3S_DATA_DIR, the caller) and differs from them would send k3s to a different directory, with the +# CA we prepared, and the cluster, left behind. +tn_ca_datadir_consistent() { + TN_CA_CFG_DATADIR=; TN_CA_CFG_DATADIR_SRC= + _tn_cd=${TN_K3S_CONFIG_DIR:-/etc/rancher/${PRODUCT:-k3s}} + if [ -d "$_tn_cd/config.yaml.d" ]; then + for _tn_f in $(tn_ca_config_dropins "$_tn_cd/config.yaml.d"); do + _tn_x=$(tn_ca_yaml_datadir "$_tn_f") + if [ -n "$_tn_x" ]; then TN_CA_CFG_DATADIR=$_tn_x; TN_CA_CFG_DATADIR_SRC=$_tn_f; fi + done + fi + if [ -z "$TN_CA_CFG_DATADIR" ]; then + _tn_x=$(tn_ca_yaml_datadir "$_tn_cd/config.yaml") + if [ -n "$_tn_x" ]; then TN_CA_CFG_DATADIR=$_tn_x; TN_CA_CFG_DATADIR_SRC=$_tn_cd/config.yaml; fi + fi + [ -n "$TN_CA_CFG_DATADIR" ] || return 0 + _tn_a=$(printf '%s\n' "$DATA_DIR" | sed 's|//*|/|g; s|\(.\)/$|\1|') + _tn_b=$(printf '%s\n' "$TN_CA_CFG_DATADIR" | sed 's|//*|/|g; s|\(.\)/$|\1|') + [ "$_tn_a" = "$_tn_b" ] +} + +# tn_ca_require_consistent_datadir: tn_ca_datadir_consistent, or stop with advice. Needs DATA_DIR (call +# tn_ca_resolve_datadir first). A server script calls it right after tn_ca_resolve_datadir, before the +# k3s release step changes anything; tn_k3s_ca_setup calls it again (cheap, and the library stays safe on its own). +tn_ca_require_consistent_datadir() { + tn_ca_datadir_consistent || + tn_ca_fatal "k3s runs from ${DATA_DIR} (${TN_CA_DATADIR_SRC}), but ${TN_CA_CFG_DATADIR_SRC} sets data-dir: ${TN_CA_CFG_DATADIR}. The k3s installer that follows writes a new unit without --data-dir, after which k3s would start from ${TN_CA_CFG_DATADIR}: an empty directory, with its own new CA, and the cluster in ${DATA_DIR} left behind. Make them agree first: either set data-dir in ${TN_CA_CFG_DATADIR_SRC} to ${DATA_DIR} (if that is where the cluster is), or remove that line and run this script again. The CA and k3s were not changed" +} + +# ----------------------------------------------------------------- the root certificate + +# tn_ca_reissue_root SRC_CERT KEY OUT_CERT +# New self-signed root from the SAME key and subject as SRC_CERT (and the same subject key +# identifier), with CA:TRUE and keyUsage keyCertSign, valid 7300 days and a random 128-bit +# serial (never the serial of SRC_CERT). The extensions of SRC_CERT are not carried over (no +# duplicates, no extendedKeyUsage). +tn_ca_reissue_root() { + _tn_rw=$(mktemp -d) || return 1 + _tn_rski=$(tn_ca_ski "$1") + [ -n "$_tn_rski" ] || _tn_rski='hash' + cat > "$_tn_rw/root.cnf" </dev/null 2>&1 + _tn_rc=$? + rm -rf "$_tn_rw" + if [ "$_tn_rc" -ne 0 ]; then tn_ca_warn "cannot re-issue the root certificate from $1"; return 1; fi + tn_ca_has_certsign "$3" || { tn_ca_warn "re-issued root has no keyCertSign"; return 1; } + if [ "$("$OPENSSL" x509 -in "$1" -noout -serial 2>/dev/null)" = "$("$OPENSSL" x509 -in "$3" -noout -serial 2>/dev/null)" ]; then + tn_ca_warn "re-issued root has the serial of the original certificate (this openssl ignores -set_serial)" + fi +} + +# ----------------------------------------------------------------- PKI generation + +# tn_ca_generate_pki TLS_DIR ROOT_CERT ROOT_KEY SUFFIX +# TLS_DIR must exist and be empty. Creates root-ca.{pem,key,crt}, intermediate-ca.*, the five +# leaf CAs and service.key in it exactly as the old bootstrap scripts did, below ROOT_CERT. +# Runs in a subshell: the caller's directory and traps are not touched. +tn_ca_generate_pki() { + ( + # `set -e` is only a convenience here: this function is always called in an `if`/`||` + # context, where every POSIX shell ignores errexit, even when it is set again in a + # subshell. So EVERY step ends in `|| exit 1`, and the caller validates the result. + set -e + tls=$1 root=$2 rkey=$3 suffix=$4 + product=${PRODUCT:-k3s} + cd "$tls" || exit 1 + mkdir -p etcd .ca/certs || exit 1 + cp "$root" root-ca.pem || exit 1 + cp "$rkey" root-ca.key || exit 1 + : > .ca/index || exit 1 + "$OPENSSL" rand -hex 8 > .ca/serial || exit 1 + cat > .ca/config <<'EOF' || exit 1 +[ca] +default_ca = ca_default +[ca_default] +dir = ./.ca +database = $dir/index +serial = $dir/serial +new_certs_dir = $dir/certs +default_md = sha256 +policy = policy_anything +[policy_anything] +commonName = supplied +[req] +distinguished_name = req_distinguished_name +[req_distinguished_name] +[v3_ca] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +basicConstraints = critical, CA:true +keyUsage = critical, digitalSignature, keyEncipherment, keyCertSign +EOF + if "$OPENSSL" version | grep -qF 'OpenSSL 3'; then GENRSA_FLAGS=-traditional; else GENRSA_FLAGS=; fi + + # a CN is at most 64 characters (long device names used to break the bootstrap here) + cn() { printf '%s' "$1" | cut -c1-64; } + # quiet openssl, but show what it said when it fails + quiet() { "$@" > .ca/log 2>&1 || { cat .ca/log >&2; return 1; }; } + # tn_issue CSR OUT SIGNER_KEY SIGNER_CERT. Requests go through files, not a pipe: no + # /dev/stdin (missing in some chroots), and a failing `req` is not hidden by the pipeline. + tn_issue() { + quiet "$OPENSSL" ca -batch -notext -days 3700 -in "$1" -out "$2" \ + -keyfile "$3" -cert "$4" -config .ca/config -extensions v3_ca + } + + tn_ca_info "Generating Kubernetes service account issuer RSA key" + # shellcheck disable=SC2086 + quiet "$OPENSSL" genrsa $GENRSA_FLAGS -out service.key 2048 || exit 1 + + cat root-ca.pem > root-ca.crt || exit 1 + + tn_ca_info "Generating intermediate certificate authority RSA key and certificate" + # shellcheck disable=SC2086 + quiet "$OPENSSL" genrsa $GENRSA_FLAGS -out intermediate-ca.key 4096 || exit 1 + quiet "$OPENSSL" req -new -nodes -subj "/CN=$(cn "${product}-intermediate-ca@${suffix}")" \ + -key intermediate-ca.key -out .ca/req.csr || exit 1 + tn_issue .ca/req.csr intermediate-ca.pem root-ca.key root-ca.pem || exit 1 + cat intermediate-ca.pem root-ca.pem > intermediate-ca.crt || exit 1 + + # a new leaf CA for every control-plane and etcd component + for TYPE in client server request-header etcd/peer etcd/server; do + name="${product}-$(echo "$TYPE" | tr / -)-ca" + tn_ca_info "Generating ${name} leaf certificate authority EC key and certificate" + quiet "$OPENSSL" ecparam -name prime256v1 -genkey -noout -out "${TYPE}-ca.key" || exit 1 + quiet "$OPENSSL" req -new -nodes -subj "/CN=$(cn "${name}@${suffix}")" -key "${TYPE}-ca.key" \ + -out .ca/req.csr || exit 1 + tn_issue .ca/req.csr "${TYPE}-ca.pem" intermediate-ca.key intermediate-ca.pem || exit 1 + cat "${TYPE}-ca.pem" intermediate-ca.pem root-ca.pem > "${TYPE}-ca.crt" || exit 1 + done + rm -rf .ca || exit 1 + ) +} + +# tn_ca_validate_pki TLS_DIR: 0 only if TLS_DIR holds a complete PKI: every file k3s reads is +# there and not empty, the bundles have the expected number of certificates, every key matches +# its certificate, and the intermediate and all leaf CAs chain to root-ca.pem. Warns about the +# first problem. Run on a freshly generated directory BEFORE it replaces the live one. +tn_ca_validate_pki() { + _tn_vd=$1 + for _tn_f in root-ca.pem root-ca.key root-ca.crt service.key intermediate-ca.pem intermediate-ca.key intermediate-ca.crt \ + client-ca.pem client-ca.key client-ca.crt server-ca.pem server-ca.key server-ca.crt \ + request-header-ca.pem request-header-ca.key request-header-ca.crt \ + etcd/peer-ca.pem etcd/peer-ca.key etcd/peer-ca.crt etcd/server-ca.pem etcd/server-ca.key etcd/server-ca.crt; do + [ -s "$_tn_vd/$_tn_f" ] || { tn_ca_warn "generated PKI is incomplete: $_tn_f is missing or empty"; return 1; } + done + _tn_vw=$(mktemp -d) || return 1 + _tn_vok=1 + for _tn_t in intermediate client server request-header etcd/peer etcd/server; do + _tn_c=$_tn_vd/${_tn_t}-ca.pem + "$OPENSSL" verify -CAfile "$_tn_vd/root-ca.pem" -untrusted "$_tn_vd/intermediate-ca.pem" "$_tn_c" >/dev/null 2>&1 || + { tn_ca_warn "generated PKI: ${_tn_t}-ca.pem does not chain to root-ca.pem"; _tn_vok=0; break; } + [ "$(tn_ca_pubhash_cert "$_tn_c")" = "$(tn_ca_pubhash_key "$_tn_vd/${_tn_t}-ca.key")" ] || + { tn_ca_warn "generated PKI: ${_tn_t}-ca.key does not belong to ${_tn_t}-ca.pem"; _tn_vok=0; break; } + case $_tn_t in intermediate) _tn_want=2 ;; *) _tn_want=3 ;; esac + rm -rf "$_tn_vw/b" + _tn_n=$(tn_ca_split "$_tn_vd/${_tn_t}-ca.crt" "$_tn_vw/b") + [ "${_tn_n:-0}" -eq "$_tn_want" ] || + { tn_ca_warn "generated PKI: ${_tn_t}-ca.crt holds ${_tn_n:-0} certificates, expected $_tn_want"; _tn_vok=0; break; } + done + if [ "$_tn_vok" = 1 ]; then + [ "$(tn_ca_pubhash_cert "$_tn_vd/root-ca.pem")" = "$(tn_ca_pubhash_key "$_tn_vd/root-ca.key")" ] || + { tn_ca_warn "generated PKI: root-ca.key does not belong to root-ca.pem"; _tn_vok=0; } + fi + rm -rf "$_tn_vw" + [ "$_tn_vok" = 1 ] +} + +# ----------------------------------------------------------------- state of a data dir + +# tn_ca_find_key DATA_DIR ROOT_CERT -> sets TN_CA_KEY to a private key file whose public +# key matches ROOT_CERT (server/tls/root-ca.key first, then the device key); returns 1 if none. +tn_ca_find_key() { + TN_CA_KEY=; TN_CA_UNREADABLE= + _tn_want=$(tn_ca_pubhash_cert "$2") + [ -n "$_tn_want" ] || return 1 + for _tn_k in "$1/server/tls/root-ca.key" "${CONFIG_PATH:-/etc/teknoir}/rsa_private.pem"; do + [ -f "$_tn_k" ] || continue + [ -r "$_tn_k" ] || { TN_CA_UNREADABLE=$_tn_k; continue; } + if [ "$(tn_ca_pubhash_key "$_tn_k")" = "$_tn_want" ]; then TN_CA_KEY=$_tn_k; return 0; fi + done + return 1 +} + +# tn_ca_state DATA_DIR +# Sets TN_CA_STATE and TN_CA_MSG; returns the `check` exit code. +# not-a-server no server/tls/server-ca.crt and no k3s server unit (an agent node, or nothing +# installed yet) +# no-server-data a k3s server unit exists but there is no server/tls/server-ca.crt in the data +# dir we resolved: k3s has not started yet, or its data dir is somewhere else. +# Never reported as ok (rc 4) +# ok the root has keyCertSign and is valid for more than 90 days +# needs-rotation initialised, root unusable, we hold the key +# needs-regeneration never initialised, PKI unusable, we hold the key +# foreign root unusable and we do not hold its key +# Also sets TN_CA_ROOT_USABLE: 0 when k3s refuses the root as it is (a multi-certificate bundle +# whose root has no keyCertSign, or an expired root), 1 otherwise (including "expires soon"). +tn_ca_state() { + TN_CA_STATE=; TN_CA_MSG=; TN_CA_ROOT_USABLE=1 + _tn_tls=$1/server/tls + if [ ! -f "$_tn_tls/server-ca.crt" ]; then + if tn_ca_unit_exists; then + TN_CA_STATE=no-server-data + TN_CA_MSG="the k3s server unit $(tn_ca_unit_name) exists, but $_tn_tls/server-ca.crt does not (data dir $1, taken from ${TN_CA_DATADIR_SRC:-the caller}): k3s has not started yet, or its data dir is elsewhere. Check 'systemctl cat $(tn_ca_unit_name)' and ${TN_K3S_CONFIG_DIR:-/etc/rancher/${PRODUCT:-k3s}}/config.yaml*, then pass --data-dir DIR" + return 4 + fi + TN_CA_STATE=not-a-server; TN_CA_MSG="no $_tn_tls/server-ca.crt and no k3s server unit: nothing to check"; return 0 + fi + _tn_sw=$(mktemp -d) || return 1 + if ! tn_ca_lastcert "$_tn_tls/server-ca.crt" "$_tn_sw/root.pem"; then + rm -rf "$_tn_sw"; TN_CA_STATE=error; TN_CA_MSG="cannot read the root certificate from $_tn_tls/server-ca.crt"; return 1 + fi + _tn_ncert=$(tn_ca_split "$_tn_tls/server-ca.crt" "$_tn_sw/b") + _tn_why= + TN_CA_ROOT_USABLE=1 # 0: k3s refuses this root now (no keyCertSign in a bundle, or expired) + if [ "$_tn_ncert" -gt 1 ] && ! tn_ca_has_certsign "$_tn_sw/root.pem"; then + _tn_why="the root certificate has no keyCertSign"; TN_CA_ROOT_USABLE=0 + fi + if ! "$OPENSSL" x509 -in "$_tn_sw/root.pem" -noout -checkend 0 >/dev/null 2>&1; then + TN_CA_ROOT_USABLE=0 + [ -n "$_tn_why" ] || _tn_why="the root certificate is expired" + elif [ -z "$_tn_why" ] && ! "$OPENSSL" x509 -in "$_tn_sw/root.pem" -noout -checkend 7776000 >/dev/null 2>&1; then + _tn_why="the root certificate expires within 90 days" + fi + if [ -z "$_tn_why" ]; then + rm -rf "$_tn_sw"; TN_CA_STATE=ok; TN_CA_MSG="the root certificate is acceptable to k3s"; return 0 + fi + if ! tn_ca_find_key "$1" "$_tn_sw/root.pem" && [ -n "$TN_CA_UNREADABLE" ]; then + rm -rf "$_tn_sw"; TN_CA_STATE=error; TN_CA_MSG="cannot read $TN_CA_UNREADABLE: run as root"; return 1 + fi + if tn_ca_find_key "$1" "$_tn_sw/root.pem"; then + if tn_ca_initialised "$1"; then + TN_CA_STATE=needs-rotation; TN_CA_MSG="$_tn_why: k3s v1.34.12/v1.35.9/v1.36.5/v1.37.1 and later will refuse to start; run: sh tn-k3s-ca.sh remediate" + rm -rf "$_tn_sw"; return 2 + fi + TN_CA_STATE=needs-regeneration; TN_CA_MSG="$_tn_why, and k3s never finished its first start here: the PKI can simply be regenerated; run: sh tn-k3s-ca.sh remediate" + rm -rf "$_tn_sw"; return 3 + fi + TN_CA_STATE=foreign; TN_CA_MSG="$_tn_why, but this is not the Teknoir device CA (no matching key in $_tn_tls/root-ca.key or ${CONFIG_PATH:-/etc/teknoir}/rsa_private.pem)" + rm -rf "$_tn_sw"; return 4 +} + +# tn_ca_suffix DATA_DIR: "." for the CN of new CAs +tn_ca_suffix() { + if [ -n "${_DEVICE_ID:-}" ] && [ -n "${_DOMAIN:-}" ]; then echo "${_DEVICE_ID}.${_DOMAIN}"; return 0; fi + _tn_f=$1/server/tls/intermediate-ca.pem + if [ -f "$_tn_f" ]; then + _tn_cn=$("$OPENSSL" x509 -in "$_tn_f" -noout -subject 2>/dev/null | sed -n 's/.*CN *= *//p' | sed 's/,.*//') + case $_tn_cn in *@*) echo "${_tn_cn#*@}"; return 0 ;; esac + fi + echo "$(hostname 2>/dev/null || echo device).local" +} + +# ----------------------------------------------------------------- setup (sourced by the bootstrap scripts) + +# tn_ca_pki_current DATA_DIR: 0 if the PKI in server/tls is complete, belongs to the current +# device key, has a usable root and chains from the leaf CAs to it. +tn_ca_pki_current() { + _tn_tls=$1/server/tls + for _tn_f in $TN_CA_BUNDLES root-ca.pem root-ca.key service.key intermediate-ca.pem intermediate-ca.key; do + [ -s "$_tn_tls/$_tn_f" ] || return 1 + done + tn_ca_state "$1" >/dev/null || return 1 + [ "$TN_CA_STATE" = ok ] || return 1 + [ "$(tn_ca_pubhash_cert "$_tn_tls/root-ca.pem")" = "$(tn_ca_pubhash_key "${CONFIG_PATH:-/etc/teknoir}/rsa_private.pem")" ] || return 1 + _tn_cw=$(mktemp -d) || return 1 + _tn_cok=0 + tn_ca_split "$_tn_tls/server-ca.crt" "$_tn_cw" >/dev/null + if "$OPENSSL" verify -CAfile "$_tn_tls/root-ca.pem" -untrusted "$_tn_tls/intermediate-ca.pem" "$_tn_cw/c0.pem" >/dev/null 2>&1; then _tn_cok=1; fi + rm -rf "$_tn_cw" + [ "$_tn_cok" = 1 ] +} + +# tn_ca_backup_tls DATA_DIR: tar server/tls (if it holds anything) to server/tls-backup-tn-.tar. +# The tars hold keys: the newest 3 are kept. +tn_ca_backup_tls() { + [ -d "$1/server/tls" ] || return 0 + _tn_bk="$1/server/tls-backup-tn-$(tn_ca_ts).tar" + ( umask 077; tar -C "$1/server" -cf "$_tn_bk" tls ) || { tn_ca_warn "cannot back up $1/server/tls"; return 1; } + chmod 600 "$_tn_bk" 2>/dev/null || true + for _tn_o in $(find "$1/server" -maxdepth 1 -type f -name 'tls-backup-tn-*.tar' 2>/dev/null | LC_ALL=C sort | + awk '{ a[NR] = $0 } END { for (i = 1; i <= NR - 3; i++) print a[i] }'); do rm -f "$_tn_o"; done + tn_ca_info "Previous CA files saved to $_tn_bk" +} + +# tn_ca_run_bounded SECONDS CMD...: run CMD, give up (return 124) after SECONDS. The k3s unit has +# TimeoutStartSec=0, so `systemctl start|restart` of a k3s that cannot start (the very case this tool +# repairs) never returns. Without `timeout` (very old coreutils) the command runs unbounded. +tn_ca_run_bounded() { + _tn_rb=$1; shift + if command -v timeout >/dev/null 2>&1; then timeout "$_tn_rb" "$@"; else "$@"; fi +} +tn_ca_svc_timeout() { echo "${TN_K3S_SERVICE_TIMEOUT:-600}"; } + +# tn_ca_unit_state: the ActiveState of the k3s unit (active, activating, ...), or nothing +tn_ca_unit_state() { + command -v systemctl >/dev/null 2>&1 || return 0 + systemctl is-active "$(tn_ca_unit_name)" 2>/dev/null || true +} + +# k3s around the replacement of server/tls. Stopped if the unit runs OR is starting (activating: a +# k3s that hangs in its first start, or restarts in a loop, is exactly the device that is repaired +# here, and it would rewrite files under us), never when there is no unit (a bootstrap of a new +# device has no k3s yet), and started again only if we stopped it. +tn_ca_k3s_active() { + if [ -n "${TN_K3S_STOP_CMD:-}" ]; then return 0; fi + case $(tn_ca_unit_state) in active|activating|reloading) return 0 ;; esac + return 1 +} +# tn_ca_k3s_ready: the unit is up and READY (Type=notify: k3s said so). Only that state. +tn_ca_k3s_ready() { [ "$(tn_ca_unit_state)" = active ]; } +tn_ca_k3s_stop() { + if [ -n "${TN_K3S_STOP_CMD:-}" ]; then tn_ca_run_bounded "$(tn_ca_svc_timeout)" sh -c "$TN_K3S_STOP_CMD" + else tn_ca_run_bounded "$(tn_ca_svc_timeout)" systemctl stop "$(tn_ca_unit_name)"; fi +} +tn_ca_k3s_start() { + if [ -n "${TN_K3S_START_CMD:-}" ]; then tn_ca_run_bounded "$(tn_ca_svc_timeout)" sh -c "$TN_K3S_START_CMD" + else tn_ca_run_bounded "$(tn_ca_svc_timeout)" systemctl start "$(tn_ca_unit_name)"; fi +} + +# tn_ca_guard_unknown_datadir DATA_DIR: 0 if it is fine to go on. A data dir that k3s never +# initialised (no server/token) while the k3s unit is up and READY means that k3s runs from a +# DIFFERENT data dir than the one we resolved (a --config file, a drop-in we do not read ...): +# generating a CA here would not touch the CA that k3s uses, and the installer that follows would +# upgrade a k3s whose CA was never checked. +tn_ca_guard_unknown_datadir() { + tn_ca_initialised "$1" && return 0 + tn_ca_k3s_ready || return 0 + tn_ca_warn "the k3s unit $(tn_ca_unit_name) is active and ready, but the data dir $1 (taken from ${TN_CA_DATADIR_SRC:-the caller}) was never initialised by k3s (no server/token): k3s runs from another data dir, so the CA that k3s uses was not looked at. Find it ('systemctl cat $(tn_ca_unit_name)', ${TN_K3S_CONFIG_DIR:-/etc/rancher/${PRODUCT:-k3s}}/config.yaml*, k3s server --config) and run again with DATA_DIR=/that/dir (or --data-dir). The CA and k3s were not changed" + return 1 +} + +# tn_ca_install_pki DATA_DIR SUFFIX: build a complete new PKI from the device certificate and +# replace server/tls with it. The data dir must not be initialised. Order: generate and validate in +# a work dir; stop k3s (if it runs: a hung start would rewrite files under us); back server/tls up; +# move the old dir aside (rename); check that the target is gone; move the new dir in; validate the +# INSTALLED result. Any failure puts the old dir back and starts k3s again if it was stopped. +tn_ca_install_pki() { + _tn_dd=$1 _tn_sfx=$2 + _tn_cfg=${CONFIG_PATH:-/etc/teknoir} + [ -s "$_tn_cfg/rsa_public.pem" ] && [ -s "$_tn_cfg/rsa_private.pem" ] || + { tn_ca_warn "device certificate or key missing in $_tn_cfg"; return 1; } + mkdir -p "$_tn_dd/server" || return 1 + _tn_new=$(mktemp -d "$_tn_dd/server/.tn-tls-new.XXXXXX") || return 1 + _tn_rt=$(mktemp -d) || { rm -rf "$_tn_new"; return 1; } + tn_ca_info "Re-issuing the device certificate as the k3s root CA (keyCertSign)" + if ! tn_ca_reissue_root "$_tn_cfg/rsa_public.pem" "$_tn_cfg/rsa_private.pem" "$_tn_rt/root.pem"; then + rm -rf "$_tn_new" "$_tn_rt"; return 1 + fi + mkdir -p "$_tn_new/etcd" + if ! tn_ca_generate_pki "$_tn_new" "$_tn_rt/root.pem" "$_tn_cfg/rsa_private.pem" "$_tn_sfx"; then + tn_ca_warn "PKI generation failed" + rm -rf "$_tn_new" "$_tn_rt"; return 1 + fi + if ! tn_ca_validate_pki "$_tn_new"; then + tn_ca_warn "the generated PKI is not usable: $_tn_dd/server/tls was not changed" + rm -rf "$_tn_new" "$_tn_rt"; return 1 + fi + rm -rf "$_tn_rt" + _tn_tlsd=$_tn_dd/server/tls + _tn_stopped=0 + if tn_ca_k3s_active; then + tn_ca_info "Stopping k3s while $_tn_tlsd is replaced" + if ! tn_ca_k3s_stop; then + tn_ca_warn "cannot stop k3s: $_tn_tlsd was not changed" + rm -rf "$_tn_new"; return 1 + fi + _tn_stopped=1 + fi + _tn_aside= + _tn_ok=1 + if [ -d "$_tn_tlsd" ]; then + tn_ca_backup_tls "$_tn_dd" || _tn_ok=0 + # keep an old service account key so tokens issued before stay valid + if [ "$_tn_ok" = 1 ] && [ -s "$_tn_tlsd/service.key" ]; then + cat "$_tn_tlsd/service.key" >> "$_tn_new/service.key" || _tn_ok=0 + fi + if [ "$_tn_ok" = 1 ]; then + _tn_aside=$_tn_dd/server/.tn-tls-old.$(tn_ca_ts).$$ + if ! mv "$_tn_tlsd" "$_tn_aside"; then _tn_aside=; _tn_ok=0; fi + fi + fi + if [ "$_tn_ok" = 1 ]; then + if [ -e "$_tn_tlsd" ] || [ -L "$_tn_tlsd" ]; then + tn_ca_warn "$_tn_tlsd is still there after it was moved aside: not replacing it"; _tn_ok=0 + elif ! mv "$_tn_new" "$_tn_tlsd"; then + tn_ca_warn "cannot move the new PKI to $_tn_tlsd"; _tn_ok=0 + else + chmod 750 "$_tn_tlsd" 2>/dev/null || true + tn_ca_validate_pki "$_tn_tlsd" || { tn_ca_warn "the installed PKI in $_tn_tlsd does not validate"; _tn_ok=0; } + fi + fi + if [ "$_tn_ok" != 1 ]; then + if [ -n "$_tn_aside" ]; then + rm -rf "$_tn_tlsd" + if mv "$_tn_aside" "$_tn_tlsd"; then tn_ca_warn "the previous $_tn_tlsd was restored" + else tn_ca_warn "CANNOT RESTORE the previous server/tls: it is in $_tn_aside (a backup tar is next to it)"; fi + fi + rm -rf "$_tn_new" + if [ "$_tn_stopped" = 1 ]; then + tn_ca_k3s_start || tn_ca_warn "k3s was stopped and could not be started again: start it yourself (systemctl start $(tn_ca_unit_name))" + fi + return 1 + fi + [ -z "$_tn_aside" ] || rm -rf "$_tn_aside" + tn_ca_changed "the k3s PKI in $_tn_tlsd was generated from the device certificate" + if [ "$_tn_stopped" = 1 ]; then + if tn_ca_k3s_start; then + tn_ca_changed "k3s was stopped and started again" + else + tn_ca_changed "k3s was stopped and could not be started again" + tn_ca_warn "the new CA is installed, but k3s could not be started: start it yourself (systemctl start $(tn_ca_unit_name))" + fi + fi + return 0 +} + +# tn_k3s_ca_setup: the bootstrap step. Uses CONFIG_PATH, DATA_DIR, PRODUCT, _DEVICE_ID, _DOMAIN. +# never started -> create the PKI below the re-issued root +# started, ok -> leave everything alone (re-running a bootstrap script is safe) +# started, old -> repair it first (TN_K3S_CA_AUTO_REMEDIATE=false: stop with advice) +# started, foreign (not the Teknoir device CA, we hold no key): left alone while k3s still +# accepts its root; otherwise STOP (exit 1): the k3s installer that follows +# would upgrade k3s and the device would no longer start +tn_k3s_ca_setup() { + tn_ca_info "Prepare K3s CA Certs" + export PRODUCT="${PRODUCT:-k3s}" + tn_ca_openssl || tn_ca_fatal "openssl not found or missing Elliptic Curve (ecparam) support." + tn_ca_info "Using ${OPENSSL}: $("$OPENSSL" version)" + tn_ca_resolve_datadir + tn_ca_info "k3s data dir: ${DATA_DIR} (${TN_CA_DATADIR_SRC})" + tn_ca_require_consistent_datadir + _tn_cfg=${CONFIG_PATH:-/etc/teknoir} + [ -s "$_tn_cfg/rsa_public.pem" ] && [ -s "$_tn_cfg/rsa_private.pem" ] || + tn_ca_fatal "device certificate or key missing in $_tn_cfg" + + if tn_ca_initialised "$DATA_DIR"; then + tn_ca_state "$DATA_DIR" || true + case $TN_CA_STATE in + ok|not-a-server) + [ "$TN_CA_STATE" != ok ] || tn_ca_clean_stages "$DATA_DIR" + tn_ca_info "k3s has already started on this device: the existing CA is left alone ($TN_CA_STATE: $TN_CA_MSG)" + return 0 ;; + foreign) + # not the Teknoir device CA: we cannot repair it. A root k3s refuses must stop the + # bootstrap, or the installer below would upgrade k3s and brick the device. + if [ "$TN_CA_ROOT_USABLE" != 1 ]; then + tn_ca_fatal "$TN_CA_MSG. The k3s CA of this device cannot be repaired here, and a k3s upgrade would stop it from starting: k3s must not be upgraded. The CA and k3s were not changed. Restore the matching CA key, or contact Teknoir support" + fi + tn_ca_warn "$TN_CA_MSG" + tn_ca_info "k3s has already started on this device: the existing CA is left alone (its root still has keyCertSign; it only expires soon, so replace this CA before it does)" + return 0 ;; + needs-rotation) + tn_ca_warn "$TN_CA_MSG" + if [ "${TN_K3S_CA_AUTO_REMEDIATE:-true}" != true ]; then + tn_ca_fatal "TN_K3S_CA_AUTO_REMEDIATE is not true: run 'sh tn-k3s-ca.sh remediate' and then this script again, or the k3s installer below will break the device" + fi + tn_ca_info "Repairing the CA of this device before k3s is installed or updated" + _tn_rr=0 + tn_ca_remediate all || _tn_rr=$? + case $_tn_rr in + 0) return 0 ;; + 5) tn_ca_fatal "the k3s CA of this device WAS changed (see above: the datastore of k3s holds the new CA), but a later step failed. Do not roll back and do not run the k3s installer yet: fix that step, make sure k3s is running, then run this script again" ;; + *) tn_ca_fatal "CA remediation failed; the device was not changed beyond the saved backup. See the messages above" ;; + esac ;; + *) tn_ca_fatal "unexpected CA state '$TN_CA_STATE': $TN_CA_MSG" ;; + esac + fi + + # not initialised here. If k3s is nevertheless up and ready, it runs from another data dir. + tn_ca_guard_unknown_datadir "$DATA_DIR" || tn_ca_fatal "cannot tell which data dir k3s uses (see above)" + if tn_ca_pki_current "$DATA_DIR"; then + tn_ca_clean_stages "$DATA_DIR" + tn_ca_info "A valid CA already exists in ${DATA_DIR}/server/tls (k3s has not started yet): keeping it" + return 0 + fi + tn_ca_install_pki "$DATA_DIR" "$(tn_ca_suffix "$DATA_DIR")" || tn_ca_fatal "Cannot generate the k3s CA" + return 0 +} + +# ----------------------------------------------------------------- remediation + +# A staged rotation lives in DATA_DIR/server/.tn-ca-rotate-/ (mode 700): +# server/tls/ the new bootstrap data for `k3s certificate rotate-ca --path` +# trust.pem the server-ca of the running server, trusted via SSL_CERT_FILE for the call +# ROTATE_CMD the exact command; touches ROTATED when it succeeds +# `finalize` copies the new files into server/tls once ROTATED exists, and removes the dir. + +tn_ca_newest_stage() { # DATA_DIR -> prints the newest staging dir, or nothing + find "$1/server" -maxdepth 1 -type d -name '.tn-ca-rotate-*' 2>/dev/null | LC_ALL=C sort | tail -n 1 +} + +# tn_ca_clean_stages DATA_DIR: remove every staging dir of a rotation (.tn-ca-rotate-*) and every work +# dir of a regeneration that was interrupted (.tn-tls-old.* and .tn-tls-new.*: the previous server/tls +# that was moved aside, and the new one before it was moved in). Only when the CA is fine: they all +# hold copies of the CA keys, and a crashed run leaves them behind. +tn_ca_clean_stages() { + find "$1/server" -maxdepth 1 -type d \( -name '.tn-ca-rotate-*' -o -name '.tn-tls-old.*' -o -name '.tn-tls-new.*' \) 2>/dev/null | + while read -r _tn_cs; do + rm -rf "$_tn_cs" && tn_ca_info "Removed the leftover staging dir $_tn_cs (it held copies of the CA keys)" + done + return 0 +} + +tn_ca_k3s_bin() { echo "${K3S_BIN:-k3s}"; } + +# tn_ca_rotate_supported: 0 if the k3s binary has `certificate rotate-ca` or its version cannot be +# told, 1 if it is too old (added in v1.24.12, v1.25.8, v1.26.3; every v1.27+ has it). Sets +# TN_CA_K3S_VERSION. +tn_ca_rotate_supported() { + TN_CA_K3S_VERSION= + _tn_vo=$($(tn_ca_k3s_bin) --version 2>/dev/null | sed -n '1s/^.*version v\{0,1\}\([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*$/\1 \2 \3/p') || return 0 + [ -n "$_tn_vo" ] || return 0 + IFS=' ' read -r _tn_vma _tn_vmi _tn_vpa </dev/null || true + ( umask 077; mkdir -p "$_tn_s/server/tls/etcd" "$_tn_s/w" ) || return 1 + tn_ca_split "$_tn_tls/server-ca.crt" "$_tn_s/w" >/dev/null || { rm -rf "$_tn_s"; return 1; } + tn_ca_lastcert "$_tn_tls/server-ca.crt" "$_tn_s/oldroot.pem" || { rm -rf "$_tn_s"; return 1; } + tn_ca_find_key "$_tn_dd" "$_tn_s/oldroot.pem" || { rm -rf "$_tn_s"; return 1; } + _tn_key=$TN_CA_KEY + tn_ca_reissue_root "$_tn_s/oldroot.pem" "$_tn_key" "$_tn_s/newroot.pem" || { rm -rf "$_tn_s"; return 1; } + _tn_oldfp=$(tn_ca_fp "$_tn_s/oldroot.pem") + + # a copy of everything k3s keeps there (keys, service.key, bundles) + ( cd "$_tn_tls" && tar -cf - . ) | ( cd "$_tn_s/server/tls" && tar -xf - ) || { rm -rf "$_tn_s"; return 1; } + rm -rf "$_tn_s/server/tls/.ca" + # every bundle: the old root replaced by the new one, all other certificates kept. This function + # runs in an `||` context, where `set -e` does not exist: EVERY write is checked, and each rebuilt + # bundle is verified before anything is handed to rotate-ca. + _tn_newfp=$(tn_ca_fp "$_tn_s/newroot.pem") || { rm -rf "$_tn_s"; return 1; } + _tn_rbok=1 + for _tn_b in $TN_CA_BUNDLES; do + [ -f "$_tn_tls/$_tn_b" ] || continue + rm -rf "$_tn_s/b" "$_tn_s/v" + mkdir -p "$_tn_s/b" "$_tn_s/v" || { _tn_rbok=0; break; } + _tn_n=$(tn_ca_split "$_tn_tls/$_tn_b" "$_tn_s/b") || { _tn_rbok=0; break; } + if [ "${_tn_n:-0}" -lt 1 ]; then tn_ca_warn "$_tn_b holds no certificate"; _tn_rbok=0; break; fi + : > "$_tn_s/server/tls/$_tn_b" || { _tn_rbok=0; break; } + _tn_i=0 + while [ "$_tn_i" -lt "$_tn_n" ]; do + if [ "$(tn_ca_fp "$_tn_s/b/c$_tn_i.pem")" = "$_tn_oldfp" ]; then + cat "$_tn_s/newroot.pem" >> "$_tn_s/server/tls/$_tn_b" || { _tn_rbok=0; break; } + else + cat "$_tn_s/b/c$_tn_i.pem" >> "$_tn_s/server/tls/$_tn_b" || { _tn_rbok=0; break; } + fi + _tn_i=$((_tn_i + 1)) + done + [ "$_tn_rbok" = 1 ] || break + # the rebuilt bundle: as many certificates as before, and the new root last + _tn_m=$(tn_ca_split "$_tn_s/server/tls/$_tn_b" "$_tn_s/v") || _tn_m= + if [ "${_tn_m:-x}" != "$_tn_n" ]; then + tn_ca_warn "the rebuilt $_tn_b holds ${_tn_m:-no} certificates, the original $_tn_n"; _tn_rbok=0; break + fi + if [ "$(tn_ca_fp "$_tn_s/v/c$((_tn_n - 1)).pem")" != "$_tn_newfp" ]; then + tn_ca_warn "the last certificate of the rebuilt $_tn_b is not the new root (unexpected layout of the original bundle)" + _tn_rbok=0; break + fi + done + if [ "$_tn_rbok" != 1 ]; then + tn_ca_warn "cannot rebuild the CA bundles (disk full? unexpected layout?): the CA and k3s were not changed" + rm -rf "$_tn_s"; return 1 + fi + rm -rf "$_tn_s/b" "$_tn_s/v" + cp "$_tn_s/newroot.pem" "$_tn_s/server/tls/root-ca.pem" || { rm -rf "$_tn_s"; return 1; } + + # every leaf CA must verify through the new root + for _tn_b in server-ca.crt client-ca.crt request-header-ca.crt etcd/peer-ca.crt etcd/server-ca.crt; do + [ -f "$_tn_s/server/tls/$_tn_b" ] || continue + rm -rf "$_tn_s/b"; mkdir -p "$_tn_s/b" + tn_ca_split "$_tn_s/server/tls/$_tn_b" "$_tn_s/b" >/dev/null + if ! "$OPENSSL" verify -CAfile "$_tn_s/newroot.pem" -untrusted "$_tn_s/server/tls/intermediate-ca.crt" "$_tn_s/b/c0.pem" >/dev/null 2>&1; then + tn_ca_warn "the new root does not verify $_tn_b: the CA and k3s were not changed" + rm -rf "$_tn_s"; return 1 + fi + done + rm -rf "$_tn_s/b" + + # k3s only accepts the new bundle if the OLD root can still vouch for the chain (it verifies the + # new leaf CAs with the old chain). An old root that is expired or not yet valid, that has no + # basicConstraints or CA:FALSE, or that has a keyUsage without keyCertSign (the device-controller + # certificates) cannot (Go refuses such a parent), however good the new bundle is. Verified above + # with openssl; then k3s's own check is skipped (--force). The explicit tests are followed by a + # general one: if openssl cannot verify the OLD chain against the old root, but the new chain + # verifies (done above), the old root is no voucher either. + _tn_force=; _tn_fwhy= + _tn_otxt=$("$OPENSSL" x509 -in "$_tn_s/oldroot.pem" -noout -text 2>/dev/null) + _tn_obc=$(printf '%s\n' "$_tn_otxt" | awk '/X509v3 Basic Constraints/ { getline; print; exit }') + if ! "$OPENSSL" x509 -in "$_tn_s/oldroot.pem" -noout -checkend 0 >/dev/null 2>&1; then + _tn_fwhy="the old root has expired" + elif [ -z "$_tn_obc" ]; then + _tn_fwhy="the old root has no basicConstraints" + elif printf '%s\n' "$_tn_obc" | grep -q 'CA:FALSE'; then + _tn_fwhy="the old root has CA:FALSE" + elif printf '%s\n' "$_tn_otxt" | grep -q 'X509v3 Key Usage' && ! tn_ca_has_certsign "$_tn_s/oldroot.pem"; then + _tn_fwhy="the old root has a key usage without keyCertSign" + else + _tn_vout=$("$OPENSSL" verify -CAfile "$_tn_s/oldroot.pem" -untrusted "$_tn_tls/intermediate-ca.crt" "$_tn_s/w/c0.pem" 2>&1) || + _tn_fwhy="openssl cannot verify the old chain against the old root ($(printf '%s\n' "$_tn_vout" | grep -i 'error' | head -n 1 | sed 's/^error [0-9]* at [0-9]* depth lookup: *//'))" + fi + if [ -n "$_tn_fwhy" ]; then + _tn_force=' --force' + tn_ca_info "rotate-ca will use --force ($_tn_fwhy, so k3s cannot verify the new CA bundle against it); the new bundle was verified here" + fi + tn_ca_backup_tls "$_tn_dd" || { rm -rf "$_tn_s"; return 1; } + cp "$_tn_s/w/c0.pem" "$_tn_s/trust.pem" || { rm -rf "$_tn_s"; return 1; } + rm -rf "$_tn_s/w" "$_tn_s/oldroot.pem" + # SSL_CERT_FILE: a post-fix k3s that refuses the old root still has to talk to its own + # supervisor. Trusting the running server's own leaf CA for this one call allows that. + printf '%s\n' "SSL_CERT_FILE='$_tn_s/trust.pem' $(tn_ca_k3s_bin) certificate rotate-ca --data-dir '$_tn_dd' --path '$_tn_s/server'$_tn_force && touch '$_tn_s/ROTATED'" > "$_tn_s/ROTATE_CMD" || + { rm -rf "$_tn_s"; return 1; } + tn_ca_info "Staged $_tn_s" + tn_ca_info "Next: run its ROTATE_CMD against the running k3s server (remediate --phase rotate), then --phase finalize and restart k3s" +} + +# rotate: run ROTATE_CMD of the newest staging +tn_ca_rotate() { + _tn_s=$(tn_ca_newest_stage "$1") + if [ -z "$_tn_s" ]; then + tn_ca_state "$1" >/dev/null || true + case $TN_CA_STATE in needs-regeneration|ok|not-a-server) tn_ca_info "nothing to rotate"; return 0 ;; esac + tn_ca_warn "no staged rotation in $1/server: run 'remediate --phase prepare' first"; return 1 + fi + [ -f "$_tn_s/ROTATE_CMD" ] || { tn_ca_warn "$_tn_s has no ROTATE_CMD"; return 1; } + if ! tn_ca_rotate_supported; then tn_ca_warn_rotate_unsupported; return 1; fi + tn_ca_info "Rotating the CA in the running k3s server (rotate-ca)" + # rotate-ca talks to the k3s server. One that is starting right now (a unit that was just restarted, or + # one that restarts itself in a loop, like a k3s that a cgroup v1 host made exit) does not answer yet: + # try again for a while, but only when systemd says that the unit is there and starting or running + _tn_tries=1 + case $(tn_ca_unit_state) in active|activating|reloading) _tn_tries=${TN_K3S_ROTATE_TRIES:-6} ;; esac + _tn_try=1 + while ! sh -c "$(cat "$_tn_s/ROTATE_CMD")"; do + if [ "$_tn_try" -ge "$_tn_tries" ]; then + tn_ca_warn "k3s certificate rotate-ca failed. Is the k3s server running (systemctl status k3s)? A server that refuses the old root is fine, it only has to be up." + tn_ca_warn "The CA of k3s and its files in $1/server/tls were not changed. To go back to a working state: stop k3s and install the previous k3s patch release (v1.34.11, v1.35.8, v1.36.4 or v1.37.0), then run remediate again." + return 1 + fi + tn_ca_info "rotate-ca failed (attempt $_tn_try of $_tn_tries): the k3s server may still be starting; trying again in ${TN_K3S_ROTATE_WAIT:-10}s" + sleep "${TN_K3S_ROTATE_WAIT:-10}" + _tn_try=$((_tn_try + 1)) + done + tn_ca_changed "the CA was rotated in the datastore of k3s (k3s certificate rotate-ca)" +} + +# finalize: put the rotated files on disk (k3s keeps them in step with its datastore) +tn_ca_finalize() { + _tn_dd=$1 + _tn_s=$(tn_ca_newest_stage "$_tn_dd") + if [ -z "$_tn_s" ]; then tn_ca_info "nothing to finalize"; return 0; fi + [ -f "$_tn_s/ROTATED" ] || { tn_ca_warn "the rotation in $_tn_s has not completed (no ROTATED marker): not touching $_tn_dd/server/tls"; return 1; } + for _tn_f in $TN_CA_BUNDLES root-ca.pem; do + [ -f "$_tn_s/server/tls/$_tn_f" ] || continue + cp "$_tn_s/server/tls/$_tn_f" "$_tn_dd/server/tls/$_tn_f.tn-new" && mv "$_tn_dd/server/tls/$_tn_f.tn-new" "$_tn_dd/server/tls/$_tn_f" || return 1 + done + rm -rf "$_tn_s" + tn_ca_changed "the CA files in $_tn_dd/server/tls were replaced" + tn_ca_info "CA files updated in $_tn_dd/server/tls" +} + +# tn_ca_restart: restart k3s, but not forever (TN_K3S_SERVICE_TIMEOUT, 600 s): the unit waits for a +# k3s that may never become ready. `systemctl restart` that is given up on leaves the unit starting, +# so the unit is watched for TN_K3S_READY_GRACE (120) more seconds: a slow device that gets there is a +# success. Otherwise a timeout is a failure (return 124): the caller reports that the CA WAS changed +# and k3s did not come back in time. +tn_ca_restart() { + _tn_rt=$(tn_ca_svc_timeout) + _tn_rrc=0 _tn_watched="" + _tn_gw=0 + if [ -n "${TN_K3S_RESTART_CMD:-}" ]; then + tn_ca_info "Restarting k3s: $TN_K3S_RESTART_CMD" + tn_ca_run_bounded "$_tn_rt" sh -c "$TN_K3S_RESTART_CMD" || _tn_rrc=$? + elif command -v systemctl >/dev/null 2>&1 && systemctl cat "$(tn_ca_unit_name)" >/dev/null 2>&1; then + tn_ca_info "Restarting k3s" + tn_ca_run_bounded "$_tn_rt" systemctl restart "$(tn_ca_unit_name)" || _tn_rrc=$? + else + tn_ca_warn "restart k3s yourself now (no systemd unit found and TN_K3S_RESTART_CMD is not set)" + return 0 + fi + if [ "$_tn_rrc" -eq 124 ] && [ -z "${TN_K3S_RESTART_CMD:-}" ]; then + _tn_gr=${TN_K3S_READY_GRACE:-120} _tn_gp=${TN_K3S_READY_POLL:-5} _tn_watched=1 + tn_ca_info "the restart of k3s did not finish within ${_tn_rt}s: watching the unit for up to ${_tn_gr}s more" + while :; do + if tn_ca_k3s_ready; then + tn_ca_info "k3s is active now (${_tn_gw}s after the restart gave up)" + _tn_rrc=0 + break + fi + [ "$_tn_gw" -lt "$_tn_gr" ] || break + sleep "$_tn_gp" + _tn_gw=$((_tn_gw + _tn_gp)) + done + fi + if [ "$_tn_rrc" -eq 124 ]; then + if [ -n "${_tn_watched:-}" ]; then + tn_ca_warn "the restart of k3s did not finish within ${_tn_rt}s (TN_K3S_SERVICE_TIMEOUT), and the unit was not active ${_tn_gw}s later either: it is still starting, or it cannot start" + else + tn_ca_warn "the restart of k3s did not finish within ${_tn_rt}s (TN_K3S_SERVICE_TIMEOUT): it is still starting, or it cannot start" + fi + elif [ "$_tn_rrc" -eq 0 ]; then + tn_ca_changed "k3s was restarted" + fi + return "$_tn_rrc" +} + +# After this point the CA has been changed (rotated in the datastore, or the PKI regenerated): +# a failure must not claim that the device is unchanged. Return code 5. +tn_ca_warn_changed_finalize() { + tn_ca_warn "k3s certificate rotate-ca SUCCEEDED: the datastore of k3s already holds the new CA. Only putting the new files into $1/server/tls failed (see above: disk space? permissions?). Do not roll back. Fix that, then run: sh tn-k3s-ca.sh remediate --phase finalize, and restart k3s." +} +tn_ca_warn_changed_restart() { # DATA_DIR WAS + if [ "$2" = needs-rotation ]; then + tn_ca_warn "k3s certificate rotate-ca SUCCEEDED: the datastore of k3s holds the new CA, and so does $1/server/tls, but k3s could not be restarted." + else + tn_ca_warn "The PKI in $1/server/tls was regenerated (the previous one is in $1/server/tls-backup-tn-*.tar), but k3s could not be restarted." + fi + tn_ca_warn "Look at: systemctl status $(tn_ca_unit_name); journalctl -u $(tn_ca_unit_name) -n 50. Then restart it: systemctl restart $(tn_ca_unit_name). Do not roll back files: restoring the backup tar does not undo a rotation (see 'Rollback limits' at the top of this file)." +} + +# tn_ca_remediate PHASE (prepare | rotate | finalize | all) +tn_ca_remediate() { + tn_ca_openssl || return 1 + tn_ca_resolve_datadir + _tn_dd=$DATA_DIR + case $1 in + prepare) tn_ca_prepare "$_tn_dd" ;; + rotate) tn_ca_rotate "$_tn_dd" ;; + finalize) tn_ca_finalize "$_tn_dd" ;; + all) + tn_ca_state "$_tn_dd" >/dev/null || true + case $TN_CA_STATE in + ok) tn_ca_clean_stages "$_tn_dd"; tn_ca_info "$TN_CA_MSG"; return 0 ;; + not-a-server) tn_ca_info "$TN_CA_MSG"; return 0 ;; + esac + _tn_was=$TN_CA_STATE + _tn_s=$(tn_ca_newest_stage "$_tn_dd") + if [ "$_tn_was" = needs-rotation ] && [ -n "$_tn_s" ] && [ -f "$_tn_s/ROTATED" ]; then + # an earlier run rotated the CA in the datastore and stopped before the files were copied: + # do not stage and rotate again + tn_ca_info "A completed rotation is waiting in $_tn_s: finalizing it" + tn_ca_finalize "$_tn_dd" || { tn_ca_warn_changed_finalize "$_tn_dd"; return 5; } + else + tn_ca_prepare "$_tn_dd" || return $? + if [ "$_tn_was" = needs-rotation ]; then + tn_ca_rotate "$_tn_dd" || return 1 + tn_ca_finalize "$_tn_dd" || { tn_ca_warn_changed_finalize "$_tn_dd"; return 5; } + fi + fi + tn_ca_restart || { tn_ca_warn_changed_restart "$_tn_dd" "$_tn_was"; return 5; } + ;; + *) tn_ca_warn "unknown phase '$1' (prepare, rotate, finalize or all)"; return 2 ;; + esac +} + +# tn_ca_check: print the state, exit code as documented at the top +tn_ca_check() { + tn_ca_openssl || return 1 + tn_ca_resolve_datadir + echo "tn-k3s-ca: data dir ${DATA_DIR} (${TN_CA_DATADIR_SRC})" >&2 + _tn_rc=0 + tn_ca_state "$DATA_DIR" || _tn_rc=$? + echo "tn-k3s-ca: ${TN_CA_STATE}: ${TN_CA_MSG}" + return "$_tn_rc" +} + +# ----------------------------------------------------------------- program mode + +tn_ca_usage() { + cat <<'EOF' +usage: sh tn-k3s-ca.sh check [--data-dir DIR] [--config-dir DIR] + sh tn-k3s-ca.sh remediate [--phase prepare|rotate|finalize|all] [--data-dir DIR] [--config-dir DIR] + +check Is the k3s root CA of this device acceptable to k3s v1.34.12/v1.35.9/v1.36.5/v1.37.1+? + exit 0 ok, 2 needs rotation, 3 needs regeneration, 4 foreign CA or no server data found, + 1 error. Without --data-dir the data dir is resolved the way k3s does (ExecStart of the + k3s unit, config.yaml.d/*.yaml, config.yaml, default /var/lib/rancher/k3s) +remediate Re-issue the root with keyCertSign (same key and subject) and put it in place. + Phases: prepare (offline, stages the new files), rotate (k3s certificate rotate-ca against + the running server), finalize (copy the files), all (those three plus a k3s restart). + A device whose k3s never finished its first start gets its PKI regenerated instead. +Run as root on the device. Environment: K3S_BIN, TN_K3S_RESTART_CMD, TN_K3S_STOP_CMD, +TN_K3S_START_CMD, TN_K3S_SERVICE_TIMEOUT (seconds for a stop/start/restart of k3s, 600), CONFIG_PATH, DATA_DIR. +EOF +} + +tn_ca_main() { + _tn_cmd=$1; shift + _tn_phase=all + while [ $# -gt 0 ]; do + case $1 in + --phase|--data-dir|--config-dir) + if [ $# -lt 2 ] || [ -z "$2" ]; then tn_ca_warn "$1 needs a value"; tn_ca_usage >&2; return 2; fi + case $1 in + --phase) _tn_phase=$2 ;; + --data-dir) DATA_DIR=$2 ;; + --config-dir) CONFIG_PATH=$2 ;; + esac + shift 2 ;; + -h|--help) tn_ca_usage; return 0 ;; + *) tn_ca_usage >&2; return 2 ;; + esac + done + [ -z "${CONFIG_PATH:-}" ] || export CONFIG_PATH + [ -z "${DATA_DIR:-}" ] || export DATA_DIR + case $_tn_cmd in + check) tn_ca_check ;; + remediate) + [ "$(id -u)" -eq 0 ] || { tn_ca_warn "run as root"; return 1; } + tn_ca_remediate "$_tn_phase" ;; + esac +} + +# Only the command words start the program: sourcing this file defines functions and nothing +# else. (A bootstrap script passes no arguments of its own; the template that embeds this file +# sets TN_K3S_CA_NO_MAIN anyway.) +case "${1:-}" in + check|remediate) + if [ -z "${TN_K3S_CA_NO_MAIN:-}" ]; then + CONFIG_PATH=${CONFIG_PATH:-/etc/teknoir} + tn_ca_main "$@" + exit $? + fi + ;; +esac + +# The k3s release policy (a re-run keeps the installed k3s) and the cgroup v1 kubelet flag. +#!/bin/sh +# tn-k3s-install.sh - what a Teknoir bootstrap script decides before it runs the k3s installer: +# which k3s release (a re-run keeps what is installed), and whether the kubelet must be told that +# cgroup v1 is acceptable. Used by the server and the agent script (sourced), next to +# tn-k3s-ca.sh (the server only). +# +# . tn-k3s-install.sh; tn_inst_prepare server|agent (once, right before the installer) +# tn_inst_finish (once, right after the installer) +# tn_inst_config_ensure FILE KEY TEXT (config.yaml without clobbering it) +# +# POSIX sh. It runs under `set -e` and `set -o noglob` (as the bootstrap scripts do) and does not +# change shell options, traps or the working directory of the caller. Sourcing defines functions only. +# +# ---------------------------------------------------------------- version policy +# A bootstrap script is meant to be re-run (it repairs the CA, refreshes the manifests). It must not +# upgrade k3s as a side effect: the pin in the script (INSTALL_K3S_VERSION, `tnctl bootstrap +# --k3s-version`) is the release for a NEW device. +# - k3s is not installed: the pin is installed (a SERVER; an agent: see below). +# - the version cannot be read although a k3s binary or DATA_DIR/server/db exists: REFUSED (the +# installer would install or replace k3s blind), unless TN_K3S_FORCE=true. +# - k3s is installed: its version is KEPT. The exact installed release is handed to the +# installer (which then only rewrites the unit and the symlinks), and +# a different pin is reported and ignored. +# - TN_K3S_UPGRADE=true (tnctl bootstrap --k3s-upgrade) lets the pin replace the installed +# release. A pin below the installed release is refused (k3s does not +# downgrade: the datastore has been migrated), and so is a jump of more +# than one minor version (v1.34 -> v1.36) unless TN_K3S_FORCE=true +# (tnctl bootstrap --force): Kubernetes supports one minor at a time. +# A channel is checked too: a minor (v1.35) by its major.minor against the +# installed one (lower or other major: refused; more than +1: needs force); +# stable/latest/testing is resolved at update.k3s.io like the installer does +# and then checked as an exact release. A channel that cannot be resolved +# is refused unless TN_K3S_FORCE=true (then it goes on unchecked). +# +# An AGENT follows its server. A kubelet must never be newer than the apiserver it joins, so a new +# agent (and an agent that TN_K3S_UPGRADE=true replaces) does not install the pin of the script: it +# reads the release of the server and installs exactly that one. Where the release comes from: k3s +# does NOT answer /version to an anonymous caller (its apiserver runs with anonymous-auth=false: 401, +# verified with v1.30, v1.34 and v1.36), and the join token is only good for the supervisor +# endpoints. So the Teknoir SERVER script publishes the release it runs (tn_inst_publish_release) as +# /server/static/teknoir/k3s-version, which k3s serves without authentication at +# https:///static/teknoir/k3s-version +# and the agent reads that file (then, for a server that allows anonymous access, /version). The +# pin only counts when the person who runs the script asked for it (`tnctl bootstrap --k3s-version`, or +# INSTALL_K3S_VERSION in the environment; TN_K3S_PIN_SOURCE=flag|env, the template sets it): then it is +# installed if it is not newer than the server (older gets a warning), and refused if it is. When the +# server's release cannot be read (down, unreachable, bootstrapped by a script that does not publish it +# yet): an explicit exact pin is installed with a warning, and otherwise the script REFUSES, unless +# TN_K3S_FORCE=true. An OS image build (OS_BUILD) cannot reach the server and installs the pin. The +# cgroup logic below is keyed on the release that is installed. +# +# ---------------------------------------------------------------- cgroup v1 +# Kubernetes >= 1.35: the kubelet refuses to start on a cgroup v1 host (failCgroupV1 defaults to +# true) and k3s does not change that. Jetson L4T r32/r35 and Ubuntu 18.04/20.04 are cgroup v1. When +# the host is on cgroup v1 (stat -fc %T /sys/fs/cgroup is not cgroup2fs), or cannot be told (an OS +# build runs elsewhere than the device), and the k3s that will run is >= v1.31 (the kubelet knows +# the flag since 1.31; it only matters from 1.35, but a later upgrade, by hand or by the k3s +# installer alone, must not stop the kubelet), the script writes (atomically, only when it differs) +# /etc/rancher/k3s/config.yaml.d/99-teknoir-cgroupv1.yaml: kubelet-arg+: ["fail-cgroupv1=false"] +# `kubelet-arg+` APPENDS to the kubelet-arg of config.yaml and the other files (verified with +# rancher/k3s v1.36.5, a kubelet-arg already in config.yaml is kept); the flag is harmless on a +# cgroup v2 host. A fail-cgroupv1 that the device sets itself is respected. cgroup v1 is +# deprecated upstream: move the device to cgroup v2 (kernel command line +# systemd.unified_cgroup_hierarchy=1, or a newer OS release). +# +# Order of effects. Everything before the installer (tn_inst_prepare, the CA step, the download) can +# be interrupted by a restart of k3s (the CA step restarts it) or by a reboot of the device, and the +# running k3s then reads the configuration again. So tn_inst_prepare only makes changes that are safe +# for the k3s that is running NOW, and never takes a protection away before its replacement is in place: +# - the drop-in is REMOVED early only when the running k3s does not need it: the host is on cgroup v2; +# the release that will run is older than v1.31 (an older kubelet fails on the unknown flag); or the +# device sets fail-cgroupv1 in a PERSISTENT config file that k3s already reads (config.yaml or another +# config.yaml.d/*.yaml). A fail-cgroupv1 only in INSTALL_K3S_EXEC reaches the unit when the installer +# has run, so until then the drop-in stays (or is written): the same value twice is harmless, and +# the command line wins when they differ. +# - the drop-in is WRITTEN early, except when the k3s that is installed now is older than v1.31 (an +# upgrade): its kubelet would fail on the flag if it restarted before the installer replaced it. +# That write waits for tn_inst_finish, after the installer. An existing drop-in is removed at once in +# that case, because the old kubelet cannot take it. +# - but the installer starts the NEW release as soon as it has rewritten the unit, and a release >= v1.35 +# refuses to start on cgroup v1 without the flag (install.sh's `systemctl restart` fails, and under +# `set -e` the script stops before tn_inst_finish: a k3s that crash-loops). So when the write is held +# back and the release the installer will start is not KNOWN to be older than v1.35 (a channel such as +# stable or latest, or an exact v1.35 or newer, also with TN_K3S_FORCE=true), the flag is appended to +# INSTALL_K3S_EXEC (--kubelet-arg=fail-cgroupv1=false, once, and exported): it is then in the NEW unit +# from its first start and the old, running kubelet never sees it. An exact v1.31 - v1.34 does not need +# it on its first start (it accepts it, it only ignores it on cgroup v1), so it gets none. tn_inst_finish +# still writes the drop-in afterwards, which is what keeps later runs and upgrades safe. +# +# What counts as "the device sets fail-cgroupv1 itself": an uncommented line that sets it to false in +# config.yaml, config.yaml.d/*.yaml or *.yml (not our own file), or in INSTALL_K3S_EXEC. A comment does +# not count, and neither does fail-cgroupv1=true: then the drop-in is written (and kept) as usual, and +# it wins, because kubelet-arg+ comes after. A device that sets fail-cgroupv1=true on a cgroup v1 host +# on purpose is not supported (the kubelet of v1.35 and newer cannot start there anyway); delete the +# drop-in and set cgroup v2 instead. +# +# Environment (all optional): +# INSTALL_K3S_VERSION, INSTALL_K3S_CHANNEL the pin of the script (read, and set to what the installer must get) +# TN_K3S_UPGRADE true: the pin may replace an installed k3s +# TN_K3S_FORCE true: allow an upgrade across more than one minor version; an agent newer than its +# server, or without knowing the version of its server; a k3s whose version cannot be read +# TN_K3S_PIN_SOURCE who chose INSTALL_K3S_VERSION: flag / env (the person who runs the script) or +# default (the template); the template sets it, an agent follows the server unless it is flag/env +# K3S_URL the server of an agent (https://host:6443) +# TN_K3S_SERVER_VERSION_TRIES, TN_K3S_SERVER_VERSION_WAIT tries (3) and seconds between them (3) to reach the +# server; a server that answers without a release is not asked again +# DATA_DIR the k3s data dir (/var/lib/rancher/k3s), where a datastore says that k3s is there +# TN_K3S_CGROUPV1 true / false: do not detect, take this as the answer +# TN_CGROUP_FSTYPE the file system type of /sys/fs/cgroup, instead of `stat` (tests) +# K3S_BIN the k3s command used to read the installed version (k3s, then /usr/local/bin/k3s) +# TN_K3S_CONFIG_DIR /etc/rancher/k3s +# OS_BUILD set by the OS image builds: the host is not the device, cgroup cannot be detected + +TN_K3S_INSTALL_LIB_VERSION=3 +export TN_K3S_INSTALL_LIB_VERSION + +# ----------------------------------------------------------------- messages +# Own functions, in the format of header.sh.tmpl: a caller's info/warn/fatal may not exist, and `info` +# can be the GNU info reader. +tn_inst_info() { echo '[INFO] ' "$@"; } +tn_inst_warn() { echo '[WARN] ' "$@" >&2; } +tn_inst_fatal() { echo '[ERROR] ' "$@" >&2; exit 1; } + +# tn_inst_state_note: how a refusal describes the device. This library has changed nothing when it +# refuses; the CA step (tn-k3s-ca.sh) may have, earlier in the same run, and then says exactly what. +tn_inst_state_note() { + if [ -n "${TN_K3S_CA_CHANGED:-}" ]; then + printf 'The k3s release was not changed, but earlier in this run: %s' "$TN_K3S_CA_CHANGED" + else + printf 'Nothing was changed' + fi +} + +# ----------------------------------------------------------------- versions + +# tn_inst_parse VERSION: sets TN_INST_MAJ TN_INST_MIN TN_INST_PAT TN_INST_REV from vMAJ.MIN.PAT[+k3sREV] +# (REV 0 when there is none). Fails for anything else (stable, latest, a commit). +tn_inst_parse() { + TN_INST_MAJ=; TN_INST_MIN=; TN_INST_PAT=; TN_INST_REV= + _tn_pv=$(printf '%s\n' "$1" | sed -n 's/^v\([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\)\(+k3s\([0-9][0-9]*\)\)\{0,1\}$/\1 \2 \3 \5/p') + [ -n "$_tn_pv" ] || return 1 + IFS=' ' read -r TN_INST_MAJ TN_INST_MIN TN_INST_PAT TN_INST_REV < $i + 0) { print 1; exit } } print 0 }') +} + +# the k3s command that reads the installed version +tn_inst_k3s_cmd() { + if [ -n "${K3S_BIN:-}" ]; then echo "$K3S_BIN" + elif command -v k3s >/dev/null 2>&1; then echo k3s + elif [ -x /usr/local/bin/k3s ]; then echo /usr/local/bin/k3s + else echo k3s; fi +} + +# tn_inst_installed_version: sets TN_INST_INSTALLED to the release of the installed k3s +# (v1.34.11+k3s1) or to nothing when there is none. +tn_inst_installed_version() { + TN_INST_INSTALLED=$($(tn_inst_k3s_cmd) --version 2>/dev/null | sed -n '1s/^.*version \(v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\(+k3s[0-9][0-9]*\)\{0,1\}\).*$/\1/p') || TN_INST_INSTALLED= +} + +# tn_inst_k3s_present: 0 if there is a k3s binary or a server datastore, whatever its version is. +# Sets TN_INST_PRESENT to what was found. +tn_inst_k3s_present() { + TN_INST_PRESENT= + _tn_first= + for _tn_w in $(tn_inst_k3s_cmd); do _tn_first=$_tn_w; break; done + if [ -n "$_tn_first" ] && _tn_found=$(command -v "$_tn_first" 2>/dev/null) && [ -n "$_tn_found" ]; then + TN_INST_PRESENT="the k3s command $_tn_found"; return 0 + fi + if [ -d "${DATA_DIR:-/var/lib/rancher/k3s}/server/db" ]; then + TN_INST_PRESENT="the datastore ${DATA_DIR:-/var/lib/rancher/k3s}/server/db"; return 0 + fi + return 1 +} + +# tn_inst_http_get URL: the body on stdout. Returns 0 for 2xx, 1 when the server ANSWERED with an error +# (404: no such file, 401: no anonymous access), 2 when it could not be reached (or there is neither curl +# nor wget). No certificate check: the answer decides which release to install and nothing else (the +# server is the one this device joins with its token anyway). +tn_inst_http_get() { + _tn_hc=0 + if command -v curl >/dev/null 2>&1; then + curl -fsk --max-time 10 "$1" || _tn_hc=$? + case $_tn_hc in 0) return 0 ;; 22) return 1 ;; *) return 2 ;; esac + elif command -v wget >/dev/null 2>&1; then + wget -qO- --no-check-certificate -T 10 "$1" || _tn_hc=$? + case $_tn_hc in 0) return 0 ;; 8) return 1 ;; *) return 2 ;; esac + fi + return 2 +} + +# tn_inst_server_version URL: sets TN_INST_SERVER to the k3s release the server at URL runs +# (v1.34.11+k3s1) and TN_INST_SERVER_SRC to where it was read, or leaves them empty (and sets +# TN_INST_SERVER_WHY to unreachable or no-release). Fails when it cannot tell. Two places, in this order: +# /static/teknoir/k3s-version the file that the Teknoir server script publishes (anonymous) +# /version gitVersion, for a server that allows anonymous access (k3s does not by default) +tn_inst_server_version() { + TN_INST_SERVER=; TN_INST_SERVER_SRC=; TN_INST_SERVER_WHY=unreachable + _tn_su=${1%/} + _tn_sv_max=${TN_K3S_SERVER_VERSION_TRIES:-3} + _tn_sv_try=0 + while [ "$_tn_sv_try" -lt "$_tn_sv_max" ]; do + _tn_sv_try=$((_tn_sv_try + 1)) + _tn_unreach=0 + _tn_rc=0 + _tn_body=$(tn_inst_http_get "$_tn_su/static/teknoir/k3s-version" 2>/dev/null) || _tn_rc=$? + if [ "$_tn_rc" = 0 ]; then + TN_INST_SERVER=$(printf '%s' "$_tn_body" | tr -d ' \t\r\n' | sed -n 's/^\(v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*+k3s[0-9][0-9]*\)$/\1/p') + if [ -n "$TN_INST_SERVER" ]; then TN_INST_SERVER_SRC=$_tn_su/static/teknoir/k3s-version; return 0; fi + elif [ "$_tn_rc" = 2 ]; then + _tn_unreach=1 + fi + if [ "$_tn_unreach" = 0 ]; then + _tn_rc=0 + _tn_body=$(tn_inst_http_get "$_tn_su/version" 2>/dev/null) || _tn_rc=$? + if [ "$_tn_rc" = 0 ]; then + # {"major":"1","minor":"34","gitVersion":"v1.34.11+k3s1",...}, over several lines or one + TN_INST_SERVER=$(printf '%s\n' "$_tn_body" | tr -d '\n' | sed -n 's/^.*"gitVersion"[[:space:]]*:[[:space:]]*"\(v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*+k3s[0-9][0-9]*\)".*$/\1/p') + if [ -n "$TN_INST_SERVER" ]; then TN_INST_SERVER_SRC=$_tn_su/version; return 0; fi + elif [ "$_tn_rc" = 2 ]; then + _tn_unreach=1 + fi + fi + # the server answered, but has no release to give: asking again does not help + if [ "$_tn_unreach" = 0 ]; then TN_INST_SERVER_WHY=no-release; return 1; fi + if [ "$_tn_sv_try" -lt "$_tn_sv_max" ]; then sleep "${TN_K3S_SERVER_VERSION_WAIT:-3}"; fi + done + return 1 +} + +# tn_inst_publish_release: a SERVER script calls it after the k3s installer. Publishes the installed +# release as /server/static/teknoir/k3s-version; k3s serves server/static without +# authentication (https://:6443/static/teknoir/k3s-version), and an agent script reads it to +# install the release of its server. Written atomically; unchanged when it is right; it never fails the +# script. An out-of-band upgrade of k3s leaves it saying the older release, which is the safe direction +# (an agent may be older than its server, never newer): a re-run of the server script updates it. +tn_inst_publish_release() { + tn_inst_installed_version + if [ -z "$TN_INST_INSTALLED" ]; then + tn_inst_warn "cannot read the release of the installed k3s: it is not published, so agents cannot follow this server (they need INSTALL_K3S_VERSION)" + return 0 + fi + _tn_pd=${DATA_DIR:-/var/lib/rancher/k3s}/server/static/teknoir + _tn_pf=$_tn_pd/k3s-version + if [ -f "$_tn_pf" ] && [ "$(cat "$_tn_pf" 2>/dev/null)" = "$TN_INST_INSTALLED" ]; then + tn_inst_info "the k3s release $TN_INST_INSTALLED is already published for the agents ($_tn_pf)" + return 0 + fi + _tn_pt=$_tn_pd/.k3s-version.tmp.$$ + if mkdir -p "$_tn_pd" && printf '%s\n' "$TN_INST_INSTALLED" > "$_tn_pt" && mv -f "$_tn_pt" "$_tn_pf"; then + tn_inst_info "Published the k3s release $TN_INST_INSTALLED for the agents: $_tn_pf (served at /static/teknoir/k3s-version)" + else + rm -f "$_tn_pt" 2>/dev/null || true + tn_inst_warn "cannot publish the k3s release to $_tn_pf: agents cannot follow this server (they need INSTALL_K3S_VERSION)" + fi + return 0 +} + +# ----------------------------------------------------------------- the release to install + +# tn_inst_agent_target: for an agent that is to be installed (or replaced): which release. Sets and +# exports INSTALL_K3S_VERSION (and unsets INSTALL_K3S_CHANNEL) or exits through tn_inst_fatal. +tn_inst_agent_target() { + _tn_pin=${INSTALL_K3S_VERSION:-} + _tn_exp=0 + case ${TN_K3S_PIN_SOURCE:-default} in flag|env) _tn_exp=1 ;; esac + if [ -n "${OS_BUILD:-}" ]; then + tn_inst_warn "OS build: the k3s server is not reachable from here, so the release of the agent cannot follow it. ${_tn_pin:-The release the installer picks} is installed: the server must run this release or a newer one" + return 0 + fi + TN_INST_SERVER= + if [ -z "${K3S_URL:-}" ]; then + tn_inst_warn "K3S_URL is not set: the k3s release of the server cannot be read" + else + tn_inst_info "Reading the k3s release of the server at ${K3S_URL%/}" + tn_inst_server_version "$K3S_URL" || true + fi + if [ -n "$TN_INST_SERVER" ]; then + tn_inst_info "the k3s server at $K3S_URL runs $TN_INST_SERVER (read from $TN_INST_SERVER_SRC)" + if [ "$_tn_exp" = 1 ] && [ -n "$_tn_pin" ] && tn_inst_cmp "$_tn_pin" "$TN_INST_SERVER"; then + case $TN_INST_CMP in + 0) tn_inst_info "the requested release $_tn_pin is the release of the server" ;; + -1) tn_inst_warn "the requested release $_tn_pin is OLDER than the release of the server ($TN_INST_SERVER): it is installed as asked. An agent should run the release of its server (Kubernetes supports a kubelet up to three minor versions older than the apiserver)" ;; + *) if [ "${TN_K3S_FORCE:-}" = true ]; then + tn_inst_warn "the requested release $_tn_pin is NEWER than the release of the server ($TN_INST_SERVER); TN_K3S_FORCE=true: it is installed anyway. A kubelet newer than its apiserver is not supported" + else + tn_inst_fatal "refusing to install k3s $_tn_pin on this agent: the server at $K3S_URL runs the older $TN_INST_SERVER, and a kubelet must not be newer than the apiserver it joins. Without --k3s-version (INSTALL_K3S_VERSION) the agent gets the release of its server, $TN_INST_SERVER. Or upgrade the server first, or set TN_K3S_FORCE=true. $(tn_inst_state_note)" + fi ;; + esac + return 0 + fi + if [ "$_tn_exp" = 1 ]; then + tn_inst_warn "the requested k3s release (${_tn_pin:-${INSTALL_K3S_CHANNEL:-stable}}) is not an exact release, so it cannot be compared with the server: the release of the server is installed instead" + elif [ -n "$_tn_pin" ] && [ "$_tn_pin" != "$TN_INST_SERVER" ]; then + tn_inst_info "the script's default $_tn_pin is not used: an agent runs the release of its server" + fi + INSTALL_K3S_VERSION=$TN_INST_SERVER + export INSTALL_K3S_VERSION + unset INSTALL_K3S_CHANNEL + tn_inst_info "the agent gets the release of its server: $TN_INST_SERVER" + return 0 + fi + # the server cannot be asked + if [ "$_tn_exp" = 1 ] && [ -n "$_tn_pin" ] && tn_inst_parse "$_tn_pin"; then + tn_inst_warn "cannot read the k3s release of the server at ${K3S_URL:-(K3S_URL is not set)}: the requested $_tn_pin is installed as asked. The server must run this release or a newer one" + return 0 + fi + if [ "${TN_K3S_FORCE:-}" = true ]; then + tn_inst_warn "cannot read the k3s release of the server at ${K3S_URL:-(K3S_URL is not set)}; TN_K3S_FORCE=true: ${_tn_pin:-the release the installer picks} is installed anyway. If the server runs an older release, this agent will not work" + return 0 + fi + case ${TN_INST_SERVER_WHY:-unreachable} in + no-release) _tn_why="The server answered, but does not publish its release: it was bootstrapped by a script that does not yet (run the server script again: it keeps k3s as it is and publishes its release), and k3s does not allow an anonymous /version." ;; + *) _tn_why="Is the server up and reachable from this device? Start it and run this script again." ;; + esac + tn_inst_fatal "cannot read the k3s release of the server at ${K3S_URL:-(K3S_URL is not set)} (${K3S_URL:-https://:6443}/static/teknoir/k3s-version): an agent must not run a newer k3s than its server, so this script does not guess. $_tn_why Or say which release the server runs: INSTALL_K3S_VERSION=v1.xx.y+k3sN (tnctl bootstrap --k3s-version), or set TN_K3S_FORCE=true to install ${_tn_pin:-the release the installer picks} anyway. $(tn_inst_state_note)" +} + +# tn_inst_normalize_pin: the k3s installer takes INSTALL_K3S_VERSION as an EXACT release (it downloads that +# tag, so "stable" would be a 404) and INSTALL_K3S_CHANNEL as a channel. A channel name given as the version +# (stable, latest, testing, or a minor such as v1.34) is moved to INSTALL_K3S_CHANNEL and the version is +# unset, so that every later check sees what the installer will see. An exact release is not touched. +tn_inst_normalize_pin() { + [ -n "${INSTALL_K3S_VERSION:-}" ] || return 0 + case $INSTALL_K3S_VERSION in + stable|latest|testing) ;; + v[0-9]*) printf '%s\n' "$INSTALL_K3S_VERSION" | grep -Eq '^v[0-9]+\.[0-9]+$' || return 0 ;; + *) return 0 ;; + esac + if [ -n "${INSTALL_K3S_CHANNEL:-}" ] && [ "$INSTALL_K3S_CHANNEL" != "$INSTALL_K3S_VERSION" ]; then + tn_inst_fatal "conflicting k3s requests: INSTALL_K3S_VERSION=$INSTALL_K3S_VERSION is a channel and INSTALL_K3S_CHANNEL=$INSTALL_K3S_CHANNEL is another. Set only one of them. $(tn_inst_state_note)" + fi + INSTALL_K3S_CHANNEL=$INSTALL_K3S_VERSION + export INSTALL_K3S_CHANNEL + unset INSTALL_K3S_VERSION + tn_inst_info "$INSTALL_K3S_CHANNEL is a k3s channel, not a release: the installer gets INSTALL_K3S_CHANNEL=$INSTALL_K3S_CHANNEL" +} + +# tn_inst_plan: applies the policy at the top. Reads INSTALL_K3S_VERSION / INSTALL_K3S_CHANNEL, +# sets and exports them for the installer. Sets TN_INST_INSTALLED, TN_INST_ACTION +# (install|same|keep|upgrade) and TN_INST_EFFECTIVE (the exact release that will run, or nothing +# when only a channel is known). Exits through tn_inst_fatal when it refuses. +tn_inst_plan() { + tn_inst_normalize_pin + tn_inst_installed_version + if [ -z "$TN_INST_INSTALLED" ] && tn_inst_k3s_present; then + if [ "${TN_K3S_FORCE:-}" = true ]; then + tn_inst_warn "$TN_INST_PRESENT exists, but its k3s version cannot be read; TN_K3S_FORCE=true: going on as if k3s were not installed. The installer may replace it with ${INSTALL_K3S_VERSION:-${INSTALL_K3S_CHANNEL:-the release the installer picks}}, without the upgrade and downgrade checks" + else + tn_inst_fatal "$TN_INST_PRESENT exists, but its k3s version cannot be read ('$(tn_inst_k3s_cmd) --version' gave no release), so this script cannot tell whether the installer would upgrade it, downgrade it or break its datastore, and does not install anything. Look at '$(tn_inst_k3s_cmd) --version' and at the datastore; repair or remove k3s on purpose, or set TN_K3S_FORCE=true to install anyway. $(tn_inst_state_note)" + fi + fi + if [ "${TN_INST_ROLE:-server}" = agent ] && { [ -z "$TN_INST_INSTALLED" ] || [ "${TN_K3S_UPGRADE:-}" = true ]; }; then + tn_inst_agent_target + fi + _tn_t=${INSTALL_K3S_VERSION:-} + _tn_want=${_tn_t:-${INSTALL_K3S_CHANNEL:-the default channel of the k3s installer (stable)}} + TN_INST_ACTION=install + TN_INST_EFFECTIVE= + if [ -z "$TN_INST_INSTALLED" ]; then + TN_INST_EFFECTIVE=$_tn_t + if [ -n "$_tn_t" ]; then tn_inst_info "k3s is not installed: installing $_tn_t"; else tn_inst_info "k3s is not installed: installing $_tn_want"; fi + return 0 + fi + tn_inst_info "k3s $TN_INST_INSTALLED is installed on this device" + + if [ "${TN_K3S_UPGRADE:-}" != true ]; then + # keep: an exact release is not "newer", it is "this one" + TN_INST_EFFECTIVE=$TN_INST_INSTALLED + if [ "$_tn_t" = "$TN_INST_INSTALLED" ]; then + TN_INST_ACTION=same + tn_inst_info "k3s $TN_INST_INSTALLED is the requested release: it is installed again, nothing is upgraded" + return 0 + fi + TN_INST_ACTION=keep + INSTALL_K3S_VERSION=$TN_INST_INSTALLED + export INSTALL_K3S_VERSION + unset INSTALL_K3S_CHANNEL + if tn_inst_cmp "$_tn_t" "$TN_INST_INSTALLED" && [ "$TN_INST_CMP" = -1 ]; then + tn_inst_warn "the requested k3s release $_tn_t is OLDER than the installed $TN_INST_INSTALLED: k3s cannot be downgraded, $TN_INST_INSTALLED is kept" + else + tn_inst_warn "k3s $TN_INST_INSTALLED is kept; the requested $_tn_want is NOT installed. A re-run of this script repairs the device, it does not upgrade k3s. To upgrade, run it with TN_K3S_UPGRADE=true (tnctl bootstrap --k3s-upgrade), one minor version at a time" + fi + return 0 + fi + + # TN_K3S_UPGRADE=true + if [ -z "$_tn_t" ]; then + # a channel: the installer picks the release, so find out which, or compare the minor + _tn_ch=${INSTALL_K3S_CHANNEL:-stable} + if printf '%s\n' "$_tn_ch" | grep -Eq '^v[0-9]+\.[0-9]+$'; then + tn_inst_parse "${_tn_ch}.0"; _tn_cmaj=$TN_INST_MAJ; _tn_cmin=$TN_INST_MIN + tn_inst_parse "$TN_INST_INSTALLED"; _tn_imaj=$TN_INST_MAJ; _tn_imin=$TN_INST_MIN + if [ "$_tn_cmaj" != "$_tn_imaj" ] || [ "$_tn_cmin" -lt "$_tn_imin" ]; then + tn_inst_fatal "refusing to downgrade k3s from $TN_INST_INSTALLED to the channel $_tn_ch (v$_tn_cmaj.$_tn_cmin is not v$_tn_imaj.$_tn_imin or a later minor of the same major): k3s cannot go back (the datastore has been migrated by the newer release). $(tn_inst_state_note). Use a channel of v$_tn_imaj.$_tn_imin or $_tn_imaj.$((_tn_imin + 1)), or restore the device from a backup" + fi + if [ $((_tn_cmin - _tn_imin)) -gt 1 ]; then + if [ "${TN_K3S_FORCE:-}" = true ]; then + tn_inst_warn "upgrading k3s from $TN_INST_INSTALLED to the channel $_tn_ch skips minor versions (TN_K3S_FORCE=true). Kubernetes supports one minor version at a time: this can leave the cluster in a state that does not start. Have a backup of the datastore" + else + tn_inst_fatal "refusing to upgrade k3s from $TN_INST_INSTALLED to the channel $_tn_ch: that is more than one minor version (v$_tn_imaj.$_tn_imin -> v$_tn_cmaj.$_tn_cmin). Upgrade one minor at a time (channel v$_tn_imaj.$((_tn_imin + 1)) first), or set TN_K3S_FORCE=true (tnctl bootstrap --force) if you know what you are doing. $(tn_inst_state_note)" + fi + fi + TN_INST_ACTION=upgrade + tn_inst_info "upgrading k3s from $TN_INST_INSTALLED to the latest release of the channel $_tn_ch (TN_K3S_UPGRADE=true). A server must be upgraded before its agents" + return 0 + fi + # stable, latest, testing: resolve like the installer does (the last path segment of the redirect) + TN_INST_RESOLVED=$(curl -w '%{url_effective}' -L -s -S -o /dev/null "https://update.k3s.io/v1-release/channels/$_tn_ch" 2>/dev/null | sed 's|^.*/||; s|%2[Bb]|+|') || TN_INST_RESOLVED= + if tn_inst_parse "$TN_INST_RESOLVED" 2>/dev/null; then + _tn_t=$TN_INST_RESOLVED + tn_inst_info "the k3s channel $_tn_ch resolves to $_tn_t: it is pinned, so that the installer gets what was checked" + INSTALL_K3S_VERSION=$_tn_t + export INSTALL_K3S_VERSION + unset INSTALL_K3S_CHANNEL + elif [ "${TN_K3S_FORCE:-}" = true ]; then + TN_INST_ACTION=upgrade + tn_inst_warn "TN_K3S_UPGRADE=true with the channel $_tn_ch, which cannot be resolved from here (update.k3s.io): the k3s installer picks the release, so a downgrade or a jump over several minor versions cannot be checked. TN_K3S_FORCE=true: going on without those checks. Pin an exact release (INSTALL_K3S_VERSION=v1.xx.y+k3sN) to get them" + return 0 + else + tn_inst_fatal "refusing to upgrade k3s from $TN_INST_INSTALLED to the channel $_tn_ch: the channel cannot be resolved to a release from here (update.k3s.io is not reachable), so a downgrade or a jump over several minor versions cannot be checked. Pin an exact release (INSTALL_K3S_VERSION=v1.xx.y+k3sN, tnctl bootstrap --k3s-version), or set TN_K3S_FORCE=true to go on without the checks. $(tn_inst_state_note)" + fi + fi + if ! tn_inst_cmp "$_tn_t" "$TN_INST_INSTALLED"; then + tn_inst_fatal "cannot compare the requested k3s release '$_tn_t' with the installed $TN_INST_INSTALLED. Pin an exact release (INSTALL_K3S_VERSION=v1.xx.y+k3sN). $(tn_inst_state_note)" + fi + TN_INST_EFFECTIVE=$_tn_t + case $TN_INST_CMP in + 0) TN_INST_ACTION=same; tn_inst_info "k3s $TN_INST_INSTALLED is the requested release: nothing to upgrade"; return 0 ;; + -1) tn_inst_fatal "refusing to downgrade k3s from $TN_INST_INSTALLED to $_tn_t: k3s cannot go back (the datastore has been migrated by the newer release). $(tn_inst_state_note). Pin $TN_INST_INSTALLED or a newer release, or restore the device from a backup" ;; + esac + tn_inst_parse "$TN_INST_INSTALLED"; _tn_imaj=$TN_INST_MAJ; _tn_imin=$TN_INST_MIN + tn_inst_parse "$_tn_t" + if [ "$TN_INST_MAJ" != "$_tn_imaj" ] || [ $((TN_INST_MIN - _tn_imin)) -gt 1 ]; then + if [ "${TN_K3S_FORCE:-}" = true ]; then + tn_inst_warn "upgrading k3s from $TN_INST_INSTALLED to $_tn_t skips minor versions (TN_K3S_FORCE=true). Kubernetes supports one minor version at a time: this can leave the cluster in a state that does not start. Have a backup of the datastore" + else + tn_inst_fatal "refusing to upgrade k3s from $TN_INST_INSTALLED to $_tn_t: that is more than one minor version (v$_tn_imaj.$_tn_imin -> v$TN_INST_MAJ.$TN_INST_MIN). Upgrade one minor at a time (to v$_tn_imaj.$((_tn_imin + 1)).x first), or set TN_K3S_FORCE=true (tnctl bootstrap --force) if you know what you are doing. $(tn_inst_state_note)" + fi + fi + TN_INST_ACTION=upgrade + tn_inst_info "upgrading k3s from $TN_INST_INSTALLED to $_tn_t (TN_K3S_UPGRADE=true). A server must be upgraded before its agents" +} + +# ----------------------------------------------------------------- cgroup v1 + +TN_INST_CGROUP_FILE=99-teknoir-cgroupv1.yaml + +# tn_inst_cgroup_state: prints v1 (cgroup v1 or hybrid), v2 or unknown +tn_inst_cgroup_state() { + case ${TN_K3S_CGROUPV1:-} in true) echo v1; return 0 ;; false) echo v2; return 0 ;; esac + # an OS image is built on another machine than the one it will run on + if [ -n "${OS_BUILD:-}" ]; then echo unknown; return 0; fi + _tn_cg=${TN_CGROUP_FSTYPE:-} + [ -n "$_tn_cg" ] || _tn_cg=$(stat -fc %T /sys/fs/cgroup 2>/dev/null) || _tn_cg= + case $_tn_cg in + cgroup2fs) echo v2 ;; + '') echo unknown ;; + *) echo v1 ;; # tmpfs: cgroup v1, or the hybrid layout + esac +} + +# tn_inst_cgroup_flag_in_text: reads TEXT on stdin; 0 if an uncommented part of it sets fail-cgroupv1 to +# false (kubelet-arg: ["fail-cgroupv1=false"], - fail-cgroupv1=false, --kubelet-arg=fail-cgroupv1=false). +# A whole-line or trailing YAML comment is dropped first; fail-cgroupv1=true does not match. +tn_inst_cgroup_flag_in_text() { + sed 's/^[[:space:]]*#.*$//; s/[[:space:]]#.*$//' | grep -Eiq 'fail-cgroupv1[[:space:]]*=[[:space:]]*false([^A-Za-z0-9_]|$)' +} + +# tn_inst_cgroup_configured_elsewhere: 0 if the device sets fail-cgroupv1=false itself (not in our file). +# Sets TN_INST_CGROUP_WHERE to where, and TN_INST_CGROUP_PERSISTENT to 1 when that is a config file that +# the running k3s reads (config.yaml, config.yaml.d/*.yaml and *.yml), and to 0 when it is only INSTALL_K3S_EXEC +# (which reaches the unit when the installer has run). Files win over INSTALL_K3S_EXEC. A comment, and +# fail-cgroupv1=true, do not count. +tn_inst_cgroup_configured_elsewhere() { + _tn_cd=${TN_K3S_CONFIG_DIR:-/etc/rancher/k3s} + TN_INST_CGROUP_WHERE=; TN_INST_CGROUP_PERSISTENT=0 + for _tn_cf in "$_tn_cd/config.yaml" $(find "$_tn_cd/config.yaml.d" -maxdepth 1 -type f \( -iname '*.yaml' -o -iname '*.yml' \) 2>/dev/null | LC_ALL=C sort); do + [ -r "$_tn_cf" ] || continue + [ "$_tn_cf" != "$_tn_cd/config.yaml.d/$TN_INST_CGROUP_FILE" ] || continue + if tn_inst_cgroup_flag_in_text < "$_tn_cf"; then TN_INST_CGROUP_WHERE=$_tn_cf; TN_INST_CGROUP_PERSISTENT=1; return 0; fi + done + if printf '%s\n' "${INSTALL_K3S_EXEC:-}" | tn_inst_cgroup_flag_in_text; then TN_INST_CGROUP_WHERE=INSTALL_K3S_EXEC; return 0; fi + return 1 +} + +tn_inst_cgroup_remove() { # FILE WHY + [ -f "$1" ] || return 0 + rm -f "$1" || { tn_inst_warn "cannot remove $1"; return 0; } + tn_inst_info "Removed $1 ($2)" +} + +# tn_inst_cgroup: writes or removes the drop-in, see the top. Uses TN_INST_EFFECTIVE (the release +# that will run; nothing = a channel, taken as new enough: stable and latest are >= v1.35; a channel +# named vMAJ.MIN is read as that minor). The flag goes in from v1.31, where the kubelet learned it (an +# older kubelet would not start with an unknown argument). +tn_inst_cgroup() { + _tn_cdir=${TN_K3S_CONFIG_DIR:-/etc/rancher/k3s}/config.yaml.d + _tn_cfile=$_tn_cdir/$TN_INST_CGROUP_FILE + TN_INST_CGROUP_DEFERRED= + _tn_cst=$(tn_inst_cgroup_state) + if [ "$_tn_cst" = v2 ]; then + tn_inst_cgroup_remove "$_tn_cfile" "this host uses cgroup v2" + return 0 + fi + _tn_eff=${TN_INST_EFFECTIVE:-} + if [ -z "$_tn_eff" ]; then + case ${INSTALL_K3S_CHANNEL:-} in v[0-9]*.[0-9]*) tn_inst_parse "${INSTALL_K3S_CHANNEL}.0" && _tn_eff=${INSTALL_K3S_CHANNEL}.0 ;; esac + fi + _tn_flagok=1 _tn_must=1 + if [ -n "$_tn_eff" ] && tn_inst_cmp "$_tn_eff" v1.31.0 && [ "$TN_INST_CMP" = -1 ]; then _tn_flagok=0; fi + if [ -n "$_tn_eff" ] && tn_inst_cmp "$_tn_eff" v1.35.0 && [ "$TN_INST_CMP" = -1 ]; then _tn_must=0; fi + if [ "$_tn_cst" = v1 ]; then + tn_inst_warn "this host uses cgroup v1 (or the hybrid layout). cgroup v1 is deprecated: Kubernetes >= 1.35 refuses to start the kubelet on it. Move the device to cgroup v2 (kernel command line systemd.unified_cgroup_hierarchy=1, or a newer OS release)" + else + tn_inst_info "cannot tell whether the device uses cgroup v1 (OS build, or no /sys/fs/cgroup): the kubelet flag is set to be safe, it is harmless on cgroup v2" + fi + if [ "$_tn_flagok" = 0 ]; then + tn_inst_info "k3s $_tn_eff is older than v1.31 (its kubelet has no fail-cgroupv1 flag) and starts on cgroup v1 without help" + tn_inst_cgroup_remove "$_tn_cfile" "k3s is older than v1.31" + return 0 + fi + if [ "$_tn_must" = 0 ]; then + tn_inst_info "k3s $_tn_eff starts on cgroup v1 without help; the kubelet flag is set anyway, so that an upgrade to v1.35 or newer (also one made by hand) does not stop the kubelet" + fi + if tn_inst_cgroup_configured_elsewhere; then + if [ "$TN_INST_CGROUP_PERSISTENT" = 1 ]; then + tn_inst_info "fail-cgroupv1=false is set by the device itself ($TN_INST_CGROUP_WHERE): left as it is" + tn_inst_cgroup_remove "$_tn_cfile" "the device sets fail-cgroupv1=false itself, in a file that k3s reads" + return 0 + fi + # only on the command line of the installer: k3s does not read it until the unit is rewritten, so the + # drop-in must keep protecting a k3s that restarts before that. The same value twice does no harm. + tn_inst_info "fail-cgroupv1=false is set by the device itself ($TN_INST_CGROUP_WHERE): left as it is. The drop-in $_tn_cfile stays until the installer has put it in the unit; the same value twice is harmless, and the one on the command line wins" + fi + # an installed k3s older than v1.31 (about to be upgraded) cannot start with the flag + if [ -n "${TN_INST_INSTALLED:-}" ] && tn_inst_cmp "$TN_INST_INSTALLED" v1.31.0 && [ "$TN_INST_CMP" = -1 ]; then + TN_INST_CGROUP_DEFERRED=1 + tn_inst_info "the installed k3s $TN_INST_INSTALLED is older than v1.31: the kubelet flag is set after the installer has upgraded it, not before" + tn_inst_cgroup_remove "$_tn_cfile" "the installed k3s is older than v1.31 and cannot take the flag until it is upgraded" + # The installer starts the new release as soon as it has written the unit. From v1.35 that kubelet + # does not start on cgroup v1 without the flag, and tn_inst_finish would never run (the installer's + # restart fails under set -e). So the unit gets the flag from its first start, through INSTALL_K3S_EXEC: + # the old kubelet, which is running now, never reads it. Not needed for an exact v1.31 - v1.34. + if [ "$_tn_must" = 1 ]; then + if printf '%s\n' "${INSTALL_K3S_EXEC:-}" | tn_inst_cgroup_flag_in_text; then + tn_inst_info "INSTALL_K3S_EXEC already has fail-cgroupv1=false: the new unit carries it from its first start" + else + INSTALL_K3S_EXEC="${INSTALL_K3S_EXEC:+$INSTALL_K3S_EXEC }--kubelet-arg=fail-cgroupv1=false" + export INSTALL_K3S_EXEC + tn_inst_info "k3s ${_tn_eff:-from a channel} may need the flag on its first start: --kubelet-arg=fail-cgroupv1=false is added to INSTALL_K3S_EXEC, so that the new unit has it. The old kubelet does not read it" + fi + fi + return 0 + fi + tn_inst_cgroup_write +} + +# tn_inst_cgroup_write: the drop-in, written atomically and only when it differs +tn_inst_cgroup_write() { + _tn_cdir=${TN_K3S_CONFIG_DIR:-/etc/rancher/k3s}/config.yaml.d + _tn_cfile=$_tn_cdir/$TN_INST_CGROUP_FILE + _tn_want=$(printf '%s\n' \ + '# Written by the Teknoir bootstrap script. Kubernetes >= 1.35 refuses to start the kubelet on cgroup v1' \ + '# unless this is false. cgroup v1 is deprecated: move the device to cgroup v2 and delete this file.' \ + '# "kubelet-arg+" appends to the kubelet-arg of config.yaml; harmless on cgroup v2.' \ + 'kubelet-arg+:' \ + ' - "fail-cgroupv1=false"') + if [ -f "$_tn_cfile" ] && [ "$(cat "$_tn_cfile")" = "$_tn_want" ]; then + tn_inst_info "kubelet-arg fail-cgroupv1=false is already in place ($_tn_cfile)" + return 0 + fi + # atomically: k3s may be restarting in a loop right now and must never read half a file. The temporary + # name does not end in .yaml, so k3s does not read it either. + _tn_tmp=$_tn_cdir/.$TN_INST_CGROUP_FILE.tmp.$$ + if mkdir -p "$_tn_cdir" && find "$_tn_cdir" -maxdepth 1 -name ".$TN_INST_CGROUP_FILE.tmp.*" -exec rm -f {} + && + printf '%s\n' "$_tn_want" > "$_tn_tmp" && mv -f "$_tn_tmp" "$_tn_cfile"; then + : + else + rm -f "$_tn_tmp" 2>/dev/null || true + tn_inst_fatal "cannot write $_tn_cfile: k3s >= v1.35 would not start on this cgroup v1 host" + fi + tn_inst_info "Set the kubelet argument fail-cgroupv1=false in $_tn_cfile (k3s ${TN_INST_EFFECTIVE:-from a channel} on cgroup v1)" +} + +# ----------------------------------------------------------------- config.yaml + +# tn_inst_yaml_has_key FILE KEY: is there a top-level KEY (or KEY+) in FILE +tn_inst_yaml_has_key() { + [ -r "$1" ] || return 1 + awk -v k="$2" '{ sub(/\r$/, "") } $0 ~ ("^" k "\\+?[ \t]*:") { f = 1 } END { exit f ? 0 : 1 }' "$1" +} + +# tn_inst_config_ensure FILE KEY TEXT: appends TEXT (the YAML of the key, may be several lines) to +# FILE unless FILE has the top-level KEY already. A config.yaml that the device owner has edited is +# never rewritten: a re-run of a bootstrap script used to replace it, and lost every key it did not know. +tn_inst_config_ensure() { + if tn_inst_yaml_has_key "$1" "$2"; then + tn_inst_info "$1 already sets '$2': kept" + return 0 + fi + mkdir -p "$(dirname "$1")" || return 1 + if [ -s "$1" ] && [ -n "$(tail -c 1 "$1")" ]; then echo >> "$1" || return 1; fi + printf '%s\n' "$3" >> "$1" || return 1 + tn_inst_info "$1: added '$2'" +} + +# ----------------------------------------------------------------- the entry point + +# tn_inst_prepare ROLE: the release policy, then cgroup v1. Call it after INSTALL_K3S_VERSION / +# INSTALL_K3S_CHANNEL (and INSTALL_K3S_EXEC, if the device sets fail-cgroupv1 there) are set and +# before the k3s installer. A SERVER script calls it BEFORE tn_k3s_ca_setup: the kubelet flag must be in +# place before the CA step restarts k3s, or a k3s that a cgroup v1 host made exit would not be up for +# rotate-ca (it restarts itself every few seconds and reads the flag then). The CA step must in turn +# run before the installer. An agent has no CA step. +tn_inst_prepare() { + TN_INST_ROLE=${1:-server} + tn_inst_info "Checking the k3s release and the cgroup version ($TN_INST_ROLE)" + tn_inst_plan + tn_inst_cgroup +} + +# tn_inst_finish: call it right after the k3s installer succeeded. Writes the kubelet flag that +# tn_inst_prepare held back because the k3s that was installed then could not take it (see "Order of +# effects"). Nothing to do otherwise. +tn_inst_finish() { + if [ -n "${TN_INST_CGROUP_DEFERRED:-}" ]; then + TN_INST_CGROUP_DEFERRED= + tn_inst_info "The installer has run: setting the kubelet flag that was held back" + tn_inst_cgroup_write + fi +} + + +# The node name, role labels and taints of this node (tnctl bootstrap --role ...). k3s applies the labels +# and taints only when the node registers for the first time, so they are written before the installer runs. +# tn-k3s-node.sh: the node name, role labels and taints of a k3s node (tnctl bootstrap --role ...). +# +# Embedded in the server and the agent bootstrap script (only when the script was generated with a node +# role, label, taint or name), and usable stand-alone from a test. Usage: +# +# . tn-k3s-install.sh; . tn-k3s-node.sh +# tn_node_name NAME (the node name; optional) +# tn_node_label key=value (once per label; the role and robot labels come first) +# tn_node_taint key[=value]:Effect (once per taint) +# tn_node_apply server|agent (writes the k3s configuration; once, before the k3s installer) +# +# It needs tn_inst_config_ensure, tn_inst_yaml_has_key and tn_inst_info/warn from tn-k3s-install.sh. +# POSIX sh. It runs under `set -e` and `set -o noglob` and does not change shell options. +# +# What it writes (TN_K3S_CONFIG_DIR, /etc/rancher/k3s by default): +# config.yaml node-name, added only when config.yaml has no node-name already: a +# name the device owner set is kept (a different one is reported) +# config.yaml.d/98-teknoir-node.yaml node-label+ and node-taint+: the labels and taints of the role. +# "+" APPENDS to the node-label and node-taint lists of config.yaml and +# the other drop-ins, so the entries of the device owner are kept. The +# file is ours: it is rewritten (atomically) when it differs and left +# alone when it does not, so a re-run changes nothing. +# +# k3s applies node-label and node-taint ONLY when a node registers for the first time. When the node has +# registered already and the file changes (or is new), the script says so and prints the kubectl commands +# that apply the labels and taints to the running node; run them on the k3s server. +# +# Environment: +# TN_K3S_CONFIG_DIR /etc/rancher/k3s +# DATA_DIR the k3s data dir (/var/lib/rancher/k3s): /agent/client-kubelet.crt exists +# once the kubelet of this node has registered +# TN_NODE_REGISTERED true / false: do not look at the data dir, take this as the answer (tests) + +TN_K3S_NODE_LIB_VERSION=1 +export TN_K3S_NODE_LIB_VERSION + +TN_NODE_NAME= +TN_NODE_LABELS= +TN_NODE_TAINTS= +TN_NODE_DROPIN=98-teknoir-node.yaml + +tn_node_name() { TN_NODE_NAME=$1; } +tn_node_label() { TN_NODE_LABELS="${TN_NODE_LABELS:+$TN_NODE_LABELS +}$1"; } +tn_node_taint() { TN_NODE_TAINTS="${TN_NODE_TAINTS:+$TN_NODE_TAINTS +}$1"; } + +# tn_node_yaml_value FILE KEY: the value of the top-level scalar KEY (quotes and a trailing comment removed) +tn_node_yaml_value() { + [ -r "$1" ] || return 0 + awk -v k="$2" ' + { sub(/\r$/, "") } + $0 ~ ("^" k "[ \t]*:") { + v = $0 + sub("^" k "[ \t]*:[ \t]*", "", v) + sub(/[ \t]+#.*$/, "", v) + gsub(/^["\047]|["\047]$/, "", v) + print v + exit + }' "$1" +} + +# tn_node_registered: has the kubelet of this node a client certificate, i.e. has the node registered +tn_node_registered() { + case ${TN_NODE_REGISTERED:-} in + true) return 0 ;; + false) return 1 ;; + esac + [ -s "${DATA_DIR:-/var/lib/rancher/k3s}/agent/client-kubelet.crt" ] +} + +# tn_node_dropin_text: the content of the labels and taints file +tn_node_dropin_text() { + printf '%s\n' \ + '# Written by the Teknoir bootstrap script (tnctl bootstrap --role ...). Do not edit: a re-run rewrites it.' \ + '# "+" appends to the node-label and node-taint lists of config.yaml and the other drop-ins.' \ + '# k3s applies them only when the node registers for the first time.' + if [ -n "$TN_NODE_LABELS" ]; then + echo 'node-label+:' + printf '%s\n' "$TN_NODE_LABELS" | while IFS= read -r _tn_l; do printf ' - "%s"\n' "$_tn_l"; done + fi + if [ -n "$TN_NODE_TAINTS" ]; then + echo 'node-taint+:' + printf '%s\n' "$TN_NODE_TAINTS" | while IFS= read -r _tn_t; do printf ' - "%s"\n' "$_tn_t"; done + fi +} + +# tn_node_kubectl_help NODE: how to apply the labels and taints to a node that has registered already +tn_node_kubectl_help() { + _tn_kl= + if [ -n "$TN_NODE_LABELS" ]; then + _tn_kl=$(printf '%s\n' "$TN_NODE_LABELS" | tr '\n' ' ') + tn_inst_warn " k3s kubectl label node $1 ${_tn_kl}--overwrite" + fi + if [ -n "$TN_NODE_TAINTS" ]; then + _tn_kl=$(printf '%s\n' "$TN_NODE_TAINTS" | tr '\n' ' ') + tn_inst_warn " k3s kubectl taint node $1 ${_tn_kl}--overwrite" + fi +} + +# tn_node_apply ROLE: write the node name, and the labels and taints, into the k3s configuration +tn_node_apply() { + _tn_nrole=${1:-server} + _tn_ncfg=${TN_K3S_CONFIG_DIR:-/etc/rancher/k3s} + _tn_nyaml=$_tn_ncfg/config.yaml + mkdir -p "$_tn_ncfg" || tn_inst_fatal "cannot create $_tn_ncfg" + + # the name of the node: a node-name in config.yaml wins (it is the owner's), a different one is reported + if [ -n "$TN_NODE_NAME" ]; then + if tn_inst_yaml_has_key "$_tn_nyaml" node-name; then + _tn_have=$(tn_node_yaml_value "$_tn_nyaml" node-name) + if [ "$_tn_have" != "$TN_NODE_NAME" ]; then + tn_inst_warn "$_tn_nyaml sets node-name '$_tn_have', not '$TN_NODE_NAME' as asked for this $_tn_nrole: kept. The node keeps its name; edit the file to rename it (k3s registers a renamed node as a new one)" + fi + fi + tn_inst_config_ensure "$_tn_nyaml" node-name "node-name: $TN_NODE_NAME" + fi + + [ -n "$TN_NODE_LABELS$TN_NODE_TAINTS" ] || return 0 + + _tn_nfile=$_tn_ncfg/config.yaml.d/$TN_NODE_DROPIN + _tn_nwant=$(tn_node_dropin_text) + if [ -f "$_tn_nfile" ] && [ "$(cat "$_tn_nfile")" = "$_tn_nwant" ]; then + tn_inst_info "node labels and taints are already in place ($_tn_nfile)" + return 0 + fi + # atomically, and under a name that k3s does not read (it does not end in .yaml) + _tn_ntmp=$_tn_ncfg/config.yaml.d/.$TN_NODE_DROPIN.tmp.$$ + if mkdir -p "$_tn_ncfg/config.yaml.d" && printf '%s\n' "$_tn_nwant" > "$_tn_ntmp" && mv -f "$_tn_ntmp" "$_tn_nfile"; then + : + else + rm -f "$_tn_ntmp" 2>/dev/null || true + tn_inst_fatal "cannot write $_tn_nfile" + fi + tn_inst_info "Wrote the node labels and taints of this $_tn_nrole to $_tn_nfile" + + if tn_node_registered; then + _tn_nname=$(tn_node_yaml_value "$_tn_nyaml" node-name) + _tn_nname=${_tn_nname:-${TN_NODE_NAME:-$(hostname | tr 'A-Z' 'a-z')}} + tn_inst_warn "This node has registered with the cluster already, and k3s applies node-label and node-taint only when a node registers for the first time: the labels and taints above will NOT show up on it by themselves." + tn_inst_warn "Apply them to the running node from the k3s server (this changes the Node object only; the file above keeps them if the node ever registers again):" + tn_node_kubectl_help "$_tn_nname" + fi +} + +tn_node_name 'mobile-01-vision' +tn_node_label 'teknoir.org/node-role=vision' +tn_node_label 'teknoir.org/robot=mobile-01' +tn_node_label 'teknoir.org/gpu=nvidia' +tn_node_taint 'nvidia.com/gpu=true:NoSchedule' + +# Who chose the k3s release: the person who runs this script (INSTALL_K3S_VERSION / INSTALL_K3S_CHANNEL in the +# environment, or `tnctl bootstrap --k3s-version`) or the default of this script. An agent installs the +# release of its server unless it was asked for one (it is then refused if it is newer than the server). +if [ -n "${INSTALL_K3S_VERSION:-}${INSTALL_K3S_CHANNEL:-}" ]; then TN_K3S_PIN_SOURCE=env; else TN_K3S_PIN_SOURCE=default; fi +export TN_K3S_PIN_SOURCE +# k3s is pinned to v1.36.5+k3s1 (tnctl bootstrap --k3s-version). Override when running the script: +# INSTALL_K3S_VERSION=v1.xx.y+k3sN sh bootstrap_*.sh (an agent must run the version of its server) +# This is the release for a NEW device: a re-run on a device with k3s installed keeps the installed version +# (TN_K3S_UPGRADE=true, tnctl bootstrap --k3s-upgrade, replaces it; never a downgrade). +export INSTALL_K3S_VERSION="${INSTALL_K3S_VERSION:-v1.36.5+k3s1}" + +# Order: (0) the arguments of the k3s installer, (1) the k3s release policy and the cgroup v1 kubelet +# flag, (2) the CA, (3) the k3s installer. +# (0) comes first because (1) reads INSTALL_K3S_EXEC: it must see what the installer will get, or a +# fail-cgroupv1 that the caller put there would be taken as "set by the device" (the drop-in removed) +# and then lost when the variable is rebuilt. A value the caller gave is kept, after ours. +# (1) comes before the CA step: it refuses before anything on the device is changed, and the kubelet +# flag is in place when the CA step restarts k3s (a k3s that a cgroup v1 host made exit restarts itself +# in a loop, and has to be up for rotate-ca). The data dir is looked up first, the way the CA step does +# it, because a k3s datastore there decides whether a k3s whose version cannot be read is installed; a +# data dir that disagrees with config.yaml is refused here, before (1) changes anything. +# BEGIN install-k3s-exec +export INSTALL_K3S_EXEC="server --token ${_K3S_TOKEN} --tls-san ${_DEVICE_ID}.${_DOMAIN} --tls-san 10.0.0.10 --tls-san orin.example.test --tls-san 10.0.0.5${INSTALL_K3S_EXEC:+ ${INSTALL_K3S_EXEC}}" +if [ "${USE_DOCKER}" = true ]; then + info "Use docker container-runtime for K3s" + export INSTALL_K3S_EXEC="${INSTALL_K3S_EXEC} --docker" +fi +# END install-k3s-exec +tn_ca_resolve_datadir +tn_ca_require_consistent_datadir +tn_inst_prepare server +tn_k3s_ca_setup + +info "Install Rancher K3s" +export K3S_CONFIG_PATH=/etc/rancher/k3s +$SUDO mkdir -p ${K3S_CONFIG_PATH} +# A re-run must not replace a config.yaml that already exists (it may hold keys of the device owner, +# like a kubelet-arg): the Teknoir keys are added when they are missing. +tn_inst_config_ensure ${K3S_CONFIG_PATH}/config.yaml disable "disable: + - traefik" +tn_node_apply server +# The CA above was made in ${DATA_DIR}, the data dir that k3s uses on this device. The file just +# written and the unit that the installer writes (it has no --data-dir) must keep it there. +if [ "${DATA_DIR}" != "/var/lib/rancher/k3s" ]; then + tn_inst_config_ensure ${K3S_CONFIG_PATH}/config.yaml data-dir "data-dir: ${DATA_DIR}" +fi +cat >/etc/sysctl.d/teknoir-k3s.conf << EOF +fs.inotify.max_user_instances=1024 +EOF +sysctl --system + +download k3s_installer.sh https://get.k3s.io +$SUDO chmod +x k3s_installer.sh + +if [ ${OS_BUILD} ] || [ ${INSECURE} ]; then + info "Running installation without verifying ssl certs on URLs" + $SUDO sed -i "s#curl -w#curl --insecure -w#g" k3s_installer.sh + $SUDO sed -i "s#curl -o#curl --insecure -o#g" k3s_installer.sh +fi + +if [ ${OS_BUILD} ]; then + info "OS Build specifics" + export INSTALL_K3S_SKIP_START=true + $SUDO sed -i "s#-d /run/systemd#true#g" k3s_installer.sh +fi + +# The release (INSTALL_K3S_VERSION) and the kubelet flag were decided by tn_inst_prepare above. +export INSTALL_K3S_SYMLINK=force +$SUDO ./k3s_installer.sh +# The kubelet flag that had to wait for the installer (an installed k3s older than v1.31 that was upgraded) +tn_inst_finish + +# Tell the agents which k3s release this server runs: they install exactly that one (a kubelet must not be +# newer than its apiserver), and k3s does not answer /version to an anonymous caller. Served by k3s at +# https://:6443/static/teknoir/k3s-version. +tn_inst_publish_release + +info "Install Teknoir Manifests" +$SUDO mkdir -p ${DATA_DIR}/server/manifests +$SUDO tee ${DATA_DIR}/server/manifests/teknoir.yaml > /dev/null << EOL +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia + +--- +apiVersion: v1 +kind: Secret +metadata: + name: teknoir-registry-secret + namespace: kube-system +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: "TEST-ONLY-docker-secret" + +--- +apiVersion: v1 +kind: Secret +metadata: + name: teknoir-registry-secret + namespace: default +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: "TEST-ONLY-docker-secret" + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default + namespace: default +imagePullSecrets: +- name: teknoir-registry-secret + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default + namespace: kube-system +imagePullSecrets: +- name: teknoir-registry-secret + +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority-mqtt + namespace: kube-system +value: 900000000 +globalDefault: false +description: "This priority class should be used for mqtt service pods only." + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mqtt + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: mqtt + template: + metadata: + labels: + app: mqtt + spec: + priorityClassName: high-priority-mqtt + containers: + - name: mqtt + image: emqx/emqx-edge:latest + imagePullPolicy: Always + env: + - name: EMQX_MQTT__MAX_PACKET_SIZE + value: 3MB + ports: + - containerPort: 1883 + - containerPort: 18083 + resources: + requests: + memory: "128Mi" + cpu: "150m" + limits: + memory: "512Mi" + cpu: "1000m" + +--- +apiVersion: v1 +kind: Service +metadata: + name: mqtt + namespace: kube-system +spec: + type: NodePort + ports: + - nodePort: 31883 + port: 1883 + protocol: TCP + targetPort: 1883 + selector: + app: mqtt + +--- +apiVersion: v1 +kind: Service +metadata: + name: mqtt-dashboard + namespace: kube-system +spec: + type: NodePort + selector: + app: mqtt + ports: + - nodePort: 31083 + port: 18083 + protocol: TCP + targetPort: 18083 + +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority-toe + namespace: kube-system +value: 1000000000 +globalDefault: false +description: "This priority class should be used for toe service pods only." + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: toes.teknoir.org +spec: + group: teknoir.org + names: + kind: TOE + listKind: TOEList + plural: toes + singular: toe + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + config: + properties: + apiVersion: + type: string + items: + items: + properties: + apiVersion: + type: string + binaryData: + additionalProperties: + format: byte + type: string + type: object + data: + additionalProperties: + type: string + type: object + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + creationTimestamp: + format: date-time + type: string + deletionGracePeriodSeconds: + format: int64 + type: integer + deletionTimestamp: + format: date-time + type: string + finalizers: + items: + type: string + type: array + generateName: + type: string + generation: + format: int64 + type: integer + labels: + additionalProperties: + type: string + type: object + managedFields: + items: + properties: + apiVersion: + type: string + fieldsType: + type: string + fieldsV1: + type: object + manager: + type: string + operation: + type: string + subresource: + type: string + time: + format: date-time + type: string + type: object + type: array + name: + type: string + namespace: + type: string + ownerReferences: + items: + properties: + apiVersion: + type: string + blockOwnerDeletion: + type: boolean + controller: + type: boolean + kind: + type: string + name: + type: string + uid: + type: string + required: + - apiVersion + - kind + - name + - uid + type: object + x-kubernetes-map-type: atomic + type: array + resourceVersion: + type: string + selfLink: + type: string + uid: + type: string + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: + type: string + required: + - metadata + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + kind: + type: string + metadata: + properties: + continue: + type: string + remainingItemCount: + format: int64 + type: integer + resourceVersion: + type: string + selfLink: + type: string + type: object + required: + - items + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + status: + properties: + last_applied_config: + properties: + apiVersion: + type: string + items: + items: + properties: + apiVersion: + type: string + binaryData: + additionalProperties: + format: byte + type: string + type: object + data: + additionalProperties: + type: string + type: object + kind: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + creationTimestamp: + format: date-time + type: string + deletionGracePeriodSeconds: + format: int64 + type: integer + deletionTimestamp: + format: date-time + type: string + finalizers: + items: + type: string + type: array + generateName: + type: string + generation: + format: int64 + type: integer + labels: + additionalProperties: + type: string + type: object + managedFields: + items: + properties: + apiVersion: + type: string + fieldsType: + type: string + fieldsV1: + type: object + manager: + type: string + operation: + type: string + subresource: + type: string + time: + format: date-time + type: string + type: object + type: array + name: + type: string + namespace: + type: string + ownerReferences: + items: + properties: + apiVersion: + type: string + blockOwnerDeletion: + type: boolean + controller: + type: boolean + kind: + type: string + name: + type: string + uid: + type: string + required: + - apiVersion + - kind + - name + - uid + type: object + x-kubernetes-map-type: atomic + type: array + resourceVersion: + type: string + selfLink: + type: string + uid: + type: string + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: + type: string + required: + - metadata + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + kind: + type: string + metadata: + properties: + continue: + type: string + remainingItemCount: + format: int64 + type: integer + resourceVersion: + type: string + selfLink: + type: string + type: object + required: + - items + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + served: true + storage: true + subresources: + status: {} + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: teknoir + app.kubernetes.io/instance: controller-manager-sa + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: serviceaccount + app.kubernetes.io/part-of: teknoir + name: teknoir-controller-manager + namespace: kube-system +#imagePullSecrets: +#- name: teknoir-registry-secret + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: teknoir + app.kubernetes.io/instance: leader-election-role + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: role + app.kubernetes.io/part-of: teknoir + name: teknoir-leader-election-role + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: teknoir-manager-role +rules: +- apiGroups: + - teknoir.org + resources: + - toes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - teknoir.org + resources: + - toes/finalizers + verbs: + - update +- apiGroups: + - teknoir.org + resources: + - toes/status + verbs: + - get + - patch + - update + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: teknoir + app.kubernetes.io/instance: metrics-reader + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrole + app.kubernetes.io/part-of: teknoir + name: teknoir-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: teknoir + app.kubernetes.io/instance: proxy-role + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrole + app.kubernetes.io/part-of: teknoir + name: teknoir-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: teknoir + app.kubernetes.io/instance: manager-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/part-of: teknoir + name: teknoir-admin-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: teknoir-controller-manager + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: teknoir + app.kubernetes.io/instance: leader-election-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: rolebinding + app.kubernetes.io/part-of: teknoir + name: teknoir-leader-election-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: teknoir-leader-election-role +subjects: +- kind: ServiceAccount + name: teknoir-controller-manager + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: teknoir + app.kubernetes.io/instance: manager-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/part-of: teknoir + name: teknoir-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: teknoir-manager-role +subjects: +- kind: ServiceAccount + name: teknoir-controller-manager + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: teknoir + app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/part-of: teknoir + name: teknoir-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: teknoir-proxy-role +subjects: +- kind: ServiceAccount + name: teknoir-controller-manager + namespace: kube-system + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: teknoir + app.kubernetes.io/instance: controller-manager-metrics-service + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: service + app.kubernetes.io/part-of: teknoir + control-plane: controller-manager + name: toe-metrics-service + namespace: kube-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: teknoir + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: deployment + app.kubernetes.io/part-of: teknoir + control-plane: controller-manager + name: toe + namespace: kube-system +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + teknoir.org/app: toe + app: toe + control-plane: controller-manager + spec: + priorityClassName: high-priority-toe + containers: + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + env: + - name: CLIENT_ID + value: "golden-device" + - name: TOE_PROJECT + value: "teknoir" + - name: TOE_DOMAIN + value: "example.test" + - name: TOE_IOT_REGISTRY + value: "golden-ns" + - name: TOE_DEVICE + value: "golden-device" + - name: TOE_CA_CERT + value: "/etc/teknoir/roots.pem" + - name: TOE_PRIVATE_KEY + value: "/etc/teknoir/rsa_private.pem" + volumeMounts: + - name: toe-volume + mountPath: /etc/teknoir + command: + - /manager + image: docker.io/tekn0ir/toe:toe_v4 + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: toe + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + serviceAccountName: teknoir-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - name: toe-volume + hostPath: + # directory location on host + path: "/etc/teknoir" + +EOL + +INSTALL_WATCHDOG=${INSTALL_WATCHDOG:-false} +if [ "${INSTALL_WATCHDOG}" = true ]; then + info "Install Teknoir Watchdog" + $SUDO mkdir -p /usr/local/lib/teknoir + $SUDO chmod 755 /usr/local/lib/teknoir + info "Install apt dependencies" + $SUDO apt install -y python3-pip libsystemd-dev + $SUDO apt autoremove --purge -y || true + $SUDO apt clean || true + $SUDO pip3 install -U pip netifaces kubernetes systemd-watchdog python-statemachine + + $SUDO tee /usr/local/lib/teknoir/watchdog.py > /dev/null << EOL +import argparse +import logging +import socket +import subprocess +import sys +import time +import yaml +import json + +import kubernetes +from kubernetes.stream import stream +from kubernetes.stream.ws_client import ERROR_CHANNEL +import systemd_watchdog +from statemachine import StateMachine, State + +INTERVAL = 60 +ERRORS = 20 + +class WatchdogSM(StateMachine): + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + super(WatchdogSM, self).__init__() + self.wait_count = 0 + + uninitialized = State('Uninitialized', initial=True) + + running = State('Idle') + checkping = State('CheckPing') + checktoe = State('CheckTOE') + checkkubernetesapi = State('CheckK8sAPI') + + reboot = State('Reboot') + + progress = uninitialized.to(running) + + run = running.to.itself() + + healthcheck = running.to(checkping) | \ + checkping.to(checktoe) | \ + checktoe.to(checkkubernetesapi) | \ + checkkubernetesapi.to(running) + + degress = reboot.to.itself() + + error = checkping.to(checktoe) | \ + checktoe.to(checkkubernetesapi) | \ + checkkubernetesapi.to(running) | \ + uninitialized.to.itself() | \ + reboot.to.itself() + + escalate = reboot.from_(checkping, checktoe, checkkubernetesapi) + + def on_run(self): + # self.logger.info(f'Healtcheck wait count: {self.wait_count}') + self.wait_count += 1 + + +class Watchdog(object): + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + self.sm = WatchdogSM(**kwargs) + self.e_checkping = 0 + self.e_checktoe = 0 + self.e_checkkubernetesapi = 0 + + def tick(self): + self.logger.info(self.sm.current_state.name) + + if self.sm.current_state == self.sm.uninitialized: + self.sm.progress() + + ####################################################################################################################### + + elif self.sm.current_state == self.sm.running: + if self.sm.wait_count >= self.interval: + self.sm.wait_count = 0 + self.sm.healthcheck() + else: + self.sm.run() + + elif self.sm.current_state == self.sm.checkping: + if self.wrap_action(self.checkping): + self.sm.healthcheck() + self.e_checkping = 0 + else: + self.e_checkping += 1 + self.logger.warning(f'{self.sm.current_state.name} error no {self.e_checkping}') + self.sm.escalate() if self.e_checkping >= self.errors else self.sm.error() + + elif self.sm.current_state == self.sm.checktoe: + if self.wrap_action(self.checktoe): + self.sm.healthcheck() + self.e_checktoe = 0 + else: + self.e_checktoe += 1 + self.logger.warning(f'{self.sm.current_state.name} error no {self.e_checktoe}') + self.sm.escalate() if self.e_checktoe >= self.errors else self.sm.error() + + elif self.sm.current_state == self.sm.checkkubernetesapi: + if self.wrap_action(self.checkkubernetesapi): + self.sm.healthcheck() + self.e_checkkubernetesapi = 0 + else: + self.e_checkkubernetesapi += 1 + self.logger.warning(f'{self.sm.current_state.name} error no {self.e_checkkubernetesapi}') + self.sm.escalate() if self.e_checkkubernetesapi >= self.errors else self.sm.error() + + ####################################################################################################################### + + elif self.sm.current_state == self.sm.reboot: + if self.wrap_action(self.reboot): + self.sm.degress() + else: + self.sm.error() + + def wrap_action(self, action): + try: + return action() + except: + e = sys.exc_info()[0] + self.logger.warning("Exception: %s" % e) + return False + + def reboot(self): + self.logger.warning(f'Rebooting the device') + proc = subprocess.run(['reboot'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + self.logger.warning(f'The exit code was: {proc.returncode}') + return False + return True + + def checkping(self): + try: + socket.create_connection(("1.1.1.1", 53)) + return True + except OSError: + pass + self.logger.warning('Cannot ping IP: 1.1.1.1 on port 53') + return False + + def checktoe(self): + kubernetes.config.load_kube_config("/etc/rancher/k3s/k3s.yaml") + try: + api_instance = kubernetes.client.CoreV1Api() + label_selector = "app=toe" + api_response = api_instance.list_pod_for_all_namespaces(watch=False, label_selector=label_selector) + toe = next(iter(api_response.items), None) + if toe == None: + self.logger.warning(f'There is no TOE POD') + return False + toe_container_status = next(iter(toe.status.container_statuses), None) + if toe_container_status == None: + self.logger.warning(f'There is no container status for TOE POD') + return False + self.logger.info(f'TOE POD NAME: {toe_container_status.name}, Ready: {str(toe_container_status.ready)}') + return toe_container_status.ready + except kubernetes.client.rest.ApiException as e: + self.logger.warning('Found exception in reading TOE pod') + return False + + def checkkubernetesapi(self): + kubernetes.config.load_kube_config("/etc/rancher/k3s/k3s.yaml") + try: + api_instance = kubernetes.client.CoreV1Api() + name = 'healtcheck' + namespace = 'kube-system' + resp = None + try: + resp = api_instance.read_namespaced_pod(name=name, namespace=namespace) + except kubernetes.client.rest.ApiException as e: + if e.status != 404: + logger.info(f"Unknown error: {e}") + return False + if not resp: + logger.info(f"Pod {name} does not exist. Creating it...") + pod_manifest = { + 'apiVersion': 'v1', + 'kind': 'Pod', + 'metadata': { + 'name': name + }, + 'spec': { + 'containers': [{ + 'image': 'busybox', + 'name': 'sleep', + "args": [ + "/bin/sh", + "-c", + "while true;do date;sleep 5; done" + ] + }] + } + } + resp = api_instance.create_namespaced_pod(body=pod_manifest, namespace=namespace) + while True: + resp = api_instance.read_namespaced_pod(name=name, namespace=namespace) + if resp.status.phase != 'Pending': + break + time.sleep(1) + logger.info(f"Pod {name} created") + + # Calling exec and waiting for response + exec_command = [ + 'nc', '-zvn', '-w', '2', 'kubernetes.default', '443'] + client = stream(api_instance.connect_get_namespaced_pod_exec, + name, + namespace, + command=exec_command, + stderr=True, stdin=False, + stdout=True, tty=False, _preload_content=False) + + client.run_forever(timeout=10) + err = client.read_channel(ERROR_CHANNEL) + result = yaml.safe_load(err) + if result['status'] != "Success": + logger.error(f'Failed to run command "{" ".join(exec_command)}"') + logger.error('Reason: ' + result['reason']) + logger.error('Message: ' + result['message']) + logger.error('Details: ' + ';'.join(map(lambda x: json.dumps(x), result['details']['causes']))) + return False + + except kubernetes.client.rest.ApiException as e: + logger.warning(f'Exception during healthcheck of kubernetes API: {e}') + return False + + return True + +def parse_args(): + # Parse input arguments + desc = 'Teknoir Device Watchdog' + parser = argparse.ArgumentParser(description=desc) + parser.add_argument('--interval', dest='interval', + help='Set interval in seconds between check', + default=INTERVAL, type=int) + parser.add_argument('--errors', dest='errors', + help='Set number of errors to reboot', + default=ERRORS, type=int) + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse_args() + logger = logging.getLogger('tnw') + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + formatter = logging.Formatter('[%(name)s] [%(levelname)s] %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + logger.setLevel(logging.INFO) + + logger.info("TΞꓘN01R") + logger.info("TΞꓘN01R") + logger.info("TΞꓘN01R") + logger.info("TΞꓘN01R") + + wd = systemd_watchdog.watchdog() + if not wd.is_enabled: + # Then it's probably not running is systemd with watchdog enabled + raise Exception("Watchdog not enabled") + + watchdog = Watchdog(logger=logger, wd=wd, **args.__dict__) + + # Report that the program init is complete + wd.ready() + wd.status("Init is complete...") + wd.notify() + + wd.status("Starting up watchdog...") + while True: + watchdog.tick() + time.sleep(1) + wd.notify() +EOL + $SUDO chmod 644 /usr/local/lib/teknoir/watchdog.py + + $SUDO tee /lib/systemd/system/tn-watchdog.service > /dev/null << EOL +[Unit] +Description=Teknoir Device Watchdog +After=network.target +StartLimitBurst=5 +StartLimitIntervalSec=600 + +[Service] +Type=simple +Restart=always +RestartSec=10 +ExecStart=/usr/bin/python3 /usr/local/lib/teknoir/watchdog.py --interval 60 --errors 20 +Type=notify +WatchdogSec=300 + +[Install] +WantedBy=multi-user.target +EOL + + + $SUDO chmod 644 /lib/systemd/system/tn-watchdog.service + info "Enable and start watchdog service" + if [ ${OS_BUILD} ]; then + info "OS Build specifics" + $SUDO ln -s /lib/systemd/system/tn-watchdog.service /etc/systemd/system/multi-user.target.wants/tn-watchdog.service + else + $SUDO systemctl enable --now tn-watchdog.service + fi +fi + +info "Device bootstrapped successfully!" + +info "" +info " _____ _____ _ ___ _ ___ ___ ____ " +info "|_ _| ____| |/ / \ | |/ _ \_ _| _ \ " +info " | | | _| | ' /| \| | | | | || |_) | " +info " | | | |___| . \| |\ | |_| | || _ < " +info " |_| |_____|_|\_\_| \_|\___/___|_| \_\ " +info "" + +info "Notice!" +info "No networking or user settings has been changed to match settings in the platform." + +warn "To enable tunneling, the user ${_FIRST_USER_NAME} has to be created." + +on_sudo() { + $SUDO sh -- "$@" +} + +setup_user() { + $SUDO mkdir -p "/home/${_FIRST_USER_NAME}" + if ! id -u "${_FIRST_USER_NAME}" >/dev/null 2>&1; then + if command -v useradd >/dev/null 2>&1 && command -v groupadd >/dev/null 2>&1; then + if ! getent group "${_FIRST_USER_NAME}" >/dev/null 2>&1; then + $SUDO groupadd "${_FIRST_USER_NAME}" + fi + $SUDO useradd \ + --gid "${_FIRST_USER_NAME}" \ + --groups users \ + --home-dir "/home/${_FIRST_USER_NAME}" \ + --shell /bin/bash \ + "${_FIRST_USER_NAME}" + else + $SUDO adduser --disabled-password --gecos "" "${_FIRST_USER_NAME}" + fi + fi + + if command -v chpasswd &> /dev/null; then + echo "${_FIRST_USER_NAME}:${_FIRST_USER_PASS}" | $SUDO chpasswd + elif command -v passwd &> /dev/null; then + echo -e "${_FIRST_USER_PASS}\n${_FIRST_USER_PASS}" | passwd "${_FIRST_USER_NAME}" + elif command -v usermod &> /dev/null && command -v openssl &> /dev/null; then + usermod --password $(openssl passwd -1 "${_FIRST_USER_PASS}") "${_FIRST_USER_NAME}" + else + warn "Could not set password for user ${_FIRST_USER_NAME}" + info "Skipping tunneling setup" + exit + fi + + on_sudo << EOF +mkdir -p "/home/${_FIRST_USER_NAME}/.ssh" +cp "${CONFIG_PATH}/rsa_private.pem" "/home/${_FIRST_USER_NAME}/.ssh/id_rsa" +ssh-keygen -y -f "${CONFIG_PATH}/rsa_private.pem" > "/home/${_FIRST_USER_NAME}/.ssh/id_rsa.pub" +cat "/home/${_FIRST_USER_NAME}/.ssh/id_rsa.pub" > "/home/${_FIRST_USER_NAME}/.ssh/authorized_keys" +if [ -z "\${_FIRST_USER_KEY+x}" ] || [ "\${_FIRST_USER_KEY}" = "" ]; then + echo "[INFO] _FIRST_USER_KEY is not set, will continue with only password auth"; +else + echo "\${_FIRST_USER_KEY}" >> "/home/${_FIRST_USER_NAME}/.ssh/authorized_keys" +fi +chmod 600 /home/${_FIRST_USER_NAME}/.ssh/* +chmod 744 /home/${_FIRST_USER_NAME}/.ssh +chown -R "${_FIRST_USER_NAME}:${_FIRST_USER_NAME}" "/home/${_FIRST_USER_NAME}" + +# Add root user to users and sudo groups +if [ -e /etc/group ]; then + sed -i "/^users:/s/\(.*\)/\1,${_FIRST_USER_NAME}/;s/:,/:/" /etc/group + sed -i "/^sudo:/s/\(.*\)/\1,${_FIRST_USER_NAME}/;s/:,/:/" /etc/group +fi +EOF +} + + +CREATE_USER=${CREATE_USER:-ask} +if [ "${OS_BUILD}" ] || [ "${CREATE_USER}" = true ]; then + setup_user +else + if id -u "${_FIRST_USER_NAME}" >/dev/null 2>&1; then + warn "The user ${_FIRST_USER_NAME} already exist! This operation will change the password and OVERWRITE ssh keys for this users." + fi + warn "Do you want to add \"${_FIRST_USER_NAME}\" as a user and enable tunneling? [yY]" + read REPLY + + case ${REPLY} in + [Yy]* ) + setup_user + ;; + * ) + info "Skipping tunneling setup" + ;; + esac +fi + + +cd .. +info "Device bootstrapped successfully!" + +