From f118bffb0fc0c4ea1f0f8e8a69ea6d9c1a78b8d4 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Wed, 28 Oct 2026 12:49:54 +0100 Subject: [PATCH] Add interface IPs from Calico annotation to node certificates The certificate only covered the node's status addresses, so connections over other host interfaces (e.g. vlan networks reported by Calico) failed TLS validation. Which interfaces to admit is configurable via --certificate-interface-{include,exclude}-regex. Signed-off-by: Fabian Wiesel --- .../templates/deployment.yaml | 4 + .../openstack-hypervisor-operator/values.yaml | 6 + cmd/main.go | 13 +- .../controller/node_certificate_controller.go | 86 +++++++++++- .../node_certificate_controller_test.go | 131 ++++++++++++++++++ 5 files changed, 237 insertions(+), 3 deletions(-) diff --git a/charts/openstack-hypervisor-operator/templates/deployment.yaml b/charts/openstack-hypervisor-operator/templates/deployment.yaml index 0452645e..a63e9696 100644 --- a/charts/openstack-hypervisor-operator/templates/deployment.yaml +++ b/charts/openstack-hypervisor-operator/templates/deployment.yaml @@ -43,6 +43,10 @@ spec: value: {{ quote .Values.controllerManager.manager.env.certificateNamespace }} - name: CERTIFICATE_ISSUER_NAME value: {{ quote .Values.controllerManager.manager.env.certificateIssuerName }} + - name: CERTIFICATE_INTERFACE_INCLUDE_REGEX + value: {{ quote .Values.controllerManager.manager.env.certificateInterfaceIncludeRegex }} + - name: CERTIFICATE_INTERFACE_EXCLUDE_REGEX + value: {{ quote .Values.controllerManager.manager.env.certificateInterfaceExcludeRegex }} - name: LABEL_SELECTOR value: {{ quote .Values.controllerManager.manager.env.labelSelector }} - name: AGENT_NAMESPACES diff --git a/charts/openstack-hypervisor-operator/values.yaml b/charts/openstack-hypervisor-operator/values.yaml index 22f61295..2901aa69 100644 --- a/charts/openstack-hypervisor-operator/values.yaml +++ b/charts/openstack-hypervisor-operator/values.yaml @@ -6,6 +6,8 @@ controllerManager: - --health-probe-bind-address=:8081 - --certificate-namespace=$(CERTIFICATE_NAMESPACE) - --certificate-issuer-name=$(CERTIFICATE_ISSUER_NAME) + - --certificate-interface-include-regex=$(CERTIFICATE_INTERFACE_INCLUDE_REGEX) + - --certificate-interface-exclude-regex=$(CERTIFICATE_INTERFACE_EXCLUDE_REGEX) - --label-selector=$(LABEL_SELECTOR) - --agent-namespaces=$(AGENT_NAMESPACES) - --eviction-concurrency=$(EVICTION_CONCURRENCY) @@ -18,6 +20,10 @@ controllerManager: env: certificateIssuerName: nova-hypervisor-agents-ca-issuer certificateNamespace: monsoon3 + # Go regexes selecting node interfaces (Calico Interfaces annotation) whose + # addresses are added to the certificate; the exclude regex wins over the include. + certificateInterfaceIncludeRegex: "^vlan_[0-9]+$" + certificateInterfaceExcludeRegex: "" labelSelector: "" agentNamespaces: "monsoon3" # Default maximum number of VM migrations to run concurrently while draining a hypervisor. diff --git a/cmd/main.go b/cmd/main.go index 41982a56..5e4bf315 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -87,6 +87,8 @@ func main() { var tlsOpts []func(*tls.Config) var certificateNamespace string var certificateIssuerName string + var certificateInterfaceIncludeRegex string + var certificateInterfaceExcludeRegex string flag.BoolVar(&version, "version", false, "Print the version and exit.") @@ -117,6 +119,14 @@ func main() { flag.StringVar(&certificateNamespace, "certificate-namespace", "monsoon3", "The namespace for the certificates. ") flag.StringVar(&certificateIssuerName, "certificate-issuer-name", "nova-hypervisor-agents-ca-issuer", "Name of the certificate issuer.") + flag.StringVar(&certificateInterfaceIncludeRegex, "certificate-interface-include-regex", `^vlan_[0-9]+$`, + "Regular expression matching node interface names from the projectcalico.org/Interfaces annotation "+ + "whose IP addresses are added to the hypervisor certificate. "+ + "An empty expression includes all interfaces.") + flag.StringVar(&certificateInterfaceExcludeRegex, "certificate-interface-exclude-regex", "", + "Regular expression matching node interface names whose IP addresses are excluded "+ + "from the hypervisor certificate. Applied after the include expression; "+ + "an empty expression excludes no interfaces.") opts := ctrlzap.Options{ Development: true, @@ -355,7 +365,8 @@ func main() { if err = (&controller.NodeCertificateController{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr, certificateNamespace, certificateIssuerName); err != nil { + }).SetupWithManager(mgr, certificateNamespace, certificateIssuerName, + certificateInterfaceIncludeRegex, certificateInterfaceExcludeRegex); err != nil { setupLog.Error(err, "unable to create controller", "certificate", "NodeCertificate") os.Exit(1) } diff --git a/internal/controller/node_certificate_controller.go b/internal/controller/node_certificate_controller.go index 8287c482..b95363e7 100644 --- a/internal/controller/node_certificate_controller.go +++ b/internal/controller/node_certificate_controller.go @@ -19,7 +19,10 @@ package controller import ( "context" + "encoding/json" "fmt" + "net/netip" + "regexp" "slices" "time" @@ -38,13 +41,67 @@ import ( const ( NodeCertificateControllerName = "certificate" + + annotationCalicoInterfaces = "projectcalico.org/Interfaces" ) +// calicoNodeInterface is a single entry of the interfaces annotation. +type calicoNodeInterface struct { + Name string `json:"name"` + Addresses []string `json:"addresses"` +} + +// parseRoutableIP parses a plain IP address into its canonical form, +// rejecting zoned and non-routable addresses. +func parseRoutableIP(address string) (netip.Addr, bool) { + ip, err := netip.ParseAddr(address) + if err != nil || ip.Zone() != "" { + return netip.Addr{}, false + } + + ip = ip.Unmap() + return ip, ip.IsGlobalUnicast() +} + +// addInterfaceAddresses adds the annotation's interface addresses to ipAddresses, filtered by name; the exclude regex wins. +func (r *NodeCertificateController) addInterfaceAddresses(ctx context.Context, node *corev1.Node, ipAddresses map[string]bool) { + rawAnnotation := node.Annotations[annotationCalicoInterfaces] + if rawAnnotation == "" { + return + } + + var interfaces []calicoNodeInterface + if err := json.Unmarshal([]byte(rawAnnotation), &interfaces); err != nil { + logger.FromContext(ctx).Error(err, "could not parse node annotation", "annotation", annotationCalicoInterfaces) + return + } + + for _, iface := range interfaces { + if r.includeInterfaceNames != nil && !r.includeInterfaceNames.MatchString(iface.Name) { + continue + } + if r.excludeInterfaceNames != nil && r.excludeInterfaceNames.MatchString(iface.Name) { + continue + } + + for _, address := range iface.Addresses { + if ip, ok := parseRoutableIP(address); ok { + ipAddresses[ip.String()] = true + } + } + } +} + type NodeCertificateController struct { k8sclient.Client Scheme *runtime.Scheme namespace string issuerName string + + // includeInterfaceNames selects interface names for the certificate; nil admits all. + includeInterfaceNames *regexp.Regexp + // excludeInterfaceNames drops interface names; nil excludes none. Wins over include. + excludeInterfaceNames *regexp.Regexp } func getSecretAndCertName(name string) (secretName, certName string) { @@ -73,10 +130,14 @@ func (r *NodeCertificateController) ensureCertificate(ctx context.Context, node case corev1.NodeHostName, corev1.NodeInternalDNS, corev1.NodeExternalDNS: dnsNameSet[addr.Address] = true case corev1.NodeInternalIP, corev1.NodeExternalIP: - ipAddressSet[addr.Address] = true + if ip, ok := parseRoutableIP(addr.Address); ok { + ipAddressSet[ip.String()] = true + } } } + r.addInterfaceAddresses(ctx, node, ipAddressSet) + ipAddresses := make([]string, 0, len(ipAddressSet)) for k := range ipAddressSet { ipAddresses = append(ipAddresses, k) @@ -156,7 +217,28 @@ func (r *NodeCertificateController) Reconcile(ctx context.Context, req ctrl.Requ } // SetupWithManager sets up the controller with the Manager. -func (r *NodeCertificateController) SetupWithManager(mgr ctrl.Manager, namespace, issuerName string) error { +func (r *NodeCertificateController) SetupWithManager( + mgr ctrl.Manager, namespace, issuerName, interfaceIncludeRegex, interfaceExcludeRegex string, +) error { + + if interfaceIncludeRegex != "" { + includeRegex, err := regexp.Compile(interfaceIncludeRegex) + if err != nil { + return fmt.Errorf("invalid interface include regex %q: %w", interfaceIncludeRegex, err) + } + + r.includeInterfaceNames = includeRegex + } + + if interfaceExcludeRegex != "" { + excludeRegex, err := regexp.Compile(interfaceExcludeRegex) + if err != nil { + return fmt.Errorf("invalid interface exclude regex %q: %w", interfaceExcludeRegex, err) + } + + r.excludeInterfaceNames = excludeRegex + } + r.namespace = namespace r.issuerName = issuerName diff --git a/internal/controller/node_certificate_controller_test.go b/internal/controller/node_certificate_controller_test.go index e55482c4..35879b18 100644 --- a/internal/controller/node_certificate_controller_test.go +++ b/internal/controller/node_certificate_controller_test.go @@ -18,6 +18,8 @@ limitations under the License. package controller import ( + "regexp" + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -185,4 +187,133 @@ var _ = Describe("Node Certificate Controller", func() { Expect(err).NotTo(HaveOccurred()) }) }) + + Context("When reconciling a node with the Calico interfaces annotation", func() { + // updateCalicoAnnotation sets the projectcalico.org/Interfaces annotation on the node. + updateCalicoAnnotation := func(ctx SpecContext, value string) { + node := &corev1.Node{} + Expect(fakeClient.Get(ctx, types.NamespacedName{Name: nodeName}, node)).To(Succeed()) + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + node.Annotations[annotationCalicoInterfaces] = value + Expect(fakeClient.Update(ctx, node)).To(Succeed()) + } + + // setNodeInternalIP sets the node status InternalIP. + setNodeInternalIP := func(ctx SpecContext, ip string) { + node := &corev1.Node{} + Expect(fakeClient.Get(ctx, types.NamespacedName{Name: nodeName}, node)).To(Succeed()) + node.Status.Addresses = []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: ip}} + Expect(fakeClient.Status().Update(ctx, node)).To(Succeed()) + } + + // reconcileNode reconciles the node under test and expects no error. + reconcileNode := func(ctx SpecContext) { + _, err := nodeCertificateController.Reconcile(ctx, ctrl.Request{Name: nodeName}) + Expect(err).NotTo(HaveOccurred()) + } + + // fetchCertificate returns the certificate created for the node under test. + fetchCertificate := func(ctx SpecContext) *cmapi.Certificate { + _, certName := getSecretAndCertName(nodeName) + certificate := &cmapi.Certificate{} + Expect(fakeClient.Get(ctx, types.NamespacedName{Name: certName, Namespace: namespace}, certificate)).To(Succeed()) + return certificate + } + + type annotationCase struct { + includeRegex string + excludeRegex string + annotation string + statusIP string + expectedIPs []string + } + + DescribeTable("computing the certificate IP addresses from the Calico annotation", + func(ctx SpecContext, testCase annotationCase) { + if testCase.includeRegex != "" { + nodeCertificateController.includeInterfaceNames = regexp.MustCompile(testCase.includeRegex) + } + if testCase.excludeRegex != "" { + nodeCertificateController.excludeInterfaceNames = regexp.MustCompile(testCase.excludeRegex) + } + + updateCalicoAnnotation(ctx, testCase.annotation) + if testCase.statusIP != "" { + setNodeInternalIP(ctx, testCase.statusIP) + } + + reconcileNode(ctx) + + certificate := fetchCertificate(ctx) + Expect(certificate.Spec.IPAddresses).To(Equal(testCase.expectedIPs)) + }, + Entry("includes only interfaces matching the include regex", + annotationCase{ + includeRegex: `^vlan_[0-9]+$`, + annotation: `[{"name":"vlan_100","addresses":["10.10.0.5"]},{"name":"enp1s0","addresses":["10.0.0.9"]},{"name":"vlan_abc","addresses":["10.10.0.6"]}]`, + statusIP: "10.0.0.1", + expectedIPs: []string{"10.0.0.1", "10.10.0.5"}, + }), + Entry("drops interfaces matching the exclude regex even if they are included", + annotationCase{ + includeRegex: `^vlan_.*$`, + excludeRegex: `^vlan_100$`, + annotation: `[{"name":"vlan_100","addresses":["10.10.0.5"]},{"name":"vlan_200","addresses":["10.10.0.6"]}]`, + expectedIPs: []string{"10.10.0.6"}, + }), + Entry("merges annotation addresses with status addresses and deduplicates", + annotationCase{ + includeRegex: `^vlan_[0-9]+$`, + annotation: `[{"name":"vlan_100","addresses":["10.0.0.1","10.10.0.5"]}]`, + statusIP: "10.0.0.1", + expectedIPs: []string{"10.0.0.1", "10.10.0.5"}, + }), + Entry("skips non-routable and invalid addresses", + annotationCase{ + includeRegex: `.*`, + annotation: `[{"name":"vlan_100","addresses":["not-an-ip","fe80::1","127.0.0.1","10.10.0.5"]}]`, + expectedIPs: []string{"10.10.0.5"}, + }), + Entry("still creates the certificate when the annotation is malformed JSON", + annotationCase{ + includeRegex: `^vlan_[0-9]+$`, + annotation: `this-is-not-json`, + statusIP: "10.0.0.1", + expectedIPs: []string{"10.0.0.1"}, + }), + ) + + It("should remove annotation addresses from the certificate when the annotation is deleted", func(ctx SpecContext) { + By("Configuring an include filter for numbered vlan interfaces") + nodeCertificateController.includeInterfaceNames = regexp.MustCompile(`^vlan_[0-9]+$`) + + By("Annotating the node with Calico interfaces") + updateCalicoAnnotation(ctx, `[{"name":"vlan_100","addresses":["10.10.0.5"]}]`) + + By("Setting the node status InternalIP") + setNodeInternalIP(ctx, "10.0.0.1") + + By("Reconciling the node") + reconcileNode(ctx) + + By("Checking that both addresses are present") + certificate := fetchCertificate(ctx) + Expect(certificate.Spec.IPAddresses).To(Equal([]string{"10.0.0.1", "10.10.0.5"})) + + By("Deleting the Calico interfaces annotation from the node") + node := &corev1.Node{} + Expect(fakeClient.Get(ctx, types.NamespacedName{Name: nodeName}, node)).To(Succeed()) + delete(node.Annotations, annotationCalicoInterfaces) + Expect(fakeClient.Update(ctx, node)).To(Succeed()) + + By("Reconciling the node again") + reconcileNode(ctx) + + By("Checking that the annotation IP is removed and the status IP is retained") + certificate = fetchCertificate(ctx) + Expect(certificate.Spec.IPAddresses).To(Equal([]string{"10.0.0.1"})) + }) + }) })