diff --git a/charts/openstack-hypervisor-operator/templates/deployment.yaml b/charts/openstack-hypervisor-operator/templates/deployment.yaml index 0452645e..a63e9696 100644 --- a/charts/openstack-hypervisor-operator/templates/deployment.yaml +++ b/charts/openstack-hypervisor-operator/templates/deployment.yaml @@ -43,6 +43,10 @@ spec: value: {{ quote .Values.controllerManager.manager.env.certificateNamespace }} - name: CERTIFICATE_ISSUER_NAME value: {{ quote .Values.controllerManager.manager.env.certificateIssuerName }} + - name: CERTIFICATE_INTERFACE_INCLUDE_REGEX + value: {{ quote .Values.controllerManager.manager.env.certificateInterfaceIncludeRegex }} + - name: CERTIFICATE_INTERFACE_EXCLUDE_REGEX + value: {{ quote .Values.controllerManager.manager.env.certificateInterfaceExcludeRegex }} - name: LABEL_SELECTOR value: {{ quote .Values.controllerManager.manager.env.labelSelector }} - name: AGENT_NAMESPACES diff --git a/charts/openstack-hypervisor-operator/values.yaml b/charts/openstack-hypervisor-operator/values.yaml index 22f61295..2901aa69 100644 --- a/charts/openstack-hypervisor-operator/values.yaml +++ b/charts/openstack-hypervisor-operator/values.yaml @@ -6,6 +6,8 @@ controllerManager: - --health-probe-bind-address=:8081 - --certificate-namespace=$(CERTIFICATE_NAMESPACE) - --certificate-issuer-name=$(CERTIFICATE_ISSUER_NAME) + - --certificate-interface-include-regex=$(CERTIFICATE_INTERFACE_INCLUDE_REGEX) + - --certificate-interface-exclude-regex=$(CERTIFICATE_INTERFACE_EXCLUDE_REGEX) - --label-selector=$(LABEL_SELECTOR) - --agent-namespaces=$(AGENT_NAMESPACES) - --eviction-concurrency=$(EVICTION_CONCURRENCY) @@ -18,6 +20,10 @@ controllerManager: env: certificateIssuerName: nova-hypervisor-agents-ca-issuer certificateNamespace: monsoon3 + # Go regexes selecting node interfaces (Calico Interfaces annotation) whose + # addresses are added to the certificate; the exclude regex wins over the include. + certificateInterfaceIncludeRegex: "^vlan_[0-9]+$" + certificateInterfaceExcludeRegex: "" labelSelector: "" agentNamespaces: "monsoon3" # Default maximum number of VM migrations to run concurrently while draining a hypervisor. diff --git a/cmd/main.go b/cmd/main.go index 41982a56..5e4bf315 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -87,6 +87,8 @@ func main() { var tlsOpts []func(*tls.Config) var certificateNamespace string var certificateIssuerName string + var certificateInterfaceIncludeRegex string + var certificateInterfaceExcludeRegex string flag.BoolVar(&version, "version", false, "Print the version and exit.") @@ -117,6 +119,14 @@ func main() { flag.StringVar(&certificateNamespace, "certificate-namespace", "monsoon3", "The namespace for the certificates. ") flag.StringVar(&certificateIssuerName, "certificate-issuer-name", "nova-hypervisor-agents-ca-issuer", "Name of the certificate issuer.") + flag.StringVar(&certificateInterfaceIncludeRegex, "certificate-interface-include-regex", `^vlan_[0-9]+$`, + "Regular expression matching node interface names from the projectcalico.org/Interfaces annotation "+ + "whose IP addresses are added to the hypervisor certificate. "+ + "An empty expression includes all interfaces.") + flag.StringVar(&certificateInterfaceExcludeRegex, "certificate-interface-exclude-regex", "", + "Regular expression matching node interface names whose IP addresses are excluded "+ + "from the hypervisor certificate. Applied after the include expression; "+ + "an empty expression excludes no interfaces.") opts := ctrlzap.Options{ Development: true, @@ -355,7 +365,8 @@ func main() { if err = (&controller.NodeCertificateController{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr, certificateNamespace, certificateIssuerName); err != nil { + }).SetupWithManager(mgr, certificateNamespace, certificateIssuerName, + certificateInterfaceIncludeRegex, certificateInterfaceExcludeRegex); err != nil { setupLog.Error(err, "unable to create controller", "certificate", "NodeCertificate") os.Exit(1) } diff --git a/internal/controller/node_certificate_controller.go b/internal/controller/node_certificate_controller.go index 8287c482..b95363e7 100644 --- a/internal/controller/node_certificate_controller.go +++ b/internal/controller/node_certificate_controller.go @@ -19,7 +19,10 @@ package controller import ( "context" + "encoding/json" "fmt" + "net/netip" + "regexp" "slices" "time" @@ -38,13 +41,67 @@ import ( const ( NodeCertificateControllerName = "certificate" + + annotationCalicoInterfaces = "projectcalico.org/Interfaces" ) +// calicoNodeInterface is a single entry of the interfaces annotation. +type calicoNodeInterface struct { + Name string `json:"name"` + Addresses []string `json:"addresses"` +} + +// parseRoutableIP parses a plain IP address into its canonical form, +// rejecting zoned and non-routable addresses. +func parseRoutableIP(address string) (netip.Addr, bool) { + ip, err := netip.ParseAddr(address) + if err != nil || ip.Zone() != "" { + return netip.Addr{}, false + } + + ip = ip.Unmap() + return ip, ip.IsGlobalUnicast() +} + +// addInterfaceAddresses adds the annotation's interface addresses to ipAddresses, filtered by name; the exclude regex wins. +func (r *NodeCertificateController) addInterfaceAddresses(ctx context.Context, node *corev1.Node, ipAddresses map[string]bool) { + rawAnnotation := node.Annotations[annotationCalicoInterfaces] + if rawAnnotation == "" { + return + } + + var interfaces []calicoNodeInterface + if err := json.Unmarshal([]byte(rawAnnotation), &interfaces); err != nil { + logger.FromContext(ctx).Error(err, "could not parse node annotation", "annotation", annotationCalicoInterfaces) + return + } + + for _, iface := range interfaces { + if r.includeInterfaceNames != nil && !r.includeInterfaceNames.MatchString(iface.Name) { + continue + } + if r.excludeInterfaceNames != nil && r.excludeInterfaceNames.MatchString(iface.Name) { + continue + } + + for _, address := range iface.Addresses { + if ip, ok := parseRoutableIP(address); ok { + ipAddresses[ip.String()] = true + } + } + } +} + type NodeCertificateController struct { k8sclient.Client Scheme *runtime.Scheme namespace string issuerName string + + // includeInterfaceNames selects interface names for the certificate; nil admits all. + includeInterfaceNames *regexp.Regexp + // excludeInterfaceNames drops interface names; nil excludes none. Wins over include. + excludeInterfaceNames *regexp.Regexp } func getSecretAndCertName(name string) (secretName, certName string) { @@ -73,10 +130,14 @@ func (r *NodeCertificateController) ensureCertificate(ctx context.Context, node case corev1.NodeHostName, corev1.NodeInternalDNS, corev1.NodeExternalDNS: dnsNameSet[addr.Address] = true case corev1.NodeInternalIP, corev1.NodeExternalIP: - ipAddressSet[addr.Address] = true + if ip, ok := parseRoutableIP(addr.Address); ok { + ipAddressSet[ip.String()] = true + } } } + r.addInterfaceAddresses(ctx, node, ipAddressSet) + ipAddresses := make([]string, 0, len(ipAddressSet)) for k := range ipAddressSet { ipAddresses = append(ipAddresses, k) @@ -156,7 +217,28 @@ func (r *NodeCertificateController) Reconcile(ctx context.Context, req ctrl.Requ } // SetupWithManager sets up the controller with the Manager. -func (r *NodeCertificateController) SetupWithManager(mgr ctrl.Manager, namespace, issuerName string) error { +func (r *NodeCertificateController) SetupWithManager( + mgr ctrl.Manager, namespace, issuerName, interfaceIncludeRegex, interfaceExcludeRegex string, +) error { + + if interfaceIncludeRegex != "" { + includeRegex, err := regexp.Compile(interfaceIncludeRegex) + if err != nil { + return fmt.Errorf("invalid interface include regex %q: %w", interfaceIncludeRegex, err) + } + + r.includeInterfaceNames = includeRegex + } + + if interfaceExcludeRegex != "" { + excludeRegex, err := regexp.Compile(interfaceExcludeRegex) + if err != nil { + return fmt.Errorf("invalid interface exclude regex %q: %w", interfaceExcludeRegex, err) + } + + r.excludeInterfaceNames = excludeRegex + } + r.namespace = namespace r.issuerName = issuerName diff --git a/internal/controller/node_certificate_controller_test.go b/internal/controller/node_certificate_controller_test.go index e55482c4..35879b18 100644 --- a/internal/controller/node_certificate_controller_test.go +++ b/internal/controller/node_certificate_controller_test.go @@ -18,6 +18,8 @@ limitations under the License. package controller import ( + "regexp" + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -185,4 +187,133 @@ var _ = Describe("Node Certificate Controller", func() { Expect(err).NotTo(HaveOccurred()) }) }) + + Context("When reconciling a node with the Calico interfaces annotation", func() { + // updateCalicoAnnotation sets the projectcalico.org/Interfaces annotation on the node. + updateCalicoAnnotation := func(ctx SpecContext, value string) { + node := &corev1.Node{} + Expect(fakeClient.Get(ctx, types.NamespacedName{Name: nodeName}, node)).To(Succeed()) + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + node.Annotations[annotationCalicoInterfaces] = value + Expect(fakeClient.Update(ctx, node)).To(Succeed()) + } + + // setNodeInternalIP sets the node status InternalIP. + setNodeInternalIP := func(ctx SpecContext, ip string) { + node := &corev1.Node{} + Expect(fakeClient.Get(ctx, types.NamespacedName{Name: nodeName}, node)).To(Succeed()) + node.Status.Addresses = []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: ip}} + Expect(fakeClient.Status().Update(ctx, node)).To(Succeed()) + } + + // reconcileNode reconciles the node under test and expects no error. + reconcileNode := func(ctx SpecContext) { + _, err := nodeCertificateController.Reconcile(ctx, ctrl.Request{Name: nodeName}) + Expect(err).NotTo(HaveOccurred()) + } + + // fetchCertificate returns the certificate created for the node under test. + fetchCertificate := func(ctx SpecContext) *cmapi.Certificate { + _, certName := getSecretAndCertName(nodeName) + certificate := &cmapi.Certificate{} + Expect(fakeClient.Get(ctx, types.NamespacedName{Name: certName, Namespace: namespace}, certificate)).To(Succeed()) + return certificate + } + + type annotationCase struct { + includeRegex string + excludeRegex string + annotation string + statusIP string + expectedIPs []string + } + + DescribeTable("computing the certificate IP addresses from the Calico annotation", + func(ctx SpecContext, testCase annotationCase) { + if testCase.includeRegex != "" { + nodeCertificateController.includeInterfaceNames = regexp.MustCompile(testCase.includeRegex) + } + if testCase.excludeRegex != "" { + nodeCertificateController.excludeInterfaceNames = regexp.MustCompile(testCase.excludeRegex) + } + + updateCalicoAnnotation(ctx, testCase.annotation) + if testCase.statusIP != "" { + setNodeInternalIP(ctx, testCase.statusIP) + } + + reconcileNode(ctx) + + certificate := fetchCertificate(ctx) + Expect(certificate.Spec.IPAddresses).To(Equal(testCase.expectedIPs)) + }, + Entry("includes only interfaces matching the include regex", + annotationCase{ + includeRegex: `^vlan_[0-9]+$`, + annotation: `[{"name":"vlan_100","addresses":["10.10.0.5"]},{"name":"enp1s0","addresses":["10.0.0.9"]},{"name":"vlan_abc","addresses":["10.10.0.6"]}]`, + statusIP: "10.0.0.1", + expectedIPs: []string{"10.0.0.1", "10.10.0.5"}, + }), + Entry("drops interfaces matching the exclude regex even if they are included", + annotationCase{ + includeRegex: `^vlan_.*$`, + excludeRegex: `^vlan_100$`, + annotation: `[{"name":"vlan_100","addresses":["10.10.0.5"]},{"name":"vlan_200","addresses":["10.10.0.6"]}]`, + expectedIPs: []string{"10.10.0.6"}, + }), + Entry("merges annotation addresses with status addresses and deduplicates", + annotationCase{ + includeRegex: `^vlan_[0-9]+$`, + annotation: `[{"name":"vlan_100","addresses":["10.0.0.1","10.10.0.5"]}]`, + statusIP: "10.0.0.1", + expectedIPs: []string{"10.0.0.1", "10.10.0.5"}, + }), + Entry("skips non-routable and invalid addresses", + annotationCase{ + includeRegex: `.*`, + annotation: `[{"name":"vlan_100","addresses":["not-an-ip","fe80::1","127.0.0.1","10.10.0.5"]}]`, + expectedIPs: []string{"10.10.0.5"}, + }), + Entry("still creates the certificate when the annotation is malformed JSON", + annotationCase{ + includeRegex: `^vlan_[0-9]+$`, + annotation: `this-is-not-json`, + statusIP: "10.0.0.1", + expectedIPs: []string{"10.0.0.1"}, + }), + ) + + It("should remove annotation addresses from the certificate when the annotation is deleted", func(ctx SpecContext) { + By("Configuring an include filter for numbered vlan interfaces") + nodeCertificateController.includeInterfaceNames = regexp.MustCompile(`^vlan_[0-9]+$`) + + By("Annotating the node with Calico interfaces") + updateCalicoAnnotation(ctx, `[{"name":"vlan_100","addresses":["10.10.0.5"]}]`) + + By("Setting the node status InternalIP") + setNodeInternalIP(ctx, "10.0.0.1") + + By("Reconciling the node") + reconcileNode(ctx) + + By("Checking that both addresses are present") + certificate := fetchCertificate(ctx) + Expect(certificate.Spec.IPAddresses).To(Equal([]string{"10.0.0.1", "10.10.0.5"})) + + By("Deleting the Calico interfaces annotation from the node") + node := &corev1.Node{} + Expect(fakeClient.Get(ctx, types.NamespacedName{Name: nodeName}, node)).To(Succeed()) + delete(node.Annotations, annotationCalicoInterfaces) + Expect(fakeClient.Update(ctx, node)).To(Succeed()) + + By("Reconciling the node again") + reconcileNode(ctx) + + By("Checking that the annotation IP is removed and the status IP is retained") + certificate = fetchCertificate(ctx) + Expect(certificate.Spec.IPAddresses).To(Equal([]string{"10.0.0.1"})) + }) + }) })