feat: support custom labels on versioned Deployments

api/v1alpha1/workerdeployment_cel_validation_test.go

@@ -64,6 +64,22 @@ var _ = Describe("WorkerDeployment CRD CEL validation", func() {
 		Expect(err.Error()).To(ContainSubstring("spec.drainingDeployment.replicas"))
 	})
 
+	It("accepts custom deploymentLabels", func() {
+		twd := baseTWD("custom-deployment-labels")
+		twd.Spec.DeploymentLabels = map[string]string{"team": "payments"}
+		Expect(k8sClient.Create(ctx, twd)).To(Succeed())
+	})
+
+	It("rejects deploymentLabels that set controller-managed keys", func() {
+		for i, key := range []string{"temporal.io/build-id", "temporal.io/deployment-name"} {
+			twd := baseTWD(fmt.Sprintf("reserved-deployment-label-%d", i))
+			twd.Spec.DeploymentLabels = map[string]string{key: "user-supplied"}
+			err := k8sClient.Create(ctx, twd)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("are managed by the controller"))
+		}
+	})
+
 	It("rejects name longer than 63 characters", func() {
 		twd := baseTWD(strings.Repeat("a", 64))
 		err := k8sClient.Create(ctx, twd)

api/v1alpha1/workerdeployment_types.go

@@ -103,6 +103,16 @@ type WorkerDeploymentSpec struct {
 	// +optional
 	DrainingDeployment *DrainingDeploymentSpec `json:"drainingDeployment,omitempty"`
 
+	// DeploymentLabels are added to the metadata of each versioned Kubernetes
+	// Deployment that the controller creates. Pod labels belong in the pod
+	// template instead. Use this for tools that act on the Deployment object
+	// itself, such as admission webhooks that opt workloads in or out by label.
+	// Changes are applied in place to the Deployments of existing versions,
+	// without restarting their pods.
+	// +optional
+	// +kubebuilder:validation:XValidation:rule="!('temporal.io/build-id' in self) && !('temporal.io/deployment-name' in self)",message="temporal.io/build-id and temporal.io/deployment-name are managed by the controller"
+	DeploymentLabels map[string]string `json:"deploymentLabels,omitempty"`
+
 	// How to rollout new workflow executions to the target version.
 	RolloutStrategy RolloutStrategy `json:"rollout"`
 

api/v1alpha1/workerdeployment_webhook.go

@@ -11,6 +11,7 @@ import (
 	"github.com/temporalio/temporal-worker-controller/internal/defaults"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -82,6 +83,7 @@ func (r *WorkerDeployment) validateForUpdateOrCreate(ctx context.Context, obj ru
 
 func validateForUpdateOrCreate(dep *WorkerDeployment) (admission.Warnings, error) {
 	allErrs := validateRolloutStrategy(dep.Spec.RolloutStrategy)
+	allErrs = append(allErrs, metav1validation.ValidateLabels(dep.Spec.DeploymentLabels, field.NewPath("spec.deploymentLabels"))...)
 	if len(allErrs) > 0 {
 		return nil, newInvalidErr(dep, allErrs)
 	}

api/v1alpha1/workerdeployment_webhook_test.go

@@ -128,6 +128,28 @@ func TestWorkerDeployment_ValidateCreate(t *testing.T) {
 		"valid temporal worker deployment": {
 			obj: testhelpers.MakeWDWithName("valid-worker", ""),
 		},
+		"deploymentLabels with valid keys and values": {
+			obj: testhelpers.ModifyObj(testhelpers.MakeWDWithName("valid-deployment-labels", ""), func(obj *temporaliov1alpha1.WorkerDeployment) *temporaliov1alpha1.WorkerDeployment {
+				obj.Spec.DeploymentLabels = map[string]string{"team": "payments", "example.com/instrumentation": "false", "empty": ""}
+				return obj
+			}),
+		},
+		// Without this check the CRD accepts the label and the controller then fails
+		// to create the versioned Deployment, which stalls the rollout.
+		"deploymentLabels with an invalid value": {
+			obj: testhelpers.ModifyObj(testhelpers.MakeWDWithName("invalid-deployment-label-value", ""), func(obj *temporaliov1alpha1.WorkerDeployment) *temporaliov1alpha1.WorkerDeployment {
+				obj.Spec.DeploymentLabels = map[string]string{"team": "foo bar"}
+				return obj
+			}),
+			errorMsg: `spec.deploymentLabels: Invalid value: "foo bar"`,
+		},
+		"deploymentLabels with an invalid key": {
+			obj: testhelpers.ModifyObj(testhelpers.MakeWDWithName("invalid-deployment-label-key", ""), func(obj *temporaliov1alpha1.WorkerDeployment) *temporaliov1alpha1.WorkerDeployment {
+				obj.Spec.DeploymentLabels = map[string]string{"-team": "payments"}
+				return obj
+			}),
+			errorMsg: `spec.deploymentLabels: Invalid value: "-team"`,
+		},
 		"invalid object type": {
 			obj: &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{

docs/configuration.md

@@ -600,6 +600,31 @@ example after a rollback, scales back to `deployment.replicas`.
 > `terminationGracePeriodSeconds`. Long-running activities should heartbeat so
 > they resume on another pod.
 
+### Labels on versioned Deployments
+
+Labels in `spec.deployment.template.metadata.labels` go on the pods. To label the
+versioned Kubernetes Deployment objects themselves, set `spec.deploymentLabels`.
+Use this for tools that act on the Deployment rather than its pods, such as
+admission webhooks that opt workloads in or out by label, or cost allocation.
+
+```yaml
+spec:
+  deploymentLabels:
+    team: payments
+    example.com/instrumentation: "false"
+```
+
+The controller keeps these labels in sync on the Deployments of every version it
+manages, including Draining ones. A change is applied in place: it doesn't create
+a new Build ID or restart pods. Removing a key from `spec.deploymentLabels` removes
+it from the Deployments. Labels that other tools add to the Deployments are left
+alone; the controller tracks the keys it applied in the
+`temporal.io/managed-deployment-labels` annotation.
+
+`temporal.io/build-id` and `temporal.io/deployment-name` are managed by the
+controller and cannot be set. Keys and values must be valid Kubernetes labels;
+the validating webhook rejects invalid ones.
+
 ### Environment-Specific Configurations
 
 **Production Configuration:**

helm/temporal-worker-controller-crds/templates/temporal.io_workerdeployments.yaml

@@ -3927,6 +3927,15 @@ spec:
                 required:
                 - template
                 type: object
+              deploymentLabels:
+                additionalProperties:
+                  type: string
+                type: object
+                x-kubernetes-validations:
+                - message: temporal.io/build-id and temporal.io/deployment-name are
+                    managed by the controller
+                  rule: '!(''temporal.io/build-id'' in self) && !(''temporal.io/deployment-name''
+                    in self)'
               drainingDeployment:
                 properties:
                   replicas:

internal/k8s/deployments.go

@@ -10,6 +10,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"regexp"
 	"slices"
 	"sort"
@@ -44,6 +45,10 @@ const (
 	MaxDeploymentNameLen                           = 47
 	ConnectionSpecHashAnnotation                   = "temporal.io/connection-spec-hash"
 	PodTemplateSpecHashAnnotation                  = "temporal.io/pod-template-spec-hash"
+	// ManagedDeploymentLabelsAnnotation lists the spec.deploymentLabels keys the controller
+	// applied, so a key removed from the spec can be removed without touching labels that
+	// other tools put on the Deployment.
+	ManagedDeploymentLabelsAnnotation = "temporal.io/managed-deployment-labels"
 
 	// Environment variables read by the Temporal Go SDK's envconfig package
 	// (go.temporal.io/sdk/contrib/envconfig) to configure the worker's connection.
@@ -317,13 +322,13 @@ func NewDeploymentWithOwnerRef(
 	)
 	depSpec.Template.Spec = *podSpec
 
-	return &appsv1.Deployment{
+	deployment := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:                       ComputeVersionedDeploymentName(objectMeta.Name, buildID),
 			Namespace:                  objectMeta.Namespace,
 			DeletionGracePeriodSeconds: nil,
-			Labels:                     selectorLabels,
-			Annotations:                depSpec.Template.Annotations,
+			Labels:                     maps.Clone(selectorLabels),
+			Annotations:                maps.Clone(depSpec.Template.Annotations),
 			OwnerReferences: []metav1.OwnerReference{{
 				APIVersion:         typeMeta.APIVersion,
 				Kind:               typeMeta.Kind,
@@ -337,6 +342,63 @@ func NewDeploymentWithOwnerRef(
 		},
 		Spec: depSpec,
 	}
+	ApplyDeploymentLabels(deployment, spec.DeploymentLabels)
+	return deployment
+}
+
+// ApplyDeploymentLabels brings the Deployment's labels in line with spec.deploymentLabels
+// and reports whether anything changed. It removes only keys it applied earlier, so labels
+// added by other tools survive, and it never changes the selector labels.
+func ApplyDeploymentLabels(deployment *appsv1.Deployment, desired map[string]string) bool {
+	isSelectorLabel := func(key string) bool {
+		if deployment.Spec.Selector == nil {
+			return false
+		}
+		_, ok := deployment.Spec.Selector.MatchLabels[key]
+		return ok
+	}
+
+	changed := false
+	for _, key := range strings.Split(deployment.Annotations[ManagedDeploymentLabelsAnnotation], ",") {
+		if _, keep := desired[key]; keep || key == "" || isSelectorLabel(key) {
+			continue
+		}
+		if _, ok := deployment.Labels[key]; ok {
+			delete(deployment.Labels, key)
+			changed = true
+		}
+	}
+
+	managed := make([]string, 0, len(desired))
+	for key, value := range desired {
+		if isSelectorLabel(key) {
+			continue
+		}
+		managed = append(managed, key)
+		if current, ok := deployment.Labels[key]; ok && current == value {
+			continue
+		}
+		if deployment.Labels == nil {
+			deployment.Labels = map[string]string{}
+		}
+		deployment.Labels[key] = value
+		changed = true
+	}
+
+	sort.Strings(managed)
+	managedValue := strings.Join(managed, ",")
+	if deployment.Annotations[ManagedDeploymentLabelsAnnotation] != managedValue {
+		if managedValue == "" {
+			delete(deployment.Annotations, ManagedDeploymentLabelsAnnotation)
+		} else {
+			if deployment.Annotations == nil {
+				deployment.Annotations = map[string]string{}
+			}
+			deployment.Annotations[ManagedDeploymentLabelsAnnotation] = managedValue
+		}
+		changed = true
+	}
+	return changed
 }
 
 // TODO (Shivam): Change hash when secret name is updated as well.

internal/k8s/deployments_test.go

@@ -13,6 +13,7 @@ import (
 	"crypto/x509/pkix"
 	"encoding/pem"
 	"fmt"
+	"maps"
 	"math/big"
 	"os"
 	"path/filepath"
@@ -586,14 +587,105 @@ func TestNewDeploymentWithOwnerRef_Labels(t *testing.T) {
 	deployment := k8s.NewDeploymentWithOwnerRef(
 		&metav1.TypeMeta{},
 		&metav1.ObjectMeta{Name: "test-worker", Namespace: "default"},
-		&temporaliov1alpha1.WorkerDeploymentSpec{},
+		&temporaliov1alpha1.WorkerDeploymentSpec{
+			DeploymentLabels: map[string]string{"team": "payments"},
+		},
 		"test-deployment",
 		"build123",
 		temporaliov1alpha1.ConnectionSpec{},
 	)
 
 	assert.Equal(t, "test-worker", deployment.Labels[k8s.WorkerDeploymentNameLabel])
 	assert.Equal(t, "build123", deployment.Labels[k8s.BuildIDLabel])
+	assert.Equal(t, "payments", deployment.Labels["team"])
+	assert.NotContains(t, deployment.Spec.Template.Labels, "team", "deployment labels must not leak into pod labels")
+	assert.Equal(t, "team", deployment.Annotations[k8s.ManagedDeploymentLabelsAnnotation])
+	assert.NotContains(t, deployment.Spec.Template.Annotations, k8s.ManagedDeploymentLabelsAnnotation,
+		"a pod template change would roll the pods")
+	assert.Equal(t, map[string]string{
+		k8s.WorkerDeploymentNameLabel: "test-worker",
+		k8s.BuildIDLabel:              "build123",
+	}, deployment.Spec.Selector.MatchLabels)
+}
+
+func TestApplyDeploymentLabels(t *testing.T) {
+	selector := map[string]string{
+		k8s.WorkerDeploymentNameLabel: "test-worker",
+		k8s.BuildIDLabel:              "build123",
+	}
+	tests := map[string]struct {
+		labels      map[string]string
+		managed     string
+		desired     map[string]string
+		wantLabels  map[string]string
+		wantManaged string
+		wantChanged bool
+	}{
+		"adds labels to a Deployment created before the field existed": {
+			labels:      map[string]string{"argocd": "x"},
+			desired:     map[string]string{"team": "payments"},
+			wantLabels:  map[string]string{"argocd": "x", "team": "payments"},
+			wantManaged: "team",
+			wantChanged: true,
+		},
+		"updates a changed value": {
+			labels:      map[string]string{"team": "payments"},
+			managed:     "team",
+			desired:     map[string]string{"team": "billing"},
+			wantLabels:  map[string]string{"team": "billing"},
+			wantManaged: "team",
+			wantChanged: true,
+		},
+		"removes a key dropped from the spec and keeps labels other tools added": {
+			labels:      map[string]string{"team": "payments", "opt-out": "true", "argocd": "x"},
+			managed:     "opt-out,team",
+			desired:     map[string]string{"team": "payments"},
+			wantLabels:  map[string]string{"team": "payments", "argocd": "x"},
+			wantManaged: "team",
+			wantChanged: true,
+		},
+		"removes the annotation when the spec is emptied": {
+			labels:      map[string]string{"team": "payments"},
+			managed:     "team",
+			wantLabels:  map[string]string{},
+			wantChanged: true,
+		},
+		"never overrides or removes selector labels": {
+			managed:     k8s.BuildIDLabel,
+			desired:     map[string]string{k8s.BuildIDLabel: "user-supplied"},
+			wantLabels:  map[string]string{},
+			wantChanged: true,
+		},
+		"reports no change when in sync": {
+			labels:      map[string]string{"b": "2", "a": "1"},
+			managed:     "a,b",
+			desired:     map[string]string{"a": "1", "b": "2"},
+			wantLabels:  map[string]string{"a": "1", "b": "2"},
+			wantManaged: "a,b",
+		},
+	}
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			labels := maps.Clone(selector)
+			maps.Copy(labels, tc.labels)
+			deployment := &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: map[string]string{"other": "kept"}},
+				Spec:       appsv1.DeploymentSpec{Selector: &metav1.LabelSelector{MatchLabels: maps.Clone(selector)}},
+			}
+			if tc.managed != "" {
+				deployment.Annotations[k8s.ManagedDeploymentLabelsAnnotation] = tc.managed
+			}
+
+			changed := k8s.ApplyDeploymentLabels(deployment, tc.desired)
+
+			wantLabels := maps.Clone(selector)
+			maps.Copy(wantLabels, tc.wantLabels)
+			assert.Equal(t, tc.wantChanged, changed)
+			assert.Equal(t, wantLabels, deployment.Labels)
+			assert.Equal(t, tc.wantManaged, deployment.Annotations[k8s.ManagedDeploymentLabelsAnnotation])
+			assert.Equal(t, "kept", deployment.Annotations["other"])
+		})
+	}
 }
 
 func TestNewDeploymentWithOwnerRef_Strategy(t *testing.T) {