From 38444074b2e39af5a99e53461b9609aaf5808363 Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Tue, 1 Sep 2026 13:10:24 -0400 Subject: [PATCH 1/8] Derive atelet and router identities from the install namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chart already renders every resource into the release namespace and passes the resolved Service names to atenet-dns, but several identity checks still assume substrate occupies "ate-system". Each one fails closed in a relocated install, and none of them fails in a way that points at the namespace: - atunnel verifies atelet's SPIFFE ID on the credential broker connection against a hardcoded ID, so no actor can obtain a certificate. atunnel runs in the actor's pod, whose POD_NAMESPACE is the actor's and not atelet's, so the expected identity is passed in. - The NetworkPolicy the WorkerPool controller generates admits ingress only from "ate-system". Pointed at the wrong namespace it admits nobody, and the CNI drops every request to the pool's actors with no error from substrate itself. - actoridentity rejects atelet's MintCert/MintJWT calls as "not atelet". It re-declared the atelet SPIFFE segments to avoid depending on controlapi, with a comment asking for a shared package once a third consumer appeared; atunnel is that third consumer, so the segments move to installdefaults. - The egress MITM CA pool Secret was looked up in a hardcoded namespace. atunnel already took the atenet-router identity as a flag, so the atelet identity follows that convention. Both flags previously relied on their defaults being right; ate-controller now passes them explicitly, since it is the component that knows which namespace the control plane occupies. Without that the new flag would be inert — correct only where the old hardcoded value was already correct. hack/gen-rbac.sh now templatizes the namespace of the generated ate-controller Role alongside its name. A kubebuilder rbac marker has to name a literal namespace, so controller-gen emits "ate-system"; left alone, the Role lands outside the release namespace and ate-controller loses its endpointslices permission. manifests/ate-install/ is unchanged: it renders with --namespace ate-system, so the templated value produces the same bytes. Renaming the release is a separate axis and still unsupported: the chart prefixes ServiceAccount names, which SPIFFE IDs embed, and those names remain constants in installdefaults. --- charts/substrate/templates/role.yaml | 2 +- .../internal/actoridentity/actoridentity.go | 30 ++++---- .../actoridentity/actoridentity_test.go | 52 ++++++++++++-- cmd/ateapi/main.go | 2 +- .../controllers/egressmitmtrust_controller.go | 9 ++- .../egressmitmtrust_controller_test.go | 11 +-- cmd/atecontroller/internal/controllers/gen.go | 2 + .../controllers/networkpolicy_controller.go | 12 ++-- .../networkpolicy_controller_test.go | 24 ++++++- .../internal/controllers/workerpool_apply.go | 8 ++- .../controllers/workerpool_apply_test.go | 51 ++++++++++--- .../controllers/workerpool_controller.go | 6 +- .../controllers/workerpool_controller_test.go | 6 +- cmd/atecontroller/main.go | 13 ++-- cmd/ateom-gvisor/main.go | 14 +++- cmd/ateom-microvm/main.go | 14 +++- hack/gen-rbac.sh | 6 ++ internal/atunnel/credential.go | 12 +++- internal/atunnel/credential_test.go | 2 + internal/installdefaults/installdefaults.go | 36 +++++++++- .../installdefaults/installdefaults_test.go | 72 +++++++++++++++++++ 21 files changed, 321 insertions(+), 63 deletions(-) create mode 100644 internal/installdefaults/installdefaults_test.go diff --git a/charts/substrate/templates/role.yaml b/charts/substrate/templates/role.yaml index 5a240f5baf..9226661d30 100644 --- a/charts/substrate/templates/role.yaml +++ b/charts/substrate/templates/role.yaml @@ -102,7 +102,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ include "substrate.fullname" (list "ate-controller" .) }} - namespace: ate-system + namespace: {{ .Release.Namespace }} rules: - apiGroups: - discovery.k8s.io diff --git a/cmd/ateapi/internal/actoridentity/actoridentity.go b/cmd/ateapi/internal/actoridentity/actoridentity.go index 7304c43d15..9250c997cd 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity.go @@ -31,6 +31,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/principal" @@ -59,33 +60,26 @@ type Server struct { // is entitled to the actor it is asking for a credential for. store store.Interface workers *workercache.Cache + + // ateletNamespace is the namespace segment of the SPIFFE ID that the + // calling atelet must present to mint actor credentials. + ateletNamespace string } var _ ateapipb.ActorIdentityServer = (*Server)(nil) -func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache) *Server { +func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache, ateletNamespace string) *Server { return &Server{ actorIdentityJWTIssuer: actorIdentityJWTIssuer, actorIDJWTPoolFile: actorIDJWTPoolFile, actorIDCAPool: actorIDCAPool, store: store, workers: workers, + ateletNamespace: ateletNamespace, } } -// The SPIFFE identity that atelet client certs carry, as minted by the -// podidentity signer (cmd/podcertcontroller/internal/podidentitysigner). -// -// These mirror the constants the atelet dialer verifies against in -// cmd/ateapi/internal/controlapi/dialer.go. They are duplicated rather than -// imported so that this package does not depend on controlapi for three -// strings; if a third pkg that need these constants appears, they should move to a shared package. -const ( - ateletTrustDomain = "cluster.local" - ateletNamespace = "ate-system" - ateletSA = "atelet" - actorCertificateLifetime = time.Hour -) +const actorCertificateLifetime = time.Hour func (s *Server) MintJWT(ctx context.Context, req *ateapipb.MintJWTRequest) (*ateapipb.MintJWTResponse, error) { caller, ok := principal.FromContext(ctx) @@ -148,7 +142,7 @@ func (s *Server) MintJWT(ctx context.Context, req *ateapipb.MintJWTRequest) (*at } func (s *Server) MintCert(ctx context.Context, req *ateapipb.MintCertRequest) (*ateapipb.MintCertResponse, error) { - caller, err := authenticateAtelet(ctx) + caller, err := s.authenticateAtelet(ctx) if err != nil { return nil, err } @@ -243,7 +237,7 @@ type ateletCaller struct { // pod-identity CA (see buildServerCreds in cmd/ateapi/main.go), so the // extensions read here are trustworthy: only the pod-identity signer can mint // a certificate carrying a given pod's node name. -func authenticateAtelet(ctx context.Context) (*ateletCaller, error) { +func (s *Server) authenticateAtelet(ctx context.Context) (*ateletCaller, error) { p, ok := peer.FromContext(ctx) if !ok { return nil, status.Errorf(codes.Unauthenticated, "no peer transport information found") @@ -264,8 +258,8 @@ func authenticateAtelet(ctx context.Context) (*ateletCaller, error) { // rejected here. expected := (&url.URL{ Scheme: "spiffe", - Host: ateletTrustDomain, - Path: path.Join("ns", ateletNamespace, "sa", ateletSA), + Host: installdefaults.AteletTrustDomain, + Path: path.Join("ns", s.ateletNamespace, "sa", installdefaults.AteletServiceAccount), }).String() if len(leaf.URIs) == 0 || leaf.URIs[0].String() != expected { slog.WarnContext(ctx, "ActorIdentity denied: caller is not atelet", diff --git a/cmd/ateapi/internal/actoridentity/actoridentity_test.go b/cmd/ateapi/internal/actoridentity/actoridentity_test.go index d1282de67f..72e8b1cff6 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity_test.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity_test.go @@ -32,6 +32,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/principal" "github.com/agent-substrate/substrate/internal/resources" @@ -74,6 +75,15 @@ const ( // populates. Self-signing is sufficient because the code under test reads an // already transport-verified peer certificate and never re-validates the chain // itself. +// The atelet SPIFFE segments the tests build peer certificates from. They +// mirror what a default install mints, which is what the Server under test is +// configured with. +const ( + ateletTrustDomain = installdefaults.AteletTrustDomain + ateletNamespace = installdefaults.SystemNamespace + ateletSA = installdefaults.AteletServiceAccount +) + func newTestCert(t *testing.T, spiffePath string, podIdentity *substratex509.PodIdentity) *x509.Certificate { t.Helper() @@ -165,7 +175,7 @@ func newTestServer(t *testing.T, st store.Interface) *Server { t.Fatalf("start worker cache: %v", err) } } - return New("issuer", "", pool, st, workers) + return New("issuer", "", pool, st, workers, ateletNamespace) } // staleWatchStore wraps a store with a WatchWorkers that never delivers, @@ -324,11 +334,11 @@ func newTestServerWithCache(t *testing.T, st store.Interface, workers *workercac t.Fatalf("generate CA: %v", err) } pool := &localca.ConcretePool{CAs: []*localca.CA{ca}} - return New("issuer", "", pool, st, workers) + return New("issuer", "", pool, st, workers, ateletNamespace) } func TestMintJWTRequiresConfiguredJWTProvider(t *testing.T) { - srv := &Server{actorIdentityJWTIssuer: "https://kubernetes.example"} + srv := &Server{actorIdentityJWTIssuer: "https://kubernetes.example", ateletNamespace: ateletNamespace} for _, tt := range []struct { name string ctx context.Context @@ -918,7 +928,7 @@ func TestMintCertAuthorizesBeforeSigning(t *testing.T) { ActiveForSigning: "test-actor-ca", } - srv := New("issuer", "", pool, st, workers) + srv := New("issuer", "", pool, st, workers, ateletNamespace) actor, err := st.GetActor(ctx, resources.ActorRef{Atespace: testAtespace, Name: testActorName}) if err != nil { @@ -1082,3 +1092,37 @@ func TestValidateMintCertRequest(t *testing.T) { }) } } + +// TestAuthenticateAteletHonorsConfiguredNamespace checks that the namespace +// the Server is configured with is the one it accepts, and that the canonical +// namespace is rejected when the install lives elsewhere. +// +// The table in TestMintCertAuthorization covers a caller from the wrong +// namespace, but its Server is always configured with the default, so it holds +// against a hardcoded "ate-system" too. Only the relocated case distinguishes +// "reads its configuration" from "happens to agree with the constant". +func TestAuthenticateAteletHonorsConfiguredNamespace(t *testing.T) { + const relocated = "substrate-test" + + srv := &Server{ateletNamespace: relocated} + + t.Run("accepts atelet from the configured namespace", func(t *testing.T) { + id := podIdentityOn(testNode) + id.Namespace = relocated + cert := newTestCert(t, path.Join("ns", relocated, "sa", ateletSA), id) + + if _, err := srv.authenticateAtelet(ctxWithCert(cert)); err != nil { + t.Errorf("authenticateAtelet() = %v, want success for an atelet in %q", err, relocated) + } + }) + + t.Run("rejects atelet from the canonical namespace", func(t *testing.T) { + id := podIdentityOn(testNode) + id.Namespace = installdefaults.SystemNamespace + cert := newTestCert(t, path.Join("ns", installdefaults.SystemNamespace, "sa", ateletSA), id) + + if _, err := srv.authenticateAtelet(ctxWithCert(cert)); status.Code(err) != codes.PermissionDenied { + t.Errorf("authenticateAtelet() code = %v, want PermissionDenied for an atelet in %q", status.Code(err), installdefaults.SystemNamespace) + } + }) +} diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 9c9de8567a..9cb50f49b7 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -211,7 +211,7 @@ func main() { serverboot.Fatal(ctx, "while loading the Actor ID CA", err) } - actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, actorIDCAPool, persistence, workerCache) + actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, actorIDCAPool, persistence, workerCache, ateletNamespace) lisCfg := &net.ListenConfig{} lis, err := lisCfg.Listen(ctx, "tcp", *listenAddr) diff --git a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go index f2e8f87c04..c51f5a7ddd 100644 --- a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go +++ b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go @@ -40,13 +40,16 @@ import ( // CA pool. type EgressMITMTrustReconciler struct { client.Client + + // SystemNamespace is the namespace holding the egress MITM CA pool Secret. + SystemNamespace string } // EgressMITMCAPoolRef names the Secret holding the CA pool the egress gateway's // sdsmint sidecar signs per-SNI leaves with. -func EgressMITMCAPoolRef() types.NamespacedName { +func EgressMITMCAPoolRef(systemNamespace string) types.NamespacedName { const egressMITMCAPoolSecret = "egress-mitm-ca-pool" - return types.NamespacedName{Namespace: ateSystemNamespace, Name: egressMITMCAPoolSecret} + return types.NamespacedName{Namespace: systemNamespace, Name: egressMITMCAPoolSecret} } //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch @@ -162,7 +165,7 @@ func (r *EgressMITMTrustReconciler) deleteTrustBundle(ctx context.Context) error } func (r *EgressMITMTrustReconciler) SetupWithManager(mgr ctrl.Manager) error { - poolRef := EgressMITMCAPoolRef() + poolRef := EgressMITMCAPoolRef(r.SystemNamespace) // The pool Secret is the only object reconciled from. The bundle is watched // as well so that deleting or hand-editing the derived object is reverted diff --git a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go index 2d84330900..69dd952f9d 100644 --- a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go +++ b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" ) @@ -66,7 +67,7 @@ func secretForPool(t *testing.T, pool *localca.ConcretePool) *corev1.Secret { if err != nil { t.Fatalf("marshal CA pool: %v", err) } - ref := EgressMITMCAPoolRef() + ref := EgressMITMCAPoolRef(installdefaults.SystemNamespace) return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Namespace: ref.Namespace, Name: ref.Name}, Data: map[string][]byte{"pool": wire}, @@ -86,8 +87,8 @@ func rootPEM(t *testing.T, pool *localca.ConcretePool) string { func reconcilePool(t *testing.T, c client.Client) error { t.Helper() - r := &EgressMITMTrustReconciler{Client: c} - _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: EgressMITMCAPoolRef()}) + r := &EgressMITMTrustReconciler{Client: c, SystemNamespace: installdefaults.SystemNamespace} + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: EgressMITMCAPoolRef(installdefaults.SystemNamespace)}) return err } @@ -170,7 +171,7 @@ func TestEgressMITMTrustFollowsPoolRotation(t *testing.T) { rotated, rotatedPool := caPoolSecret(t, "mitm", "mitm-next") current := &corev1.Secret{} - if err := c.Get(context.Background(), EgressMITMCAPoolRef(), current); err != nil { + if err := c.Get(context.Background(), EgressMITMCAPoolRef(installdefaults.SystemNamespace), current); err != nil { t.Fatalf("get pool secret: %v", err) } current.Data = rotated.Data @@ -279,7 +280,7 @@ func TestEgressMITMTrustKeepsLastGoodBundleOnBadPool(t *testing.T) { } current := &corev1.Secret{} - if err := c.Get(context.Background(), EgressMITMCAPoolRef(), current); err != nil { + if err := c.Get(context.Background(), EgressMITMCAPoolRef(installdefaults.SystemNamespace), current); err != nil { t.Fatalf("get pool secret: %v", err) } current.Data = tc.data diff --git a/cmd/atecontroller/internal/controllers/gen.go b/cmd/atecontroller/internal/controllers/gen.go index a7ed1b1cf5..8c784d79af 100644 --- a/cmd/atecontroller/internal/controllers/gen.go +++ b/cmd/atecontroller/internal/controllers/gen.go @@ -20,6 +20,8 @@ package controllers // - internal/k8sresolver watches ateapi's EndpointSlices to dial it. // //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch +// The marker must name a literal namespace, so it carries the canonical one. +// hack/gen-rbac.sh rewrites it to the chart's release namespace on the way out. //+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch,namespace=ate-system //go:generate bash ../../../../hack/gen-rbac.sh diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go index 928c053a92..22c65bbbdd 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go @@ -33,13 +33,17 @@ import ( const ( networkPolicyFieldOwner = "ate-networkpolicy" - ateSystemNamespace = "ate-system" atenetRouterAppName = "atenet-router" ) type NetworkPolicyReconciler struct { client.Client Scheme *runtime.Scheme + + // SystemNamespace is the namespace atenet-router runs in. The generated + // ingress policy admits only that namespace, so a value that does not + // match the running router blocks every request to the worker pool. + SystemNamespace string } //+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch @@ -74,7 +78,7 @@ func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques func (r *NetworkPolicyReconciler) reconcileImpl(ctx context.Context, wp *atev1alpha1.WorkerPool) error { log := log.FromContext(ctx) - npAC := buildNetworkPolicyApplyConfig(wp) + npAC := r.buildNetworkPolicyApplyConfig(wp) if err := r.Apply(ctx, npAC, client.FieldOwner(networkPolicyFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply NetworkPolicy %s:%s: %w", *npAC.Namespace, *npAC.Name, err) @@ -86,7 +90,7 @@ func (r *NetworkPolicyReconciler) reconcileImpl(ctx context.Context, wp *atev1al return nil } -func buildNetworkPolicyApplyConfig(wp *atev1alpha1.WorkerPool) *networkingv1ac.NetworkPolicyApplyConfiguration { +func (r *NetworkPolicyReconciler) buildNetworkPolicyApplyConfig(wp *atev1alpha1.WorkerPool) *networkingv1ac.NetworkPolicyApplyConfiguration { np := networkingv1ac.NetworkPolicy(resources.NetworkPolicyName(wp.Name), wp.Namespace). WithLabels(map[string]string{ "ate.dev/worker-pool": wp.Name, @@ -110,7 +114,7 @@ func buildNetworkPolicyApplyConfig(wp *atev1alpha1.WorkerPool) *networkingv1ac.N WithFrom( networkingv1ac.NetworkPolicyPeer(). WithNamespaceSelector(metav1ac.LabelSelector(). - WithMatchLabels(map[string]string{"kubernetes.io/metadata.name": ateSystemNamespace})). + WithMatchLabels(map[string]string{"kubernetes.io/metadata.name": r.SystemNamespace})). WithPodSelector(metav1ac.LabelSelector(). WithMatchLabels(map[string]string{"app": atenetRouterAppName})), ), diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go index 0296b873fe..f1ff854d10 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go @@ -21,6 +21,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/types" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/resources" ) @@ -75,7 +76,7 @@ func TestWorkerPoolCreatesNetworkPolicy(t *testing.T) { return false, nil } fromPeer := ingressRule.From[0] - if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != ateSystemNamespace { + if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != installdefaults.SystemNamespace { return false, nil } if fromPeer.PodSelector == nil || fromPeer.PodSelector.MatchLabels["app"] != atenetRouterAppName { @@ -90,3 +91,24 @@ func TestWorkerPoolCreatesNetworkPolicy(t *testing.T) { return true, nil }) } + +// TestBuildNetworkPolicyRelocatedNamespace pins the ingress peer to the +// reconciler's SystemNamespace rather than the canonical install namespace. +// The rest of the suite configures the reconciler with the default, so it +// passes just as well against a hardcoded "ate-system"; this is the case that +// catches that. A policy naming the wrong namespace admits nobody, and the CNI +// drops every request to the pool with no error from substrate itself. +func TestBuildNetworkPolicyRelocatedNamespace(t *testing.T) { + const relocated = "substrate-test" + + r := &NetworkPolicyReconciler{SystemNamespace: relocated} + np := r.buildNetworkPolicyApplyConfig(testWorkerPoolApplyConfig(nil)) + + if len(np.Spec.Ingress) != 1 || len(np.Spec.Ingress[0].From) != 1 { + t.Fatalf("expected exactly one ingress rule with one peer, got %+v", np.Spec.Ingress) + } + got := np.Spec.Ingress[0].From[0].NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] + if got != relocated { + t.Errorf("ingress namespace selector = %q, want %q", got, relocated) + } +} diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index c2fdf57773..0315d72c88 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -26,6 +26,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/deviceplugin" + "github.com/agent-substrate/substrate/internal/installdefaults" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -74,7 +75,7 @@ const ( // Deployment managed by a WorkerPool. Only fields owned by this controller // are declared here. otel, when it carries an endpoint, is propagated to the // ateom container so it pushes telemetry to that collector. -func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettings) *appsv1ac.DeploymentApplyConfiguration { +func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettings, systemNamespace string) *appsv1ac.DeploymentApplyConfiguration { labels := map[string]string{} annotations := map[string]string{} if wp.Spec.Template != nil { @@ -96,6 +97,11 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin "--atunnel-connect-listen-address=:8443", "--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem", "--atunnel-trust-bundle="+atunnelIdentityMountPath+"/trust-bundle.pem", + // The peers atunnel authenticates live in substrate's namespace, + // not the worker's, so the controller passes their identities + // rather than letting ateom assume the default install. + "--atunnel-client-identity="+installdefaults.RouterSPIFFEID(systemNamespace), + "--atunnel-broker-identity="+installdefaults.AteletSPIFFEID(systemNamespace), "--atunnel-egress-listen-address=0.0.0.0:15001", "--atunnel-egress-trust-bundle="+atunnelEgressTrustMountPath+"/trust-bundle.pem", ). diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 85bff190f4..fbf09f5bbe 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -30,6 +30,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/deviceplugin" + "github.com/agent-substrate/substrate/internal/installdefaults" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -205,7 +206,7 @@ func TestBuildDeploymentApplyConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := buildDeploymentApplyConfig(tt.wp, ateomOTelSettings{}) + got := buildDeploymentApplyConfig(tt.wp, ateomOTelSettings{}, installdefaults.SystemNamespace) if diff := cmp.Diff(tt.want, got); diff != "" { t.Fatalf("buildDeploymentApplyConfig() mismatch (-want +got):\n%s", diff) } @@ -225,7 +226,7 @@ func TestBuildDeploymentApplyConfigMetadata(t *testing.T) { }, }) - got := buildDeploymentApplyConfig(wp, ateomOTelSettings{}) + got := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace) wantLabels := map[string]string{ "project": "agent-substrate", "team": "compute", @@ -267,7 +268,7 @@ func TestMicroVMPodShape(t *testing.T) { t.Run(tt.name, func(t *testing.T) { wp := testWorkerPoolApplyConfig(nil) wp.Spec.SandboxClass = tt.class - ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec + ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace).Spec.Template.Spec // /dev/kvm must come from the device plugin, never a hostPath: a // hostPath mount carries no cgroup device allow rule, and the @@ -360,7 +361,7 @@ func TestMicroVMDeviceRequestsPreserveTemplateResources(t *testing.T) { }, }) wp.Spec.SandboxClass = atev1alpha1.SandboxClassMicroVM - c := buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec.Containers[0] + c := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace).Spec.Template.Spec.Containers[0] if got, ok := deviceLimit(c, string(corev1.ResourceMemory)); !ok || got != "2Gi" { t.Errorf("memory limit = %q (present=%v), want 2Gi", got, ok) @@ -425,7 +426,7 @@ func TestAteomSecurityContextByClass(t *testing.T) { // TestTerminationGracePeriodSeconds asserts the pod's grace period is hardcoded to 3600s. func TestTerminationGracePeriodSeconds(t *testing.T) { wp := testWorkerPoolApplyConfig(nil) - ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec + ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace).Spec.Template.Spec if ps.TerminationGracePeriodSeconds == nil { t.Fatalf("TerminationGracePeriodSeconds not set") } @@ -449,7 +450,7 @@ func TestBuildDeploymentApplyConfigOTelEndpoint(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{Endpoint: tt.endpoint}). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{Endpoint: tt.endpoint}, installdefaults.SystemNamespace). Spec.Template.Spec.Containers[0] env := envByName(c.Env) @@ -535,7 +536,7 @@ func TestBuildDeploymentApplyConfigMetricExportTuning(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel, installdefaults.SystemNamespace). Spec.Template.Spec.Containers[0] env := envByName(c.Env) for _, k := range []string{"OTEL_METRIC_EXPORT_INTERVAL", "OTEL_METRIC_EXPORT_TIMEOUT"} { @@ -592,7 +593,7 @@ func TestBuildDeploymentApplyConfigTracesSamplerPropagation(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel, installdefaults.SystemNamespace). Spec.Template.Spec.Containers[0] env := envByName(c.Env) for _, k := range []string{"OTEL_TRACES_SAMPLER", "OTEL_TRACES_SAMPLER_ARG"} { @@ -720,6 +721,8 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf "--atunnel-connect-listen-address=:8443", "--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem", "--atunnel-trust-bundle="+atunnelIdentityMountPath+"/trust-bundle.pem", + "--atunnel-client-identity="+installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + "--atunnel-broker-identity="+installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "--atunnel-egress-listen-address=0.0.0.0:15001", "--atunnel-egress-trust-bundle="+atunnelEgressTrustMountPath+"/trust-bundle.pem", ). @@ -799,3 +802,35 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithLabels(map[string]string{"ate.dev/worker-pool": wp.Name}). WithSpec(podSpecAC))) } + +// TestBuildDeploymentAtunnelIdentitiesRelocatedNamespace pins the SPIFFE +// identities handed to ateom to the controller's namespace. atunnel runs in +// the actor's pod, so it cannot derive atelet's or the router's namespace +// itself; if these carry the wrong one, the credential broker handshake and +// actor ingress both fail closed. Every other case here passes the canonical +// namespace and so would pass against a hardcoded value too. +func TestBuildDeploymentAtunnelIdentitiesRelocatedNamespace(t *testing.T) { + const relocated = "substrate-test" + + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{}, relocated). + Spec.Template.Spec.Containers[0] + + want := map[string]string{ + "--atunnel-client-identity=": "spiffe://cluster.local/ns/substrate-test/sa/atenet-router", + "--atunnel-broker-identity=": "spiffe://cluster.local/ns/substrate-test/sa/atelet", + } + for flag, wantVal := range want { + var got string + for _, arg := range c.Args { + if strings.HasPrefix(arg, flag) { + got = strings.TrimPrefix(arg, flag) + } + } + if got == "" { + t.Fatalf("no %s argument found in %v", flag, c.Args) + } + if got != wantVal { + t.Errorf("%s%s, want %s%s", flag, got, flag, wantVal) + } + } +} diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller.go b/cmd/atecontroller/internal/controllers/workerpool_controller.go index 7c962d62dd..f8cf97a673 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller.go @@ -52,6 +52,10 @@ type WorkerPoolReconciler struct { // OTelTracesSamplerArg is the OTEL_TRACES_SAMPLER_ARG propagated to ateom // pods. Ignored unless OTelTracesSampler is set. OTelTracesSamplerArg string + // SystemNamespace is the namespace substrate's control plane runs in. It + // names the atenet-router and atelet SPIFFE identities that atunnel + // authenticates inside each worker. + SystemNamespace string desiredWorkers metric.Int64ObservableUpDownCounter readyWorkers metric.Int64ObservableUpDownCounter @@ -116,7 +120,7 @@ func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alp MetricExportTimeout: r.OTelMetricExportTimeout, TracesSampler: r.OTelTracesSampler, TracesSamplerArg: r.OTelTracesSamplerArg, - }) + }, r.SystemNamespace) if err := r.Apply(ctx, depAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply Deployment: %w", err) } diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index c1f3b8b6f7..5c7ea51366 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -43,6 +43,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/testenv" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -86,8 +87,9 @@ func TestMain(m *testing.M) { } if err := (&NetworkPolicyReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SystemNamespace: installdefaults.SystemNamespace, }).SetupWithManager(mgr); err != nil { fmt.Fprintf(os.Stderr, "netpolicy controller setup failed: %v\n", err) os.Exit(1) diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index 7ef107ec1a..c2a486abee 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -21,6 +21,7 @@ import ( "github.com/agent-substrate/substrate/cmd/atecontroller/internal/controllers" "github.com/agent-substrate/substrate/cmd/atecontroller/internal/workersync" "github.com/agent-substrate/substrate/internal/ateapiauth" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/serverboot" clientv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" @@ -152,7 +153,8 @@ func main() { ateapiClient := ateapipb.NewControlClient(ateapiConn) // EgressMITMTrustReconciler watches the Secret `egress-mitm-ca-pool`. - egressMITMCAPool := controllers.EgressMITMCAPoolRef() + systemNamespace := installdefaults.NamespaceFromPodEnv() + egressMITMCAPool := controllers.EgressMITMCAPoolRef(systemNamespace) mgr, err := ctrl.NewManager(k8sConfig, ctrl.Options{ Scheme: scheme, Cache: cache.Options{ @@ -180,21 +182,24 @@ func main() { OTelMetricExportTimeout: *otelMetricExportTimeout, OTelTracesSampler: *otelTracesSampler, OTelTracesSamplerArg: *otelTracesSamplerArg, + SystemNamespace: systemNamespace, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WorkerPool") os.Exit(1) } if err = (&controllers.NetworkPolicyReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SystemNamespace: systemNamespace, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NetPolicy") os.Exit(1) } if err = (&controllers.EgressMITMTrustReconciler{ - Client: mgr.GetClient(), + Client: mgr.GetClient(), + SystemNamespace: systemNamespace, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "EgressMITMTrust") os.Exit(1) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 90cbfaec9c..62f547cc61 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -44,6 +44,7 @@ import ( "github.com/agent-substrate/substrate/internal/childreap" "github.com/agent-substrate/substrate/internal/contextlogging" "github.com/agent-substrate/substrate/internal/imagecache" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/otlprelay" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" @@ -72,7 +73,8 @@ var ( atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", ":8443", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = pflag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") podIdentityTrustBundle = pflag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet") - atunnelClientIdentity = pflag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS") + atunnelClientIdentity = pflag.String("atunnel-client-identity", installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity allowed to call actor ingress HTTPS") + ateletIdentity = pflag.String("atunnel-broker-identity", installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity the node-local atelet must present on the credential broker connection. Override when atelet runs outside the default namespace.") atunnelEgressListenAddress = pflag.String("atunnel-egress-listen-address", "0.0.0.0:15001", "Address for transparently intercepted actor egress TCP") egressGatewayTrustBundle = pflag.String("atunnel-egress-trust-bundle", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "Service DNS trust bundle for the remote egress gateway") readinessListenAddress = pflag.String("readiness-listen-address", "0.0.0.0:8080", "Address for HTTP readiness checks") @@ -210,7 +212,7 @@ func do(ctx context.Context) error { return err } - ateomService := NewService(interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + ateomService := NewService(interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle, *ateletIdentity) svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -369,6 +371,10 @@ type AteomService struct { podIdentityTrustBundlePath string // egressGatewayTrustBundlePath verifies the remote gateway's serving cert. egressGatewayTrustBundlePath string + // ateletSPIFFEID is the identity the node-local atelet must present on the + // credential broker connection. It names atelet's namespace, not this + // worker's, so it is configured rather than derived from the downward API. + ateletSPIFFEID string // activeActor is the actor whose workload this ateom is currently running, // or nil when it is "available". An ateom serves one actor at a time, so a @@ -420,7 +426,7 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath, ateletSPIFFEID string) *AteomService { return &AteomService{ lock: newCancelableMutex(), interiorNetNS: interiorNetNS, @@ -431,6 +437,7 @@ func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, workerCredentialBundlePath: workerCredentialBundlePath, podIdentityTrustBundlePath: podIdentityTrustBundlePath, egressGatewayTrustBundlePath: egressGatewayTrustBundlePath, + ateletSPIFFEID: ateletSPIFFEID, cgroupRoot: defaultCgroupRoot, } } @@ -1048,6 +1055,7 @@ func (s *AteomService) prepareActorEgress(ctx context.Context, actorUID string, CredentialBundlePath: s.workerCredentialBundlePath, TrustBundlePath: s.podIdentityTrustBundlePath, ExpectedActorUID: actorUID, + AteletSPIFFEID: s.ateletSPIFFEID, }) if err != nil { return nil, fmt.Errorf("while configuring actor certificate broker: %w", err) diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 751881e37f..0e0fcf9230 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -45,6 +45,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateomnet" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/otlprelay" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" @@ -77,7 +78,8 @@ var ( atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", ":8443", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") podIdentityTrustBundle = flag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet") - atunnelClientIdentity = flag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS") + atunnelClientIdentity = flag.String("atunnel-client-identity", installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity allowed to call actor ingress HTTPS") + ateletIdentity = flag.String("atunnel-broker-identity", installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity the node-local atelet must present on the credential broker connection. Override when atelet runs outside the default namespace.") atunnelEgressListenAddress = flag.String("atunnel-egress-listen-address", "0.0.0.0:15001", "Address for transparently intercepted actor egress TCP") egressGatewayTrustBundle = flag.String("atunnel-egress-trust-bundle", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "Service DNS trust bundle for the remote egress gateway") readinessListenAddress = flag.String("readiness-listen-address", "0.0.0.0:8080", "Address for HTTP readiness checks") @@ -256,7 +258,7 @@ func do(ctx context.Context) error { }() slog.InfoContext(ctx, "atunnel egress serving", slog.String("address", *atunnelEgressListenAddress)) - ateomService := NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + ateomService := NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle, *ateletIdentity) svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -423,6 +425,10 @@ type AteomService struct { podIdentityTrustBundlePath string // egressGatewayTrustBundlePath verifies the remote gateway's serving cert. egressGatewayTrustBundlePath string + // ateletSPIFFEID is the identity the node-local atelet must present on the + // credential broker connection. It names atelet's namespace, not this + // worker's, so it is configured rather than derived from the downward API. + ateletSPIFFEID string // running maps actor UID -> the live micro-VM, kept so CheckpointWorkload can // pause+snapshot+teardown the same sandbox (and RestoreWorkload can track the @@ -476,7 +482,7 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath, ateletSPIFFEID string) *AteomService { return &AteomService{ lock: newCancelableMutex(), podUID: podUID, @@ -492,6 +498,7 @@ func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveM workerCredentialBundlePath: workerCredentialBundlePath, podIdentityTrustBundlePath: podIdentityTrustBundlePath, egressGatewayTrustBundlePath: egressGatewayTrustBundlePath, + ateletSPIFFEID: ateletSPIFFEID, running: map[string]*runningActor{}, } } @@ -520,6 +527,7 @@ func (s *AteomService) prepareActorEgress(ctx context.Context, actorUID string, CredentialBundlePath: s.workerCredentialBundlePath, TrustBundlePath: s.podIdentityTrustBundlePath, ExpectedActorUID: actorUID, + AteletSPIFFEID: s.ateletSPIFFEID, }) if err != nil { return nil, fmt.Errorf("while configuring actor certificate broker: %w", err) diff --git a/hack/gen-rbac.sh b/hack/gen-rbac.sh index baa22fa517..b7083b00e7 100755 --- a/hack/gen-rbac.sh +++ b/hack/gen-rbac.sh @@ -35,3 +35,9 @@ bash "${ROOT}/hack/run-tool.sh" controller-gen \ # Templatize the ClusterRole name. controller-gen emits ` name: ate-controller` # at column 0; the substitution is exact-match to stay robust. sed -i 's|^ name: ate-controller$| name: {{ include "substrate.fullname" (list "ate-controller" .) }}|' "${OUT}" + +# Templatize the namespaced Role's namespace. A kubebuilder rbac marker has to +# name a literal namespace, so controller-gen emits the canonical one; the +# chart installs into whatever namespace the release targets, and a Role left +# in ate-system would leave ate-controller without the permission there. +sed -i 's|^ namespace: ate-system$| namespace: {{ .Release.Namespace }}|' "${OUT}" diff --git a/internal/atunnel/credential.go b/internal/atunnel/credential.go index 65090af769..6b17aa26fc 100644 --- a/internal/atunnel/credential.go +++ b/internal/atunnel/credential.go @@ -23,9 +23,7 @@ import ( "crypto/x509" "fmt" "net" - "net/url" "os" - "path" "slices" "sync" "time" @@ -61,6 +59,11 @@ type BrokerConfig struct { // ExpectedActorUID prevents a mint started for an old activation from // receiving the newly assigned actor's certificate. ExpectedActorUID string + // AteletSPIFFEID is the SPIFFE ID atelet must present on the credential + // broker connection. It carries the namespace atelet runs in, which is not + // this worker's own namespace, so it is supplied by the caller rather than + // read from the downward API. See installdefaults.AteletSPIFFEID. + AteletSPIFFEID string } // NewBrokerCertificateSource creates one actor key for this activation. The key @@ -70,6 +73,9 @@ func NewBrokerCertificateSource(cfg BrokerConfig) (*BrokerCertificateSource, err if cfg.SocketPath == "" || cfg.CredentialBundlePath == "" || cfg.TrustBundlePath == "" || cfg.ExpectedActorUID == "" { return nil, fmt.Errorf("atunnel: credential broker socket, credentials, trust bundle, and expected actor UID are required") } + if cfg.AteletSPIFFEID == "" { + return nil, fmt.Errorf("atunnel: expected atelet SPIFFE ID is required") + } localCert, err := credbundle.Parse(cfg.CredentialBundlePath) if err != nil { return nil, fmt.Errorf("atunnel: load worker identity: %w", err) @@ -90,7 +96,7 @@ func NewBrokerCertificateSource(cfg BrokerConfig) (*BrokerCertificateSource, err if err != nil { return nil, fmt.Errorf("atunnel: generate actor private key: %w", err) } - expectedURI := (&url.URL{Scheme: "spiffe", Host: "cluster.local", Path: path.Join("ns", "ate-system", "sa", "atelet")}).String() + expectedURI := cfg.AteletSPIFFEID tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS13, InsecureSkipVerify: true, // Verification below supports SPIFFE Pod certificates without a DNS name. diff --git a/internal/atunnel/credential_test.go b/internal/atunnel/credential_test.go index 057b6fe9e8..8e598043e4 100644 --- a/internal/atunnel/credential_test.go +++ b/internal/atunnel/credential_test.go @@ -29,6 +29,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/substratex509" "google.golang.org/grpc" @@ -184,6 +185,7 @@ func newTestBrokerCertificateSource(t *testing.T, ateletIdentity *substratex509. CredentialBundlePath: credentialPath, TrustBundlePath: trustPath, ExpectedActorUID: "actor-uid", + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), }) if err != nil { t.Fatal(err) diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go index 8f47d84f0b..0e1995e76f 100644 --- a/internal/installdefaults/installdefaults.go +++ b/internal/installdefaults/installdefaults.go @@ -18,7 +18,11 @@ // the canonical layout pass actual values via the corresponding flags. package installdefaults -import "os" +import ( + "net/url" + "os" + "path" +) const ( // SystemNamespace is the namespace where substrate's control-plane @@ -31,6 +35,16 @@ const ( // DNSServiceName is the Service name of substrate's CoreDNS. DNSServiceName = "dns" + // AteletTrustDomain, AteletServiceAccount and RouterServiceAccount are the + // trust-domain and service-account segments of the SPIFFE IDs that atelet + // and atenet-router Pod certificates carry, as minted by the podidentity + // signer (cmd/podcertcontroller/internal/podidentitysigner). The namespace + // segment is the namespace they run in, which callers resolve themselves + // rather than assume. + AteletTrustDomain = "cluster.local" + AteletServiceAccount = "atelet" + RouterServiceAccount = "atenet-router" + // PodNamespaceEnv is the conventional env var name for the namespace // a pod is running in, exposed via Kubernetes' downward API. PodNamespaceEnv = "POD_NAMESPACE" @@ -45,3 +59,23 @@ func NamespaceFromPodEnv() string { } return SystemNamespace } + +// SPIFFEID returns the SPIFFE ID that Pod certificates for serviceAccount in +// namespace carry. Peers authenticate by comparing against this exact string. +func SPIFFEID(namespace, serviceAccount string) string { + return (&url.URL{ + Scheme: "spiffe", + Host: AteletTrustDomain, + Path: path.Join("ns", namespace, "sa", serviceAccount), + }).String() +} + +// AteletSPIFFEID returns the SPIFFE ID atelet presents when it runs in namespace. +func AteletSPIFFEID(namespace string) string { + return SPIFFEID(namespace, AteletServiceAccount) +} + +// RouterSPIFFEID returns the SPIFFE ID atenet-router presents when it runs in namespace. +func RouterSPIFFEID(namespace string) string { + return SPIFFEID(namespace, RouterServiceAccount) +} diff --git a/internal/installdefaults/installdefaults_test.go b/internal/installdefaults/installdefaults_test.go new file mode 100644 index 0000000000..f45bbb7c33 --- /dev/null +++ b/internal/installdefaults/installdefaults_test.go @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installdefaults + +import "testing" + +func TestAteletSPIFFEID(t *testing.T) { + tests := []struct { + name string + namespace string + want string + }{ + { + // The canonical install. Peers reject any other string, so this + // value is effectively wire format: changing it breaks the atelet + // mTLS handshake for every existing deployment. + name: "default namespace", + namespace: SystemNamespace, + want: "spiffe://cluster.local/ns/ate-system/sa/atelet", + }, + { + name: "namespace the install was relocated to", + namespace: "team-a-substrate", + want: "spiffe://cluster.local/ns/team-a-substrate/sa/atelet", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := AteletSPIFFEID(tt.namespace); got != tt.want { + t.Errorf("AteletSPIFFEID(%q) = %q, want %q", tt.namespace, got, tt.want) + } + }) + } +} + +func TestRouterSPIFFEID(t *testing.T) { + // Matches the --atunnel-client-identity default the ateom binaries ship + // with, which is what actor ingress authenticates the router against. + const want = "spiffe://cluster.local/ns/ate-system/sa/atenet-router" + if got := RouterSPIFFEID(SystemNamespace); got != want { + t.Errorf("RouterSPIFFEID(%q) = %q, want %q", SystemNamespace, got, want) + } +} + +func TestNamespaceFromPodEnv(t *testing.T) { + t.Run("falls back to the install default when unset", func(t *testing.T) { + t.Setenv(PodNamespaceEnv, "") + if got := NamespaceFromPodEnv(); got != SystemNamespace { + t.Errorf("NamespaceFromPodEnv() = %q, want %q", got, SystemNamespace) + } + }) + + t.Run("prefers the downward API value", func(t *testing.T) { + t.Setenv(PodNamespaceEnv, "team-a-substrate") + if got := NamespaceFromPodEnv(); got != "team-a-substrate" { + t.Errorf("NamespaceFromPodEnv() = %q, want %q", got, "team-a-substrate") + } + }) +} From 3f1f2325075003bcf570bc7f17cdb8ab10504e5f Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Thu, 3 Sep 2026 08:42:18 -0400 Subject: [PATCH 2/8] Let the install scripts and e2e harness target a non-default namespace The chart installs into whatever namespace the release targets, but the tooling around it did not: hack/install-ate.sh hardcoded "ate-system" in 52 places and internal/e2e in 9, so a relocated install could not be bootstrapped or exercised. The mTLS authorities landed in ate-system while the control plane looked for them in the release namespace, and the e2e suites addressed the router and egress CA pool in a namespace they were no longer in. install-ate.sh now reads ATE_NAMESPACE and the e2e harness reads E2E_SYSTEM_NAMESPACE, both defaulting to ate-system so existing flows and CI are unaffected. The one part that is not a substitution is namespace creation. manifests/ate-install/ate-system-namespace.yaml names ate-system and carries labels the chart also sets, so it stays the source of truth for the canonical install; ensure_ate_namespace falls back to a plain create for any other namespace. --- hack/install-ate.sh | 103 ++++++++++-------- internal/e2e/env.go | 16 +++ internal/e2e/preflight.go | 2 +- internal/e2e/router_client.go | 7 +- internal/e2e/statusz.go | 2 +- internal/e2e/suites/demo/demo_test.go | 4 +- .../suites/egressauthz/actoridentity_test.go | 8 +- .../suites/egressauthz/egressauthz_test.go | 1 - internal/e2e/suites/metrics/metrics_test.go | 2 +- .../e2e/suites/networking/grpcingress_test.go | 2 +- .../networkpolicy/networkpolicy_test.go | 5 +- internal/e2e/trustbundle.go | 13 +-- 12 files changed, 95 insertions(+), 70 deletions(-) diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 2e9fd0c03c..59434c46f7 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -35,6 +35,11 @@ if [[ -z "${KUBECTL_CONTEXT:-}" ]]; then fi # otherwise just use the current cluster in KUBECONFIG ... +# Namespace the substrate control plane is installed into. Defaults to the +# canonical ate-system so existing flows are unaffected; override it to install +# a relocated release (the chart's --namespace must match). +ATE_NAMESPACE="${ATE_NAMESPACE:-ate-system}" + # ATE_DEMOS is an array that registers the prefix name of the demo functions. ATE_DEMOS=() @@ -224,8 +229,20 @@ rollout_timeout() { echo "${timeout}" } +# ensure_ate_namespace creates ATE_NAMESPACE and waits for it to go Active. +ensure_ate_namespace() { + if [[ "${ATE_NAMESPACE}" == "ate-system" ]]; then + run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml + else + run_kubectl create namespace "${ATE_NAMESPACE}" --dry-run=client -o yaml \ + | run_kubectl apply -f - + fi + run_kubectl wait --for=jsonpath='{.status.phase}'=Active \ + "namespace/${ATE_NAMESPACE}" --timeout=60s +} + default_postgres_connection_string() { - echo "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + echo "postgresql://postgres@postgres.${ATE_NAMESPACE}.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" } use_bundled_postgres() { @@ -361,14 +378,14 @@ apply_atenet_egress() { # bootstrap arrives as a ConfigMap change, and an otherwise unchanged # Deployment will not pick that up on its own. local running=false - if run_kubectl -n ate-system get deployment/atenet-egress >/dev/null 2>&1; then + if run_kubectl -n "${ATE_NAMESPACE}" get deployment/atenet-egress >/dev/null 2>&1; then running=true fi echo "${manifests}" | run_kubectl apply -f - if [[ "${running}" == "true" ]] && additional_egress_extproc_enabled; then - run_kubectl -n ate-system rollout restart deployment/atenet-egress + run_kubectl -n "${ATE_NAMESPACE}" rollout restart deployment/atenet-egress fi } @@ -417,28 +434,28 @@ apply_otel_endpoint_override() { fi local current="" - current="$(run_kubectl -n ate-system get configmap ate-otel-config \ + current="$(run_kubectl -n "${ATE_NAMESPACE}" get configmap ate-otel-config \ -o jsonpath='{.data.OTEL_EXPORTER_OTLP_ENDPOINT}' 2>/dev/null || true)" if [[ "${current}" == "${ATE_OTLP_ENDPOINT}" ]]; then return 0 fi echo "Overriding OTEL_EXPORTER_OTLP_ENDPOINT with ${ATE_OTLP_ENDPOINT}" - run_kubectl -n ate-system patch configmap ate-otel-config --type=merge \ + run_kubectl -n "${ATE_NAMESPACE}" patch configmap ate-otel-config --type=merge \ -p "{\"data\":{\"OTEL_EXPORTER_OTLP_ENDPOINT\":\"${ATE_OTLP_ENDPOINT}\"}}" local workload for workload in deployment/ate-api-server deployment/ate-controller \ deployment/atenet-router; do - if run_kubectl -n ate-system get "${workload}" >/dev/null 2>&1; then - run_kubectl -n ate-system rollout restart "${workload}" + if run_kubectl -n "${ATE_NAMESPACE}" get "${workload}" >/dev/null 2>&1; then + run_kubectl -n "${ATE_NAMESPACE}" rollout restart "${workload}" fi done # atelet DaemonSet names carry a version suffix; restart whichever versions # are installed. local ds="" - for ds in $(run_kubectl -n ate-system get daemonset -l app=atelet -o name 2>/dev/null); do - run_kubectl -n ate-system rollout restart "${ds}" + for ds in $(run_kubectl -n "${ATE_NAMESPACE}" get daemonset -l app=atelet -o name 2>/dev/null); do + run_kubectl -n "${ATE_NAMESPACE}" rollout restart "${ds}" done } @@ -460,7 +477,7 @@ create_jwt_authority_pool_secret() { run_kubectl_ate admin make-jwt-pool \ --key-id="1" \ --name="actor-id-jwt-pool" \ - --secret-namespace=ate-system + --secret-namespace="${ATE_NAMESPACE}" } create_actor_id_ca_pool_secret() { @@ -468,7 +485,7 @@ create_actor_id_ca_pool_secret() { run_kubectl_ate admin make-ca-pool \ --ca-id="1" \ --name="actor-id-ca-pool" \ - --secret-namespace=ate-system + --secret-namespace="${ATE_NAMESPACE}" } # The egress gateway has to verify actor client certificates, which means it @@ -482,7 +499,7 @@ create_actor_id_ca_certs_secret() { # inside the create-secret argument list, which would silently produce an # empty trust bundle and an egress gateway that rejects every actor. local actorid_root="" - actorid_root=$(ca_pool_root_pem actor-id-ca-pool ate-system) + actorid_root=$(ca_pool_root_pem actor-id-ca-pool "${ATE_NAMESPACE}") if [[ -z "${actorid_root}" ]]; then echo "error: failed to extract the actor-identity CA root for actor-id-ca-certs" >&2 return 1 @@ -490,7 +507,7 @@ create_actor_id_ca_certs_secret() { run_kubectl create secret generic actor-id-ca-certs \ --from-literal=ca.crt="${actorid_root}" \ - -n ate-system \ + -n "${ATE_NAMESPACE}" \ --dry-run=client -o yaml \ | run_kubectl apply -f - } @@ -504,7 +521,7 @@ create_egress_mitm_ca_pool_secret() { run_kubectl_ate admin make-ca-pool \ --ca-id="1" \ --name="egress-mitm-ca-pool" \ - --secret-namespace=ate-system \ + --secret-namespace="${ATE_NAMESPACE}" \ --key-type=ECDSAP256 } @@ -515,7 +532,7 @@ ensure_egress_mitm_ca_pool_secret() { if [[ "${ATE_EXPERIMENTAL_USE_SDSMINT:-false}" != "true" ]]; then return 0 fi - run_kubectl get secret -n ate-system egress-mitm-ca-pool >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" egress-mitm-ca-pool >/dev/null 2>&1 \ || create_egress_mitm_ca_pool_secret } @@ -544,7 +561,7 @@ wait_for_podcertificate_trust_bundles() { create_api_server_env_vars() { log_step "create_api_server_env_vars" - run_kubectl create namespace ate-system --dry-run=client -o yaml \ + run_kubectl create namespace "${ATE_NAMESPACE}" --dry-run=client -o yaml \ | run_kubectl apply -f - local postgres_connection_string="${ATE_API_POSTGRES_CONNECTION_STRING:-}" @@ -555,7 +572,7 @@ create_api_server_env_vars() { echo "POSTGRES_CONNECTION_STRING: ${postgres_connection_string}" - run_kubectl create configmap -n ate-system ate-api-server-envvars \ + run_kubectl create configmap -n "${ATE_NAMESPACE}" ate-api-server-envvars \ --from-literal=ATE_API_POSTGRES_CONNECTION_STRING="${postgres_connection_string}" \ --from-literal=ATE_API_POSTGRES_SCHEMA="${postgres_schema}" \ --dry-run=client -o yaml \ @@ -584,7 +601,7 @@ apply_podcert_workers_override() { create_api_authentication_config() { log_step "create_api_authentication_config" - run_kubectl create namespace ate-system --dry-run=client -o yaml \ + run_kubectl create namespace "${ATE_NAMESPACE}" --dry-run=client -o yaml \ | run_kubectl apply -f - local jwt_issuer="" @@ -604,8 +621,8 @@ create_api_authentication_config() { ;; esac local authentication_config - authentication_config=$(printf 'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n issuer: %s\n audiences: [api.ate-system.svc]\n%s' "${jwt_issuer}" "${discovery_config}") - run_kubectl create configmap -n ate-system ate-api-authentication \ + authentication_config=$(printf 'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n issuer: %s\n audiences: [api.%s.svc]\n%s' "${jwt_issuer}" "${ATE_NAMESPACE}" "${discovery_config}") + run_kubectl create configmap -n "${ATE_NAMESPACE}" ate-api-authentication \ --from-literal=authentication.yaml="${authentication_config}" \ --dry-run=client -o yaml \ | run_kubectl apply -f - @@ -666,8 +683,7 @@ deploy_ate_system() { ensure_substrate_version # Ensure namespace exists before applying RBAC or CRDs - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace # The atelet DaemonSet applied below and the demo WorkerPools' worker pods # schedule only to version-labeled nodes. @@ -729,13 +745,13 @@ deploy_ate_system() { log_step "Waiting for ATE system components to be ready..." if use_bundled_postgres; then - run_kubectl rollout status statefulset/postgres -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status statefulset/postgres -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" fi - run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/ate-controller -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/ate-api-server -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/ate-controller -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-router -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-egress -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" # After the bundle, which carries its own copy of ate-otel-config. apply_otel_endpoint_override @@ -744,18 +760,18 @@ deploy_ate_system() { # Ensure secrets and configmaps required by ate-apiserver ensure_apiserver_prerequisites() { log_step "ensure_apiserver_prerequisites" - run_kubectl get secret -n ate-system actor-id-jwt-pool >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" actor-id-jwt-pool >/dev/null 2>&1 \ || create_jwt_authority_pool_secret - run_kubectl get secret -n ate-system actor-id-ca-pool >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" actor-id-ca-pool >/dev/null 2>&1 \ || create_actor_id_ca_pool_secret # Derived from actor-id-ca-pool above, so it must come after it. - run_kubectl get secret -n ate-system actor-id-ca-certs >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" actor-id-ca-certs >/dev/null 2>&1 \ || create_actor_id_ca_certs_secret run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool >/dev/null 2>&1 \ || create_podcertificate_controller_cas # Always reconcile the PostgreSQL connection settings. create_api_server_env_vars - run_kubectl get configmap -n ate-system ate-api-authentication >/dev/null 2>&1 \ + run_kubectl get configmap -n "${ATE_NAMESPACE}" ate-api-authentication >/dev/null 2>&1 \ || create_api_authentication_config } @@ -765,15 +781,14 @@ deploy_ate_apiserver() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace ensure_apiserver_prerequisites apply_otel_config apply_otel_endpoint_override run_ko apply -f manifests/ate-install/ate-api-server.yaml - run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/ate-api-server -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" } deploy_atelet() { @@ -782,8 +797,7 @@ deploy_atelet() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace label_nodes_substrate_version apply_otel_config @@ -798,7 +812,7 @@ deploy_atelet() { manifest=$(run_ko resolve -f manifests/ate-install/atelet.yaml | substitute_version) fi echo "${manifest}" | run_kubectl apply -f - - run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" } deploy_atenet() { @@ -806,8 +820,7 @@ deploy_atenet() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace apply_otel_config apply_otel_endpoint_override @@ -819,9 +832,9 @@ deploy_atenet() { ensure_egress_mitm_ca_pool_secret apply_atenet_egress run_ko apply -f manifests/ate-install/atenet-dns.yaml - run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/dns -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-router -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-egress -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/dns -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" } # get_actor_state echoes the actor's state enum (e.g. ACTOR_STATE_SUSPENDED). @@ -889,7 +902,7 @@ delete_demo_actors_substrate() { return 1 fi - if ! run_kubectl get deployment/ate-api-server -n ate-system >/dev/null 2>&1; then + if ! run_kubectl get deployment/ate-api-server -n "${ATE_NAMESPACE}" >/dev/null 2>&1; then log_step "ate-api-server not found; skipping actor cleanup" return 0 fi @@ -1070,7 +1083,7 @@ delete_ate_system() { run_kubectl delete --ignore-not-found -f manifests/ate-install fi - run_kubectl delete --ignore-not-found -n ate-system daemonset -l app=atelet + run_kubectl delete --ignore-not-found -n "${ATE_NAMESPACE}" daemonset -l app=atelet run_kubectl delete --ignore-not-found \ -f manifests/ate-install/components/agentgateway/configmap.yaml run_kubectl delete --ignore-not-found -f manifests/ate-install/postgres.yaml diff --git a/internal/e2e/env.go b/internal/e2e/env.go index ccc2e7f6b6..d59d32f57c 100644 --- a/internal/e2e/env.go +++ b/internal/e2e/env.go @@ -17,6 +17,8 @@ package e2e import ( "fmt" "os" + + "github.com/agent-substrate/substrate/internal/installdefaults" ) // CheckEnv checks the list of env vars exist and returns their value. @@ -32,3 +34,17 @@ func CheckEnv(keys ...string) (map[string]string, error) { } return env, nil } + +// SystemNamespaceEnv names the namespace the substrate control plane under test +// was installed into. It mirrors the ATE_NAMESPACE hack/install-ate.sh +// installed with, and the --namespace the chart was released into. +const SystemNamespaceEnv = "E2E_SYSTEM_NAMESPACE" + +// SystemNamespace returns the namespace the control plane under test runs in, +// falling back to the canonical install namespace. +func SystemNamespace() string { + if ns := os.Getenv(SystemNamespaceEnv); ns != "" { + return ns + } + return installdefaults.SystemNamespace +} diff --git a/internal/e2e/preflight.go b/internal/e2e/preflight.go index 14cae171d3..771d84539c 100644 --- a/internal/e2e/preflight.go +++ b/internal/e2e/preflight.go @@ -41,7 +41,7 @@ func PreflightChecks() error { "ate-controller", "ate-api-server", } - namespace := "ate-system" + namespace := SystemNamespace() for _, depName := range deployments { dep, err := clients.K8s.AppsV1().Deployments(namespace).Get(ctx, depName, metav1.GetOptions{}) if err != nil { diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 7a9e536079..f673f319b8 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -36,8 +36,7 @@ import ( ) const ( - routerNamespace = "ate-system" - routerService = "atenet-router" + routerService = "atenet-router" // routerConnectServicePort is atenet-router's Service port for // CONNECT-tunneled traffic (see manifests/ate-install/atenet-router.yaml). // It is a distinct listener from the plain HTTP one Get/PostJSON use: @@ -79,7 +78,7 @@ func NewRouterClient(ctx context.Context) (*RouterClient, error) { return nil, fmt.Errorf("creating k8s client: %w", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, routerNamespace, routerService, 80) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, SystemNamespace(), routerService, 80) if err != nil { return nil, err } @@ -188,7 +187,7 @@ func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, // in one test don't each pay for a fresh port-forward. func (c *RouterClient) ensureConnectPortForward(ctx context.Context) error { c.connectOnce.Do(func() { - localPort, stop, err := portforward.ServicePortForward(ctx, c.config, c.clientset, routerNamespace, routerService, routerConnectServicePort) + localPort, stop, err := portforward.ServicePortForward(ctx, c.config, c.clientset, SystemNamespace(), routerService, routerConnectServicePort) if err != nil { c.connectErr = fmt.Errorf("port-forwarding to the router's CONNECT listener: %w", err) return diff --git a/internal/e2e/statusz.go b/internal/e2e/statusz.go index a04a07141d..a823307f79 100644 --- a/internal/e2e/statusz.go +++ b/internal/e2e/statusz.go @@ -51,7 +51,7 @@ func NewStatuszClient(ctx context.Context) (*StatuszClient, error) { return nil, fmt.Errorf("creating k8s client: %w", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, routerNamespace, routerService, routerStatusPort) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, SystemNamespace(), routerService, routerStatusPort) if err != nil { return nil, err } diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index cd6fcd237d..9fc7ccb3de 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -1264,13 +1264,13 @@ func callActorPathOnce(t *testing.T, actorRef resources.ActorRef, method, path s t.Helper() clients := e2e.GetClients() - svc, err := clients.K8s.CoreV1().Services("ate-system").Get(context.Background(), "atenet-router", metav1.GetOptions{}) + svc, err := clients.K8s.CoreV1().Services(e2e.SystemNamespace()).Get(context.Background(), "atenet-router", metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("failed to get atenet-router service: %w", err) } selector := labels.SelectorFromSet(svc.Spec.Selector).String() - pods, err := clients.K8s.CoreV1().Pods("ate-system").List(context.Background(), metav1.ListOptions{LabelSelector: selector}) + pods, err := clients.K8s.CoreV1().Pods(e2e.SystemNamespace()).List(context.Background(), metav1.ListOptions{LabelSelector: selector}) if err != nil { return "", fmt.Errorf("failed to list atenet-router pods: %w", err) } diff --git a/internal/e2e/suites/egressauthz/actoridentity_test.go b/internal/e2e/suites/egressauthz/actoridentity_test.go index ffe82e5e2a..b2bcce6c36 100644 --- a/internal/e2e/suites/egressauthz/actoridentity_test.go +++ b/internal/e2e/suites/egressauthz/actoridentity_test.go @@ -73,16 +73,16 @@ const ( // the secret ateapi signs with. func actorIdentityCA(t *testing.T, ctx context.Context) *localca.CA { t.Helper() - secret, err := e2e.GetClients().K8s.CoreV1().Secrets(egressNamespace).Get(ctx, actorIDCASecret, metav1.GetOptions{}) + secret, err := e2e.GetClients().K8s.CoreV1().Secrets(e2e.SystemNamespace()).Get(ctx, actorIDCASecret, metav1.GetOptions{}) if err != nil { - t.Fatalf("reading actor-identity CA pool secret %s/%s: %v", egressNamespace, actorIDCASecret, err) + t.Fatalf("reading actor-identity CA pool secret %s/%s: %v", e2e.SystemNamespace(), actorIDCASecret, err) } pool, err := localca.Unmarshal(secret.Data[actorIDCASecretKey]) if err != nil { - t.Fatalf("parsing actor-identity CA pool from %s/%s key %q: %v", egressNamespace, actorIDCASecret, actorIDCASecretKey, err) + t.Fatalf("parsing actor-identity CA pool from %s/%s key %q: %v", e2e.SystemNamespace(), actorIDCASecret, actorIDCASecretKey, err) } if len(pool.CAs) == 0 { - t.Fatalf("actor-identity CA pool %s/%s contains no CA", egressNamespace, actorIDCASecret) + t.Fatalf("actor-identity CA pool %s/%s contains no CA", e2e.SystemNamespace(), actorIDCASecret) } // CAs[0] is the one that signs: ateapi's MintCert makes the same choice. return pool.CAs[0] diff --git a/internal/e2e/suites/egressauthz/egressauthz_test.go b/internal/e2e/suites/egressauthz/egressauthz_test.go index 1fa04ffaff..3885f8db8f 100644 --- a/internal/e2e/suites/egressauthz/egressauthz_test.go +++ b/internal/e2e/suites/egressauthz/egressauthz_test.go @@ -55,7 +55,6 @@ import ( const ( // Where the gateway's CA lives, fixed by hack/install-ate.sh. - egressNamespace = "ate-system" probeName = "egressprobe" ) diff --git a/internal/e2e/suites/metrics/metrics_test.go b/internal/e2e/suites/metrics/metrics_test.go index 38d61dba00..a8a59fa97a 100644 --- a/internal/e2e/suites/metrics/metrics_test.go +++ b/internal/e2e/suites/metrics/metrics_test.go @@ -43,7 +43,7 @@ func TestPlatformMetricsEmitted(t *testing.T) { tmpl := e2e.SubstrateCounterFixture() actorID := fmt.Sprintf("metrics-probe-%d", time.Now().UnixNano()) metricPrefixes := append([]string(nil), e2e.PlatformMetricPrefixes...) - router, err := clients.K8s.AppsV1().Deployments("ate-system").Get(ctx, "atenet-router", metav1.GetOptions{}) + router, err := clients.K8s.AppsV1().Deployments(e2e.SystemNamespace()).Get(ctx, "atenet-router", metav1.GetOptions{}) if err != nil { t.Fatalf("Get atenet-router deployment: %v", err) } diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 331cfdd61a..cf555cec7c 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -311,7 +311,7 @@ func routerAddress(t *testing.T, ctx context.Context) string { if err != nil { t.Fatalf("creating k8s client: %v", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, "ate-system", "atenet-router", 80) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, e2e.SystemNamespace(), "atenet-router", 80) if err != nil { t.Fatalf("port-forwarding to the router: %v", err) } diff --git a/internal/e2e/suites/networkpolicy/networkpolicy_test.go b/internal/e2e/suites/networkpolicy/networkpolicy_test.go index 54a929c3ce..22b7357f9c 100644 --- a/internal/e2e/suites/networkpolicy/networkpolicy_test.go +++ b/internal/e2e/suites/networkpolicy/networkpolicy_test.go @@ -36,7 +36,6 @@ import ( ) const ( - ateSystemNamespace = "ate-system" atenetRouterAppName = "atenet-router" ) @@ -103,8 +102,8 @@ func TestNetworkPolicyLifecycleAndReconciliation(t *testing.T) { t.Fatalf("expected exactly 1 ingress from peer, got %d", len(ingressRule.From)) } fromPeer := ingressRule.From[0] - if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != ateSystemNamespace { - t.Errorf("expected namespace selector for %s, got %v", ateSystemNamespace, fromPeer.NamespaceSelector) + if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != e2e.SystemNamespace() { + t.Errorf("expected namespace selector for %s, got %v", e2e.SystemNamespace(), fromPeer.NamespaceSelector) } if fromPeer.PodSelector == nil || fromPeer.PodSelector.MatchLabels["app"] != atenetRouterAppName { t.Errorf("expected pod selector for %s, got %v", atenetRouterAppName, fromPeer.PodSelector) diff --git a/internal/e2e/trustbundle.go b/internal/e2e/trustbundle.go index 948cf31015..30bc56795d 100644 --- a/internal/e2e/trustbundle.go +++ b/internal/e2e/trustbundle.go @@ -39,7 +39,6 @@ const ( // EgressTrustBundleObjectName is the reconciler-owned ClusterTrustBundle. EgressTrustBundleObjectName = "egress-mitm.ate.dev:mitm:primary-bundle" - egressCAPoolNamespace = "ate-system" egressCAPoolSecretName = "egress-mitm-ca-pool" egressCAPoolSecretKey = "pool" ) @@ -68,12 +67,12 @@ func ReplaceEgressTrustPool(t *testing.T, ctx context.Context, clients *Clients, if !createEgressTrustPool(t, ctx, clients, secret) { // Took over an existing pool: overwrite its contents without adopting // its cleanup, since whoever created it registered one already. - existing, err := clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Get(ctx, egressCAPoolSecretName, metav1.GetOptions{}) + existing, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Get(ctx, egressCAPoolSecretName, metav1.GetOptions{}) if err != nil { t.Fatalf("reading existing CA pool secret: %v", err) } existing.Data = secret.Data - if _, err := clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Update(ctx, existing, metav1.UpdateOptions{}); err != nil { + if _, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Update(ctx, existing, metav1.UpdateOptions{}); err != nil { t.Fatalf("updating CA pool secret: %v", err) } } @@ -103,7 +102,7 @@ func newEgressTrustPool(t *testing.T) (*corev1.Secret, string) { t.Fatalf("encoding the egress CA private key: %v", err) } return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: egressCAPoolNamespace, Name: egressCAPoolSecretName}, + ObjectMeta: metav1.ObjectMeta{Namespace: SystemNamespace(), Name: egressCAPoolSecretName}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{ egressCAPoolSecretKey: poolBytes, @@ -120,14 +119,14 @@ func newEgressTrustPool(t *testing.T) (*corev1.Secret, string) { // behind, and no caller removes a pool it merely found. func createEgressTrustPool(t *testing.T, ctx context.Context, clients *Clients, secret *corev1.Secret) bool { t.Helper() - if _, err := clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Create(ctx, secret, metav1.CreateOptions{}); err != nil { if !apierrors.IsAlreadyExists(err) { - t.Fatalf("creating CA pool secret %s/%s: %v", egressCAPoolNamespace, egressCAPoolSecretName, err) + t.Fatalf("creating CA pool secret %s/%s: %v", SystemNamespace(), egressCAPoolSecretName, err) } return false } t.Cleanup(func() { - _ = clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Delete(context.Background(), egressCAPoolSecretName, metav1.DeleteOptions{}) + _ = clients.K8s.CoreV1().Secrets(SystemNamespace()).Delete(context.Background(), egressCAPoolSecretName, metav1.DeleteOptions{}) }) return true } From 62b0e234a8380ee38ea4eda87c360c3981b05a3f Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Thu, 3 Sep 2026 09:06:57 -0400 Subject: [PATCH 3/8] chart: give ate-controller its namespace and atelet its ateapi address MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps that only appear once substrate is installed outside ate-system, both found by running the helm-e2e flow against a relocated release. ate-controller reads the namespace from the downward API to build the atelet and atenet-router SPIFFE identities it hands to each worker's atunnel, but its template projected no POD_NAMESPACE. It fell back to the compiled-in default and stamped every worker with spiffe://cluster.local/ns/ate-system/sa/atelet, so atunnel rejected the credential broker with "credential broker is not atelet" and no actor could obtain a certificate. Five other templates already project the variable; ate-controller was the only consumer that did not, and the rendered manifest under manifests/ate-install/ has carried it all along. atelet's credential broker dials ateapi to mint those certificates. The chart passes a resolved --ateapi-address to ate-controller, atenet-router and atenet-egress, but not to atelet, which fell back to k8s:///api.ate-system.svc:443 and failed with "k8sresolver: watch error for ate-system/api ... endpointslices is forbidden" — the RBAC it has covers the release namespace, not the one it was dialing. --ateapi-server-name has the same default and the same problem, since it is the name checked on ateapi's serving cert. --- .../substrate/templates/ate-controller.yaml | 10 +++++- charts/substrate/templates/atelet.yaml | 6 ++++ internal/ateclient/builder.go | 33 +++++++++++++++---- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index 31c83b9066..dfb24d2e32 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -79,8 +79,16 @@ spec: - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -{{- if .Values.otel.endpoint }} env: + # ate-controller resolves substrate's namespace from the downward API. + # It names the atelet and atenet-router SPIFFE identities handed to + # each worker's atunnel, which live in substrate's namespace and not + # the worker's, so the controller cannot infer them any other way. + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- if .Values.otel.endpoint }} - name: OTEL_EXPORTER_OTLP_ENDPOINT value: {{ .Values.otel.endpoint | quote }} {{- end }} diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 916915dc42..1570d0f8ff 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -104,6 +104,12 @@ spec: - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem + # The atelet binary defaults these to api.ate-system.svc, which is + # correct only for the canonical render. The credential broker dials + # ateapi to mint actor certificates, so pass the chart-resolved + # Service and the matching name on its serving cert. + - --ateapi-address={{ include "substrate.ateApi.endpoint" . }} + - --ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc {{- with .Values.atelet.extraArgs }} {{ toYaml . | indent 8 }} {{- end }} diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 6765db009d..7ef37182c9 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -44,9 +44,28 @@ import ( metricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned" ) -const ( - apiServerName = "api.ate-system.svc" +// NamespaceEnv overrides the namespace the client looks for substrate in. The +// client runs outside the cluster, so it has no downward API to read and no +// pod namespace to fall back on; it matches the ATE_NAMESPACE that +// hack/install-ate.sh installed with. +const NamespaceEnv = "ATE_NAMESPACE" + +// systemNamespace is the namespace the client expects ateapi to be running in. +func systemNamespace() string { + if ns := os.Getenv(NamespaceEnv); ns != "" { + return ns + } + return installdefaults.SystemNamespace +} +// apiServerName is the in-cluster DNS name of the ateapi Service. It is both +// the SNI presented on the connection and the audience of the minted token, so +// it has to track the namespace ateapi actually runs in. +func apiServerName() string { + return fmt.Sprintf("%s.%s.svc", installdefaults.APIServiceName, systemNamespace()) +} + +const ( // serviceDNSSignerName and liveBundleSelector mirror the // clusterTrustBundle projected-volume sources that in-cluster clients // mount to verify ateapi's serving cert. @@ -84,7 +103,7 @@ func (c *Client) Close() { } // NewClient creates a new Ate API client. If endpoint is empty, it automatically port-forwards -// to the ate-api-server pod in the ate-system namespace. +// to the ate-api-server pod in substrate's namespace. func NewClient(ctx context.Context, kubeconfigPath, k8sContext, endpoint, tokenFile string, traceEnabled bool) (*Client, error) { tp, err := initTracing(ctx, traceEnabled) if err != nil { @@ -168,7 +187,7 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext, tokenFile // TODO: Should we special-case a LoadBalancer "api" Service and dial its // address directly instead of port-forwarding? - localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, installdefaults.SystemNamespace, installdefaults.APIServiceName, 443) + localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, systemNamespace(), installdefaults.APIServiceName, 443) if err != nil { return nil, err } @@ -233,7 +252,7 @@ func serverTLSConfig(ctx context.Context, clientset kubernetes.Interface) (*tls. return &tls.Config{ MinVersion: tls.VersionTLS13, RootCAs: pool, - ServerName: apiServerName, + ServerName: apiServerName(), }, nil } @@ -253,11 +272,11 @@ func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset, expirationSeconds := int64(3600) tokenRequest := &authv1.TokenRequest{ Spec: authv1.TokenRequestSpec{ - Audiences: []string{apiServerName}, + Audiences: []string{apiServerName()}, ExpirationSeconds: &expirationSeconds, }, } - token, err := clientset.CoreV1().ServiceAccounts(installdefaults.SystemNamespace).CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) + token, err := clientset.CoreV1().ServiceAccounts(systemNamespace()).CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to request ateapi bearer token: %w", err) } From 3caf4cc5c53682f80475ac57a54799db15ec1910 Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Thu, 3 Sep 2026 09:15:51 -0400 Subject: [PATCH 4/8] e2e: point the egress probe at the gateway in substrate's namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe hardcoded atenet-egress.ate-system.svc:443 as its --gateway-address default and nothing overrode it, so against a relocated install every egressauthz case failed in the tunnel stage with "no such host" — before reaching the ext_proc identity check the tests exist to cover, which reported as an authorization failure rather than a lookup one. The probe runs inside the sandbox and cannot infer substrate's namespace, so the suite substitutes it into the manifest the same way it already substitutes the probe's own namespace. --- internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl | 6 +++++- internal/e2e/suites/egressauthz/egressauthz_test.go | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl b/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl index 97b78a7137..9ab467ec5b 100644 --- a/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl @@ -15,7 +15,7 @@ # The egress probe used by internal/e2e/suites/egressauthz: testserver's probe # subcommand, with the credential volumes that suite mints. It is not a # ServerPod because it needs those volumes and a namespace the suite populates -# first, so it stays a bespoke manifest. ${NAMESPACE} is substituted by the +# first, so it stays a bespoke manifest. ${NAMESPACE} and ${SYSTEM_NAMESPACE} are substituted by the # suite with the randomized namespace it created, so the probe is torn down with # that namespace and leaves nothing behind. apiVersion: v1 @@ -33,6 +33,10 @@ spec: args: - "egressprobe" - "--listen=:8080" + # The gateway lives in substrate's namespace, which the probe cannot infer + # from inside the sandbox. ${SYSTEM_NAMESPACE} is substituted alongside + # ${NAMESPACE} by the suite. + - "--gateway-address=atenet-egress.${SYSTEM_NAMESPACE}.svc:443" ports: - name: http containerPort: 8080 diff --git a/internal/e2e/suites/egressauthz/egressauthz_test.go b/internal/e2e/suites/egressauthz/egressauthz_test.go index 3885f8db8f..3b39da7162 100644 --- a/internal/e2e/suites/egressauthz/egressauthz_test.go +++ b/internal/e2e/suites/egressauthz/egressauthz_test.go @@ -162,6 +162,7 @@ func startProbe(t *testing.T, ctx context.Context) *probeClient { } manifest := filepath.Join(t.TempDir(), "egressprobe.yaml") rendered := strings.ReplaceAll(string(tmpl), "${NAMESPACE}", ns) + rendered = strings.ReplaceAll(rendered, "${SYSTEM_NAMESPACE}", e2e.SystemNamespace()) if err := os.WriteFile(manifest, []byte(rendered), 0o644); err != nil { t.Fatalf("writing rendered egressprobe manifest: %v", err) } From 4daeaf4b9d0035fc83b2742a3bcae03b59d8a52b Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Thu, 3 Sep 2026 09:32:14 -0400 Subject: [PATCH 5/8] Align with the upstream configurability change Two pieces of the upstream branch this fork's version had not picked up. The atelet dialer resolved the namespace from POD_NAMESPACE at the point of use, independently of the informer that was already given one. Both read the same variable so the behaviour matches, but the namespace is now threaded in alongside the informer's, which keeps a single resolution point and matches upstream so the eventual rebase is a no-op here. The trust-domain and service-account segments move to installdefaults with it. atenet-router's /statusz looked up its own ClusterIP under the hardcoded Service name atenet-router, which is wrong for any release this chart prefixes. It takes --router-service-name, and the chart passes the name it actually rendered. --- charts/substrate/templates/atenet-router.yaml | 3 ++ cmd/ateapi/internal/controlapi/dialer.go | 19 +++++-------- cmd/ateapi/internal/controlapi/dialer_test.go | 12 ++++---- .../controlapi/functionaltest/common_test.go | 2 +- .../controlapi/workflow_suspend_test.go | 3 +- cmd/ateapi/main.go | 2 +- cmd/atenet/internal/router/cmd.go | 2 ++ cmd/atenet/internal/router/config.go | 28 +++++++++++-------- cmd/atenet/internal/router/status.go | 2 +- 9 files changed, 39 insertions(+), 34 deletions(-) diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 78976a39a6..347d53597e 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -210,6 +210,9 @@ spec: - "--port-extproc=50051" - "--extproc-address=127.0.0.1" - "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + # /statusz looks up this instance's own ClusterIP by Service name, and + # the chart prefixes that name for any release not called "substrate". + - "--router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }}" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" - "--status-port=4040" diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 557d994aad..cccb5526b4 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -49,12 +49,7 @@ var ErrNoAteletOnNode = errors.New("no atelet pod found on node") // The SPIFFE identity that atelet serving certs carry, as minted by the // podidentity signer (cmd/podcertcontroller/internal/podidentitysigner). -// The namespace part is ateletNamespace, declared in informer.go. -const ( - trustDomainName = "cluster.local" - ateletSA = "atelet" -) - +// The namespace part is the dialer's ateletNamespace. // AteletDialer handles gRPC connections to Atelet pods. type AteletDialer struct { workerIndexer cache.Indexer @@ -83,13 +78,13 @@ func WithInsecureCredentials() DialerOption { // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet connection. -func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { +func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, ateletNamespace, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { d := &AteletDialer{ workerIndexer: workerIndexer, ateletIndexer: ateletIndexer, ateletConns: newAteletConnCache(1024), dialCredentials: func(expectedPodUID string) (credentials.TransportCredentials, error) { - tlsConfig, err := buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID) + tlsConfig, err := buildTLSConfig(ateletNamespace, clientBundlePath, serverCAPath, expectedPodUID) if err != nil { return nil, err } @@ -196,16 +191,16 @@ func (d *AteletDialer) DialForAteletOnNode(nodeName string) (*grpc.ClientConn, e return ateletConn, nil } -func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { - trustDomain, err := spiffeid.TrustDomainFromString(trustDomainName) +func buildTLSConfig(ateletNamespace, clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { + trustDomain, err := spiffeid.TrustDomainFromString(installdefaults.AteletTrustDomain) if err != nil { - return nil, fmt.Errorf("while parsing trust domain %q: %w", trustDomainName, err) + return nil, fmt.Errorf("while parsing trust domain %q: %w", installdefaults.AteletTrustDomain, err) } bundle, err := x509bundle.Load(trustDomain, serverCAPath) if err != nil { return nil, fmt.Errorf("while loading CA bundle from %s: %w", serverCAPath, err) } - expectedID, err := spiffeid.FromSegments(trustDomain, "ns", installdefaults.NamespaceFromPodEnv(), "sa", ateletSA) + expectedID, err := spiffeid.FromSegments(trustDomain, "ns", ateletNamespace, "sa", installdefaults.AteletServiceAccount) if err != nil { return nil, fmt.Errorf("while building expected atelet SPIFFE ID: %w", err) } diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 2fb103bc33..09df0b2fb1 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -43,12 +43,12 @@ import ( const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" func TestAteletDialerInsecureRequiresOptIn(t *testing.T) { - secure := NewAteletDialer(nil, nil, "", "") + secure := NewAteletDialer(nil, nil, installdefaults.SystemNamespace, "", "") if _, err := secure.dialCredentials("pod-uid"); err == nil { t.Fatal("secure dialer accepted empty credential paths") } - insecureDialer := NewAteletDialer(nil, nil, "", "", WithInsecureCredentials()) + insecureDialer := NewAteletDialer(nil, nil, installdefaults.SystemNamespace, "", "", WithInsecureCredentials()) creds, err := insecureDialer.dialCredentials("pod-uid") if err != nil { t.Fatalf("insecure dial credentials: %v", err) @@ -380,7 +380,7 @@ func TestDialForAteletOnNode(t *testing.T) { } t.Run("no atelet on node", func(t *testing.T) { - d := NewAteletDialer(nil, newTestAteletIndexer(t), "", "") + d := NewAteletDialer(nil, newTestAteletIndexer(t), installdefaults.SystemNamespace, "", "") if _, err := d.DialForAteletOnNode("node1"); !errors.Is(err, ErrNoAteletOnNode) { t.Fatalf("DialForAteletOnNode = %v, want ErrNoAteletOnNode", err) } @@ -390,7 +390,7 @@ func TestDialForAteletOnNode(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), ateletPod("atelet-2", "uid-2", "node1", "10.0.0.2"), - ), "", "") + ), installdefaults.SystemNamespace, "", "") _, err := d.DialForAteletOnNode("node1") if err == nil || errors.Is(err, ErrNoAteletOnNode) { t.Fatalf("DialForAteletOnNode = %v, want a non-ErrNoAteletOnNode error", err) @@ -400,7 +400,7 @@ func TestDialForAteletOnNode(t *testing.T) { t.Run("dials and caches the node's atelet", func(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), - ), "", "") + ), installdefaults.SystemNamespace, "", "") var credsUID string d.dialCredentials = func(expectedPodUID string) (credentials.TransportCredentials, error) { credsUID = expectedPodUID @@ -427,7 +427,7 @@ func TestDialForAteletOnNode(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), ateletPod("atelet-2", "uid-2", "node2", "10.0.0.2"), - ), "", "", WithDialCredentials(func(string) (credentials.TransportCredentials, error) { + ), installdefaults.SystemNamespace, "", "", WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil })) d.ateletConns = newAteletConnCache(1) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 2c5afa464e..7203434d0c 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -152,7 +152,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu // Dial the fake atelet over insecure transport instead of per-atelet mTLS, // so DialForWorker's real lookup/dial/cache path is exercised under test. - dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), "", "", + dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), installdefaults.SystemNamespace, "", "", controlapi.WithDialCredentials(func(_ string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil })) diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index e7dfdf659a..5a4a2dc7de 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -21,6 +21,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -218,7 +219,7 @@ func newDanglingDialer() *AteletDialer { byNamespaceAndName: func(obj any) ([]string, error) { return nil, nil }, byNode: func(obj any) ([]string, error) { return nil, nil }, }) - return NewAteletDialer(empty, empty, "", "") + return NewAteletDialer(empty, empty, installdefaults.SystemNamespace, "", "") } func TestEnsureAteletSuspended_DanglingWorkerDoesNotRecordPhantomSnapshot(t *testing.T) { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 9cb50f49b7..786fb8f995 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -199,7 +199,7 @@ func main() { if *ateletInsecure { dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) } - ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), ateletNamespace, *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) controlSrv := controlapi.NewRPCService(persistence, workerCache, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, *actorWorkflowDeadline, volPlugins) // Drive stored ActorTemplates through the golden actor flow. diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 3c70cb92cc..ba22d18060 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/cobra" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" + "github.com/agent-substrate/substrate/internal/installdefaults" ) func NewRouterCmd() *cobra.Command { @@ -46,6 +47,7 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().StringVar(&cfg.MetricsAddr, "metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") cmd.Flags().StringVar(&cfg.AtenetRouter, "atenet-router", string(atenetRouterEnvoy), "Router dataplane: envoy or agentgateway") cmd.Flags().StringVar(&cfg.Namespace, "namespace", "default", "Target operations namespace") + cmd.Flags().StringVar(&cfg.RouterServiceName, "router-service-name", installdefaults.RouterServiceName, "Service name of this atenet-router in the operations namespace. Override when the deployment renames the Service.") cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") cmd.Flags().StringVar(&cfg.AteapiAddr, "ateapi-address", "k8s:///api.ate-system.svc:443", "gRPC dial target for the cluster ateapi Control instance.") cmd.Flags().IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router") diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 16297ee68a..059b342ad1 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -74,18 +74,22 @@ type authConfig struct { // routerConfig holds deployment setup and endpoint options for the router node instance. type routerConfig struct { // Mode restricts the instance to one traffic direction. Empty means ModeAll. - Mode Mode - AtenetRouter string - Namespace string - Kubeconfig string - AteapiAddr string - HttpPort int - XdsPort int - ExtprocPort int - ExtprocAddr string - StatusPort int - HealthInterval time.Duration - HttpsPort int + Mode Mode + AtenetRouter string + Namespace string + // RouterServiceName is the Service name of this atenet-router in the + // operations namespace, used by /statusz to look up its own ClusterIP. + // Defaults to installdefaults.RouterServiceName. + RouterServiceName string + Kubeconfig string + AteapiAddr string + HttpPort int + XdsPort int + ExtprocPort int + ExtprocAddr string + StatusPort int + HealthInterval time.Duration + HttpsPort int // ConnectPlainTextPort and ConnectTLSPort are the plaintext and TLS // listener ports for CONNECT-tunneled traffic. Non-positive disables the // corresponding listener. diff --git a/cmd/atenet/internal/router/status.go b/cmd/atenet/internal/router/status.go index 6878755580..9fe196908f 100644 --- a/cmd/atenet/internal/router/status.go +++ b/cmd/atenet/internal/router/status.go @@ -68,7 +68,7 @@ func (s *RouterServer) getRouterIP(ctx context.Context) string { return "Offline Mode (No Cluster IP)" } - svc, err := s.clientset.CoreV1().Services(s.cfg.Namespace).Get(ctx, "atenet-router", metav1.GetOptions{}) + svc, err := s.clientset.CoreV1().Services(s.cfg.Namespace).Get(ctx, s.cfg.RouterServiceName, metav1.GetOptions{}) if err != nil { return fmt.Sprintf("Lookup Failed: %v", err) } From 91cf4d5740af89071299cff235da24c06c30a365 Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Thu, 3 Sep 2026 10:06:23 -0400 Subject: [PATCH 6/8] Make the atelet and router ServiceAccount names configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A SPIFFE ID names a ServiceAccount as well as a namespace, and the ServiceAccount name was still a constant. This chart prefixes every resource name for a release not called "substrate" — which is every install that consumes substrate as a subchart, since a subchart inherits the parent's release name — so ateapi and each worker's atunnel expected an account that does not exist and rejected the peer that does. ateapi takes --atelet-service-account and ate-controller takes that plus --router-service-account, both defaulting to the canonical names, and the chart passes the names it actually rendered. ateapi's dialer and actoridentity now take the whole expected identity rather than a namespace they rebuild it from. Both only ever used the namespace to construct that one string, and having a single place decide what atelet's identity is beats two places agreeing on how to spell it. --- .../substrate/templates/ate-api-server.yaml | 3 ++ .../substrate/templates/ate-controller.yaml | 6 +++ .../internal/actoridentity/actoridentity.go | 17 +++---- .../actoridentity/actoridentity_test.go | 14 +++-- cmd/ateapi/internal/controlapi/dialer.go | 10 ++-- cmd/ateapi/internal/controlapi/dialer_test.go | 12 ++--- .../controlapi/functionaltest/common_test.go | 2 +- .../controlapi/workflow_suspend_test.go | 2 +- cmd/ateapi/main.go | 8 +-- .../internal/controllers/workerpool_apply.go | 6 +-- .../controllers/workerpool_apply_test.go | 51 +++++++++++++++---- .../controllers/workerpool_controller.go | 15 ++++-- cmd/atecontroller/main.go | 5 ++ 13 files changed, 102 insertions(+), 49 deletions(-) diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index c236999e48..4fa447a91c 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -89,6 +89,9 @@ spec: - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" + # ateapi verifies atelet's SPIFFE ID, which embeds the ServiceAccount + # name this chart prefixes for a renamed release. + - "--atelet-service-account={{ include "substrate.fullname" (list "atelet" .) }}" - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" - "--drain-delay=13s" diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index dfb24d2e32..6a6ac462c0 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -78,6 +78,12 @@ spec: # dials the right backend when substrate is installed as a subchart. - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + # A SPIFFE ID names a ServiceAccount, and this chart prefixes those for + # any release not called "substrate". The controller stamps these + # identities onto every worker, so it has to be told the names actually + # rendered rather than assume the canonical ones. + - "--atelet-service-account={{ include "substrate.fullname" (list "atelet" .) }}" + - "--router-service-account={{ include "substrate.fullname" (list "atenet-router" .) }}" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" env: # ate-controller resolves substrate's namespace from the downward API. diff --git a/cmd/ateapi/internal/actoridentity/actoridentity.go b/cmd/ateapi/internal/actoridentity/actoridentity.go index 9250c997cd..4a1abbc701 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity.go @@ -31,7 +31,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/principal" @@ -61,21 +60,21 @@ type Server struct { store store.Interface workers *workercache.Cache - // ateletNamespace is the namespace segment of the SPIFFE ID that the - // calling atelet must present to mint actor credentials. - ateletNamespace string + // ateletSPIFFEID is the identity the calling atelet must present to mint + // actor credentials. + ateletSPIFFEID string } var _ ateapipb.ActorIdentityServer = (*Server)(nil) -func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache, ateletNamespace string) *Server { +func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache, ateletSPIFFEID string) *Server { return &Server{ actorIdentityJWTIssuer: actorIdentityJWTIssuer, actorIDJWTPoolFile: actorIDJWTPoolFile, actorIDCAPool: actorIDCAPool, store: store, workers: workers, - ateletNamespace: ateletNamespace, + ateletSPIFFEID: ateletSPIFFEID, } } @@ -256,11 +255,7 @@ func (s *Server) authenticateAtelet(ctx context.Context) (*ateletCaller, error) // Only atelet may mint actor credentials. Everything else with a valid // pod-identity certificate — including the actor workloads themselves — is // rejected here. - expected := (&url.URL{ - Scheme: "spiffe", - Host: installdefaults.AteletTrustDomain, - Path: path.Join("ns", s.ateletNamespace, "sa", installdefaults.AteletServiceAccount), - }).String() + expected := s.ateletSPIFFEID if len(leaf.URIs) == 0 || leaf.URIs[0].String() != expected { slog.WarnContext(ctx, "ActorIdentity denied: caller is not atelet", slog.Any("uris", leaf.URIs), slog.String("expected", expected)) diff --git a/cmd/ateapi/internal/actoridentity/actoridentity_test.go b/cmd/ateapi/internal/actoridentity/actoridentity_test.go index 72e8b1cff6..cc8b935ef3 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity_test.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity_test.go @@ -84,6 +84,10 @@ const ( ateletSA = installdefaults.AteletServiceAccount ) +// ateletSPIFFEID is the identity a default install's atelet presents, and what +// the Server under test is configured to accept. +var ateletSPIFFEID = installdefaults.SPIFFEID(ateletNamespace, ateletSA) + func newTestCert(t *testing.T, spiffePath string, podIdentity *substratex509.PodIdentity) *x509.Certificate { t.Helper() @@ -175,7 +179,7 @@ func newTestServer(t *testing.T, st store.Interface) *Server { t.Fatalf("start worker cache: %v", err) } } - return New("issuer", "", pool, st, workers, ateletNamespace) + return New("issuer", "", pool, st, workers, ateletSPIFFEID) } // staleWatchStore wraps a store with a WatchWorkers that never delivers, @@ -334,11 +338,11 @@ func newTestServerWithCache(t *testing.T, st store.Interface, workers *workercac t.Fatalf("generate CA: %v", err) } pool := &localca.ConcretePool{CAs: []*localca.CA{ca}} - return New("issuer", "", pool, st, workers, ateletNamespace) + return New("issuer", "", pool, st, workers, ateletSPIFFEID) } func TestMintJWTRequiresConfiguredJWTProvider(t *testing.T) { - srv := &Server{actorIdentityJWTIssuer: "https://kubernetes.example", ateletNamespace: ateletNamespace} + srv := &Server{actorIdentityJWTIssuer: "https://kubernetes.example", ateletSPIFFEID: ateletSPIFFEID} for _, tt := range []struct { name string ctx context.Context @@ -928,7 +932,7 @@ func TestMintCertAuthorizesBeforeSigning(t *testing.T) { ActiveForSigning: "test-actor-ca", } - srv := New("issuer", "", pool, st, workers, ateletNamespace) + srv := New("issuer", "", pool, st, workers, ateletSPIFFEID) actor, err := st.GetActor(ctx, resources.ActorRef{Atespace: testAtespace, Name: testActorName}) if err != nil { @@ -1104,7 +1108,7 @@ func TestValidateMintCertRequest(t *testing.T) { func TestAuthenticateAteletHonorsConfiguredNamespace(t *testing.T) { const relocated = "substrate-test" - srv := &Server{ateletNamespace: relocated} + srv := &Server{ateletSPIFFEID: installdefaults.SPIFFEID(relocated, ateletSA)} t.Run("accepts atelet from the configured namespace", func(t *testing.T) { id := podIdentityOn(testNode) diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index cccb5526b4..c6f6a638da 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -78,13 +78,13 @@ func WithInsecureCredentials() DialerOption { // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet connection. -func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, ateletNamespace, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { +func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, ateletSPIFFEID, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { d := &AteletDialer{ workerIndexer: workerIndexer, ateletIndexer: ateletIndexer, ateletConns: newAteletConnCache(1024), dialCredentials: func(expectedPodUID string) (credentials.TransportCredentials, error) { - tlsConfig, err := buildTLSConfig(ateletNamespace, clientBundlePath, serverCAPath, expectedPodUID) + tlsConfig, err := buildTLSConfig(ateletSPIFFEID, clientBundlePath, serverCAPath, expectedPodUID) if err != nil { return nil, err } @@ -191,7 +191,7 @@ func (d *AteletDialer) DialForAteletOnNode(nodeName string) (*grpc.ClientConn, e return ateletConn, nil } -func buildTLSConfig(ateletNamespace, clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { +func buildTLSConfig(ateletSPIFFEID, clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { trustDomain, err := spiffeid.TrustDomainFromString(installdefaults.AteletTrustDomain) if err != nil { return nil, fmt.Errorf("while parsing trust domain %q: %w", installdefaults.AteletTrustDomain, err) @@ -200,9 +200,9 @@ func buildTLSConfig(ateletNamespace, clientBundlePath, serverCAPath, expectedPod if err != nil { return nil, fmt.Errorf("while loading CA bundle from %s: %w", serverCAPath, err) } - expectedID, err := spiffeid.FromSegments(trustDomain, "ns", ateletNamespace, "sa", installdefaults.AteletServiceAccount) + expectedID, err := spiffeid.FromString(ateletSPIFFEID) if err != nil { - return nil, fmt.Errorf("while building expected atelet SPIFFE ID: %w", err) + return nil, fmt.Errorf("while parsing expected atelet SPIFFE ID %q: %w", ateletSPIFFEID, err) } verify, err := verifyAteletServerCert(bundle, expectedID, expectedPodUID) diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 09df0b2fb1..b26a71ef35 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -43,12 +43,12 @@ import ( const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" func TestAteletDialerInsecureRequiresOptIn(t *testing.T) { - secure := NewAteletDialer(nil, nil, installdefaults.SystemNamespace, "", "") + secure := NewAteletDialer(nil, nil, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") if _, err := secure.dialCredentials("pod-uid"); err == nil { t.Fatal("secure dialer accepted empty credential paths") } - insecureDialer := NewAteletDialer(nil, nil, installdefaults.SystemNamespace, "", "", WithInsecureCredentials()) + insecureDialer := NewAteletDialer(nil, nil, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", WithInsecureCredentials()) creds, err := insecureDialer.dialCredentials("pod-uid") if err != nil { t.Fatalf("insecure dial credentials: %v", err) @@ -380,7 +380,7 @@ func TestDialForAteletOnNode(t *testing.T) { } t.Run("no atelet on node", func(t *testing.T) { - d := NewAteletDialer(nil, newTestAteletIndexer(t), installdefaults.SystemNamespace, "", "") + d := NewAteletDialer(nil, newTestAteletIndexer(t), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") if _, err := d.DialForAteletOnNode("node1"); !errors.Is(err, ErrNoAteletOnNode) { t.Fatalf("DialForAteletOnNode = %v, want ErrNoAteletOnNode", err) } @@ -390,7 +390,7 @@ func TestDialForAteletOnNode(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), ateletPod("atelet-2", "uid-2", "node1", "10.0.0.2"), - ), installdefaults.SystemNamespace, "", "") + ), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") _, err := d.DialForAteletOnNode("node1") if err == nil || errors.Is(err, ErrNoAteletOnNode) { t.Fatalf("DialForAteletOnNode = %v, want a non-ErrNoAteletOnNode error", err) @@ -400,7 +400,7 @@ func TestDialForAteletOnNode(t *testing.T) { t.Run("dials and caches the node's atelet", func(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), - ), installdefaults.SystemNamespace, "", "") + ), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") var credsUID string d.dialCredentials = func(expectedPodUID string) (credentials.TransportCredentials, error) { credsUID = expectedPodUID @@ -427,7 +427,7 @@ func TestDialForAteletOnNode(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), ateletPod("atelet-2", "uid-2", "node2", "10.0.0.2"), - ), installdefaults.SystemNamespace, "", "", WithDialCredentials(func(string) (credentials.TransportCredentials, error) { + ), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil })) d.ateletConns = newAteletConnCache(1) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 7203434d0c..65cff1cdf6 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -152,7 +152,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu // Dial the fake atelet over insecure transport instead of per-atelet mTLS, // so DialForWorker's real lookup/dial/cache path is exercised under test. - dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), installdefaults.SystemNamespace, "", "", + dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", controlapi.WithDialCredentials(func(_ string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil })) diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 5a4a2dc7de..f2893f1cc4 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -219,7 +219,7 @@ func newDanglingDialer() *AteletDialer { byNamespaceAndName: func(obj any) ([]string, error) { return nil, nil }, byNode: func(obj any) ([]string, error) { return nil, nil }, }) - return NewAteletDialer(empty, empty, installdefaults.SystemNamespace, "", "") + return NewAteletDialer(empty, empty, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") } func TestEnsureAteletSuspended_DanglingWorkerDoesNotRecordPhantomSnapshot(t *testing.T) { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 786fb8f995..cfcbd75335 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -75,6 +75,7 @@ var ( actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") + ateletServiceAccount = pflag.String("atelet-service-account", installdefaults.AteletServiceAccount, "ServiceAccount atelet runs as. It is the service-account segment of the SPIFFE ID expected on atelet's certificate, so it has to match what the deployment actually creates; a Helm release that prefixes resource names needs it set.") ateletInsecure = pflag.Bool("atelet-insecure", false, "Dial atelet without transport security. Intended only for local clusters without Pod Certificates.") drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") @@ -163,7 +164,8 @@ func main() { // atelet shares ateapi's namespace in every supported deployment topology, // so we read it from Kubernetes' downward API rather than expose a flag. ateletNamespace := installdefaults.NamespaceFromPodEnv() - slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace)) + ateletSPIFFEID := installdefaults.SPIFFEID(ateletNamespace, *ateletServiceAccount) + slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace), slog.String("atelet-spiffe-id", ateletSPIFFEID)) workerPodInformerFactory, workerPodInformer := controlapi.WorkerPodInformer(clientset) ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset, ateletNamespace) @@ -199,7 +201,7 @@ func main() { if *ateletInsecure { dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) } - ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), ateletNamespace, *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), ateletSPIFFEID, *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) controlSrv := controlapi.NewRPCService(persistence, workerCache, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, *actorWorkflowDeadline, volPlugins) // Drive stored ActorTemplates through the golden actor flow. @@ -211,7 +213,7 @@ func main() { serverboot.Fatal(ctx, "while loading the Actor ID CA", err) } - actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, actorIDCAPool, persistence, workerCache, ateletNamespace) + actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, actorIDCAPool, persistence, workerCache, ateletSPIFFEID) lisCfg := &net.ListenConfig{} lis, err := lisCfg.Listen(ctx, "tcp", *listenAddr) diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index 0315d72c88..ba6a61cbec 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -75,7 +75,7 @@ const ( // Deployment managed by a WorkerPool. Only fields owned by this controller // are declared here. otel, when it carries an endpoint, is propagated to the // ateom container so it pushes telemetry to that collector. -func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettings, systemNamespace string) *appsv1ac.DeploymentApplyConfiguration { +func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettings, systemNamespace, ateletServiceAccount, routerServiceAccount string) *appsv1ac.DeploymentApplyConfiguration { labels := map[string]string{} annotations := map[string]string{} if wp.Spec.Template != nil { @@ -100,8 +100,8 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin // The peers atunnel authenticates live in substrate's namespace, // not the worker's, so the controller passes their identities // rather than letting ateom assume the default install. - "--atunnel-client-identity="+installdefaults.RouterSPIFFEID(systemNamespace), - "--atunnel-broker-identity="+installdefaults.AteletSPIFFEID(systemNamespace), + "--atunnel-client-identity="+installdefaults.SPIFFEID(systemNamespace, routerServiceAccount), + "--atunnel-broker-identity="+installdefaults.SPIFFEID(systemNamespace, ateletServiceAccount), "--atunnel-egress-listen-address=0.0.0.0:15001", "--atunnel-egress-trust-bundle="+atunnelEgressTrustMountPath+"/trust-bundle.pem", ). diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index fbf09f5bbe..6743ede58d 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -206,7 +206,7 @@ func TestBuildDeploymentApplyConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := buildDeploymentApplyConfig(tt.wp, ateomOTelSettings{}, installdefaults.SystemNamespace) + got := buildDeploymentApplyConfig(tt.wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount) if diff := cmp.Diff(tt.want, got); diff != "" { t.Fatalf("buildDeploymentApplyConfig() mismatch (-want +got):\n%s", diff) } @@ -226,7 +226,7 @@ func TestBuildDeploymentApplyConfigMetadata(t *testing.T) { }, }) - got := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace) + got := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount) wantLabels := map[string]string{ "project": "agent-substrate", "team": "compute", @@ -268,7 +268,7 @@ func TestMicroVMPodShape(t *testing.T) { t.Run(tt.name, func(t *testing.T) { wp := testWorkerPoolApplyConfig(nil) wp.Spec.SandboxClass = tt.class - ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace).Spec.Template.Spec + ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec // /dev/kvm must come from the device plugin, never a hostPath: a // hostPath mount carries no cgroup device allow rule, and the @@ -361,7 +361,7 @@ func TestMicroVMDeviceRequestsPreserveTemplateResources(t *testing.T) { }, }) wp.Spec.SandboxClass = atev1alpha1.SandboxClassMicroVM - c := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace).Spec.Template.Spec.Containers[0] + c := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec.Containers[0] if got, ok := deviceLimit(c, string(corev1.ResourceMemory)); !ok || got != "2Gi" { t.Errorf("memory limit = %q (present=%v), want 2Gi", got, ok) @@ -426,7 +426,7 @@ func TestAteomSecurityContextByClass(t *testing.T) { // TestTerminationGracePeriodSeconds asserts the pod's grace period is hardcoded to 3600s. func TestTerminationGracePeriodSeconds(t *testing.T) { wp := testWorkerPoolApplyConfig(nil) - ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace).Spec.Template.Spec + ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec if ps.TerminationGracePeriodSeconds == nil { t.Fatalf("TerminationGracePeriodSeconds not set") } @@ -450,7 +450,7 @@ func TestBuildDeploymentApplyConfigOTelEndpoint(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{Endpoint: tt.endpoint}, installdefaults.SystemNamespace). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{Endpoint: tt.endpoint}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). Spec.Template.Spec.Containers[0] env := envByName(c.Env) @@ -536,7 +536,7 @@ func TestBuildDeploymentApplyConfigMetricExportTuning(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel, installdefaults.SystemNamespace). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). Spec.Template.Spec.Containers[0] env := envByName(c.Env) for _, k := range []string{"OTEL_METRIC_EXPORT_INTERVAL", "OTEL_METRIC_EXPORT_TIMEOUT"} { @@ -593,7 +593,7 @@ func TestBuildDeploymentApplyConfigTracesSamplerPropagation(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel, installdefaults.SystemNamespace). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). Spec.Template.Spec.Containers[0] env := envByName(c.Env) for _, k := range []string{"OTEL_TRACES_SAMPLER", "OTEL_TRACES_SAMPLER_ARG"} { @@ -812,7 +812,7 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf func TestBuildDeploymentAtunnelIdentitiesRelocatedNamespace(t *testing.T) { const relocated = "substrate-test" - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{}, relocated). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{}, relocated, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). Spec.Template.Spec.Containers[0] want := map[string]string{ @@ -834,3 +834,36 @@ func TestBuildDeploymentAtunnelIdentitiesRelocatedNamespace(t *testing.T) { } } } + +// TestBuildDeploymentAtunnelIdentitiesPrefixedServiceAccounts covers a release +// that renames the ServiceAccounts — what the Helm chart does for any release +// not called "substrate", which is every install that consumes substrate as a +// subchart. The SPIFFE ID embeds the ServiceAccount name, so identities built +// from the compiled-in defaults name accounts that do not exist and atunnel +// rejects the peer. +func TestBuildDeploymentAtunnelIdentitiesPrefixedServiceAccounts(t *testing.T) { + const ( + namespace = "kagent-system" + atelet = "kagent-atelet" + router = "kagent-atenet-router" + ) + + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{}, namespace, atelet, router). + Spec.Template.Spec.Containers[0] + + want := map[string]string{ + "--atunnel-client-identity=": "spiffe://cluster.local/ns/kagent-system/sa/kagent-atenet-router", + "--atunnel-broker-identity=": "spiffe://cluster.local/ns/kagent-system/sa/kagent-atelet", + } + for flag, wantVal := range want { + var got string + for _, arg := range c.Args { + if strings.HasPrefix(arg, flag) { + got = strings.TrimPrefix(arg, flag) + } + } + if got != wantVal { + t.Errorf("%s%s, want %s%s", flag, got, flag, wantVal) + } + } +} diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller.go b/cmd/atecontroller/internal/controllers/workerpool_controller.go index f8cf97a673..7f1aeda961 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller.go @@ -52,10 +52,15 @@ type WorkerPoolReconciler struct { // OTelTracesSamplerArg is the OTEL_TRACES_SAMPLER_ARG propagated to ateom // pods. Ignored unless OTelTracesSampler is set. OTelTracesSamplerArg string - // SystemNamespace is the namespace substrate's control plane runs in. It - // names the atenet-router and atelet SPIFFE identities that atunnel - // authenticates inside each worker. - SystemNamespace string + // SystemNamespace is the namespace substrate's control plane runs in, and + // AteletServiceAccount / RouterServiceAccount are the ServiceAccounts those + // components run as. Together they name the SPIFFE identities that atunnel + // authenticates inside each worker, which is why the ServiceAccount names + // are configuration and not constants: a Helm release that prefixes + // resource names changes them. + SystemNamespace string + AteletServiceAccount string + RouterServiceAccount string desiredWorkers metric.Int64ObservableUpDownCounter readyWorkers metric.Int64ObservableUpDownCounter @@ -120,7 +125,7 @@ func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alp MetricExportTimeout: r.OTelMetricExportTimeout, TracesSampler: r.OTelTracesSampler, TracesSamplerArg: r.OTelTracesSamplerArg, - }, r.SystemNamespace) + }, r.SystemNamespace, r.AteletServiceAccount, r.RouterServiceAccount) if err := r.Apply(ctx, depAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply Deployment: %w", err) } diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index c2a486abee..f50152aece 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -71,6 +71,9 @@ var ( otelTracesSamplerArg = pflag.String("otel-traces-sampler-arg", os.Getenv("OTEL_TRACES_SAMPLER_ARG"), "Trace sampler argument set on ateom worker pods, ignored unless --otel-traces-sampler is set. Defaults to the controller's own OTEL_TRACES_SAMPLER_ARG.") + ateletServiceAccount = pflag.String("atelet-service-account", installdefaults.AteletServiceAccount, "ServiceAccount atelet runs as. It is the service-account segment of the SPIFFE ID each worker's atunnel expects on the credential broker, so it has to match what the deployment actually creates.") + routerServiceAccount = pflag.String("router-service-account", installdefaults.RouterServiceAccount, "ServiceAccount atenet-router runs as. It is the service-account segment of the SPIFFE ID each worker's atunnel accepts on actor ingress, so it has to match what the deployment actually creates.") + ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.") ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") ateapiClientCert = pflag.String("ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required.") @@ -183,6 +186,8 @@ func main() { OTelTracesSampler: *otelTracesSampler, OTelTracesSamplerArg: *otelTracesSamplerArg, SystemNamespace: systemNamespace, + AteletServiceAccount: *ateletServiceAccount, + RouterServiceAccount: *routerServiceAccount, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WorkerPool") os.Exit(1) From 8a890a90a10d67bf11aa8f264ba64c1b26d88ee1 Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Thu, 3 Sep 2026 10:14:12 -0400 Subject: [PATCH 7/8] Let the client and install script address renamed substrate resources Installing substrate as a subchart renames more than the ServiceAccounts. The Service fronting ateapi becomes -api and the ServiceAccount the CLI mints its token from becomes -ate-client, and both were constants: kubectl-ate port-forwarded to a Service that does not exist, and once that was fixed it minted a token for the wrong audience and ateapi rejected it as an invalid bearer token. The client reads ATE_API_SERVICE_NAME and ATE_CLIENT_SERVICE_ACCOUNT alongside the ATE_NAMESPACE it already read; it runs outside the cluster and cannot discover any of this. hack/install-ate.sh reads the same ATE_API_SERVICE_NAME, since the audience it writes into the API authentication config has to name the same Service the client dials. All three default to the canonical names. --- hack/install-ate.sh | 7 ++++- internal/ateclient/builder.go | 31 +++++++++++++++++++-- internal/installdefaults/installdefaults.go | 3 ++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 59434c46f7..5a997053de 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -40,6 +40,11 @@ fi # a relocated release (the chart's --namespace must match). ATE_NAMESPACE="${ATE_NAMESPACE:-ate-system}" +# Service name fronting ateapi. It is the audience the API authentication config +# accepts, so it has to match the Service the deployment actually creates; a +# Helm release that prefixes resource names needs it set. +ATE_API_SERVICE_NAME="${ATE_API_SERVICE_NAME:-api}" + # ATE_DEMOS is an array that registers the prefix name of the demo functions. ATE_DEMOS=() @@ -621,7 +626,7 @@ create_api_authentication_config() { ;; esac local authentication_config - authentication_config=$(printf 'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n issuer: %s\n audiences: [api.%s.svc]\n%s' "${jwt_issuer}" "${ATE_NAMESPACE}" "${discovery_config}") + authentication_config=$(printf 'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n issuer: %s\n audiences: [%s.%s.svc]\n%s' "${jwt_issuer}" "${ATE_API_SERVICE_NAME}" "${ATE_NAMESPACE}" "${discovery_config}") run_kubectl create configmap -n "${ATE_NAMESPACE}" ate-api-authentication \ --from-literal=authentication.yaml="${authentication_config}" \ --dry-run=client -o yaml \ diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 7ef37182c9..c3991e96eb 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -50,6 +50,31 @@ import ( // hack/install-ate.sh installed with. const NamespaceEnv = "ATE_NAMESPACE" +// APIServiceEnv and ClientServiceAccountEnv override the ateapi Service and the +// ServiceAccount the client mints its token from. The Helm chart prefixes both +// names for a release not called "substrate", and the client has no way to +// discover that from outside the cluster. +const ( + APIServiceEnv = "ATE_API_SERVICE_NAME" + ClientServiceAccountEnv = "ATE_CLIENT_SERVICE_ACCOUNT" +) + +// apiServiceName is the Service that fronts ateapi. +func apiServiceName() string { + if n := os.Getenv(APIServiceEnv); n != "" { + return n + } + return installdefaults.APIServiceName +} + +// clientServiceAccount is the ServiceAccount the bearer token is minted from. +func clientServiceAccount() string { + if n := os.Getenv(ClientServiceAccountEnv); n != "" { + return n + } + return installdefaults.ClientServiceAccount +} + // systemNamespace is the namespace the client expects ateapi to be running in. func systemNamespace() string { if ns := os.Getenv(NamespaceEnv); ns != "" { @@ -62,7 +87,7 @@ func systemNamespace() string { // the SNI presented on the connection and the audience of the minted token, so // it has to track the namespace ateapi actually runs in. func apiServerName() string { - return fmt.Sprintf("%s.%s.svc", installdefaults.APIServiceName, systemNamespace()) + return fmt.Sprintf("%s.%s.svc", apiServiceName(), systemNamespace()) } const ( @@ -187,7 +212,7 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext, tokenFile // TODO: Should we special-case a LoadBalancer "api" Service and dial its // address directly instead of port-forwarding? - localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, systemNamespace(), installdefaults.APIServiceName, 443) + localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, systemNamespace(), apiServiceName(), 443) if err != nil { return nil, err } @@ -276,7 +301,7 @@ func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset, ExpirationSeconds: &expirationSeconds, }, } - token, err := clientset.CoreV1().ServiceAccounts(systemNamespace()).CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) + token, err := clientset.CoreV1().ServiceAccounts(systemNamespace()).CreateToken(ctx, clientServiceAccount(), tokenRequest, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to request ateapi bearer token: %w", err) } diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go index 0e1995e76f..7bc83a0796 100644 --- a/internal/installdefaults/installdefaults.go +++ b/internal/installdefaults/installdefaults.go @@ -34,6 +34,9 @@ const ( RouterServiceName = "atenet-router" // DNSServiceName is the Service name of substrate's CoreDNS. DNSServiceName = "dns" + // ClientServiceAccount is the ServiceAccount an out-of-cluster client mints + // its ateapi bearer token from. + ClientServiceAccount = "ate-client" // AteletTrustDomain, AteletServiceAccount and RouterServiceAccount are the // trust-domain and service-account segments of the SPIFFE IDs that atelet From 4c0b1685fc41887a4f33f795876d64e072a59735 Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Thu, 3 Sep 2026 11:14:44 -0400 Subject: [PATCH 8/8] e2e: address substrate resources by their rendered names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness looked up ate-controller, ate-api-server, atenet-router and the egress CA pool Secret by their canonical names. A release the chart prefixes — which is any subchart install, since a subchart inherits the parent's release name — renders them as -ate-controller and so on, and every suite failed in preflight before running a single case. E2E_RESOURCE_PREFIX carries what the install under test prepends, empty by default. --- internal/e2e/env.go | 11 +++++++++++ internal/e2e/preflight.go | 4 ++-- internal/e2e/router_client.go | 5 ++--- internal/e2e/statusz.go | 2 +- internal/e2e/suites/demo/demo_test.go | 2 +- internal/e2e/suites/metrics/metrics_test.go | 2 +- internal/e2e/suites/networking/grpcingress_test.go | 2 +- internal/e2e/trustbundle.go | 11 +++++------ 8 files changed, 24 insertions(+), 15 deletions(-) diff --git a/internal/e2e/env.go b/internal/e2e/env.go index d59d32f57c..f0136102bd 100644 --- a/internal/e2e/env.go +++ b/internal/e2e/env.go @@ -48,3 +48,14 @@ func SystemNamespace() string { } return installdefaults.SystemNamespace } + +// ResourcePrefixEnv is the prefix the install under test puts on substrate's +// resource names. The Helm chart prefixes them with the release name for any +// release not called "substrate", which is what a subchart install produces, +// and the harness addresses several of those resources by name. +const ResourcePrefixEnv = "E2E_RESOURCE_PREFIX" + +// ResourceName returns name as the install under test renders it. +func ResourceName(name string) string { + return os.Getenv(ResourcePrefixEnv) + name +} diff --git a/internal/e2e/preflight.go b/internal/e2e/preflight.go index 771d84539c..938bef93d8 100644 --- a/internal/e2e/preflight.go +++ b/internal/e2e/preflight.go @@ -38,8 +38,8 @@ func PreflightChecks() error { // Check deployments. deployments := []string{ - "ate-controller", - "ate-api-server", + ResourceName("ate-controller"), + ResourceName("ate-api-server"), } namespace := SystemNamespace() for _, depName := range deployments { diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index f673f319b8..d139514cbf 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -36,7 +36,6 @@ import ( ) const ( - routerService = "atenet-router" // routerConnectServicePort is atenet-router's Service port for // CONNECT-tunneled traffic (see manifests/ate-install/atenet-router.yaml). // It is a distinct listener from the plain HTTP one Get/PostJSON use: @@ -78,7 +77,7 @@ func NewRouterClient(ctx context.Context) (*RouterClient, error) { return nil, fmt.Errorf("creating k8s client: %w", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, SystemNamespace(), routerService, 80) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, SystemNamespace(), ResourceName("atenet-router"), 80) if err != nil { return nil, err } @@ -187,7 +186,7 @@ func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, // in one test don't each pay for a fresh port-forward. func (c *RouterClient) ensureConnectPortForward(ctx context.Context) error { c.connectOnce.Do(func() { - localPort, stop, err := portforward.ServicePortForward(ctx, c.config, c.clientset, SystemNamespace(), routerService, routerConnectServicePort) + localPort, stop, err := portforward.ServicePortForward(ctx, c.config, c.clientset, SystemNamespace(), ResourceName("atenet-router"), routerConnectServicePort) if err != nil { c.connectErr = fmt.Errorf("port-forwarding to the router's CONNECT listener: %w", err) return diff --git a/internal/e2e/statusz.go b/internal/e2e/statusz.go index a823307f79..4e3516aed4 100644 --- a/internal/e2e/statusz.go +++ b/internal/e2e/statusz.go @@ -51,7 +51,7 @@ func NewStatuszClient(ctx context.Context) (*StatuszClient, error) { return nil, fmt.Errorf("creating k8s client: %w", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, SystemNamespace(), routerService, routerStatusPort) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, SystemNamespace(), ResourceName("atenet-router"), routerStatusPort) if err != nil { return nil, err } diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 9fc7ccb3de..dcbb0c1b94 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -1264,7 +1264,7 @@ func callActorPathOnce(t *testing.T, actorRef resources.ActorRef, method, path s t.Helper() clients := e2e.GetClients() - svc, err := clients.K8s.CoreV1().Services(e2e.SystemNamespace()).Get(context.Background(), "atenet-router", metav1.GetOptions{}) + svc, err := clients.K8s.CoreV1().Services(e2e.SystemNamespace()).Get(context.Background(), e2e.ResourceName("atenet-router"), metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("failed to get atenet-router service: %w", err) } diff --git a/internal/e2e/suites/metrics/metrics_test.go b/internal/e2e/suites/metrics/metrics_test.go index a8a59fa97a..d03923df26 100644 --- a/internal/e2e/suites/metrics/metrics_test.go +++ b/internal/e2e/suites/metrics/metrics_test.go @@ -43,7 +43,7 @@ func TestPlatformMetricsEmitted(t *testing.T) { tmpl := e2e.SubstrateCounterFixture() actorID := fmt.Sprintf("metrics-probe-%d", time.Now().UnixNano()) metricPrefixes := append([]string(nil), e2e.PlatformMetricPrefixes...) - router, err := clients.K8s.AppsV1().Deployments(e2e.SystemNamespace()).Get(ctx, "atenet-router", metav1.GetOptions{}) + router, err := clients.K8s.AppsV1().Deployments(e2e.SystemNamespace()).Get(ctx, e2e.ResourceName("atenet-router"), metav1.GetOptions{}) if err != nil { t.Fatalf("Get atenet-router deployment: %v", err) } diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index cf555cec7c..024abb963f 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -311,7 +311,7 @@ func routerAddress(t *testing.T, ctx context.Context) string { if err != nil { t.Fatalf("creating k8s client: %v", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, e2e.SystemNamespace(), "atenet-router", 80) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, e2e.SystemNamespace(), e2e.ResourceName("atenet-router"), 80) if err != nil { t.Fatalf("port-forwarding to the router: %v", err) } diff --git a/internal/e2e/trustbundle.go b/internal/e2e/trustbundle.go index 30bc56795d..045e5d2643 100644 --- a/internal/e2e/trustbundle.go +++ b/internal/e2e/trustbundle.go @@ -39,8 +39,7 @@ const ( // EgressTrustBundleObjectName is the reconciler-owned ClusterTrustBundle. EgressTrustBundleObjectName = "egress-mitm.ate.dev:mitm:primary-bundle" - egressCAPoolSecretName = "egress-mitm-ca-pool" - egressCAPoolSecretKey = "pool" + egressCAPoolSecretKey = "pool" ) // EnsureEgressTrustBundle makes sure the egress trust bundle exists, then @@ -67,7 +66,7 @@ func ReplaceEgressTrustPool(t *testing.T, ctx context.Context, clients *Clients, if !createEgressTrustPool(t, ctx, clients, secret) { // Took over an existing pool: overwrite its contents without adopting // its cleanup, since whoever created it registered one already. - existing, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Get(ctx, egressCAPoolSecretName, metav1.GetOptions{}) + existing, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Get(ctx, ResourceName("egress-mitm-ca-pool"), metav1.GetOptions{}) if err != nil { t.Fatalf("reading existing CA pool secret: %v", err) } @@ -102,7 +101,7 @@ func newEgressTrustPool(t *testing.T) (*corev1.Secret, string) { t.Fatalf("encoding the egress CA private key: %v", err) } return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: SystemNamespace(), Name: egressCAPoolSecretName}, + ObjectMeta: metav1.ObjectMeta{Namespace: SystemNamespace(), Name: ResourceName("egress-mitm-ca-pool")}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{ egressCAPoolSecretKey: poolBytes, @@ -121,12 +120,12 @@ func createEgressTrustPool(t *testing.T, ctx context.Context, clients *Clients, t.Helper() if _, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Create(ctx, secret, metav1.CreateOptions{}); err != nil { if !apierrors.IsAlreadyExists(err) { - t.Fatalf("creating CA pool secret %s/%s: %v", SystemNamespace(), egressCAPoolSecretName, err) + t.Fatalf("creating CA pool secret %s/%s: %v", SystemNamespace(), ResourceName("egress-mitm-ca-pool"), err) } return false } t.Cleanup(func() { - _ = clients.K8s.CoreV1().Secrets(SystemNamespace()).Delete(context.Background(), egressCAPoolSecretName, metav1.DeleteOptions{}) + _ = clients.K8s.CoreV1().Secrets(SystemNamespace()).Delete(context.Background(), ResourceName("egress-mitm-ca-pool"), metav1.DeleteOptions{}) }) return true }