diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index c236999e48..4fa447a91c 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -89,6 +89,9 @@ spec: - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" + # ateapi verifies atelet's SPIFFE ID, which embeds the ServiceAccount + # name this chart prefixes for a renamed release. + - "--atelet-service-account={{ include "substrate.fullname" (list "atelet" .) }}" - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" - "--drain-delay=13s" diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index 31c83b9066..6a6ac462c0 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -78,9 +78,23 @@ spec: # dials the right backend when substrate is installed as a subchart. - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + # A SPIFFE ID names a ServiceAccount, and this chart prefixes those for + # any release not called "substrate". The controller stamps these + # identities onto every worker, so it has to be told the names actually + # rendered rather than assume the canonical ones. + - "--atelet-service-account={{ include "substrate.fullname" (list "atelet" .) }}" + - "--router-service-account={{ include "substrate.fullname" (list "atenet-router" .) }}" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -{{- if .Values.otel.endpoint }} env: + # ate-controller resolves substrate's namespace from the downward API. + # It names the atelet and atenet-router SPIFFE identities handed to + # each worker's atunnel, which live in substrate's namespace and not + # the worker's, so the controller cannot infer them any other way. + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- if .Values.otel.endpoint }} - name: OTEL_EXPORTER_OTLP_ENDPOINT value: {{ .Values.otel.endpoint | quote }} {{- end }} diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 916915dc42..1570d0f8ff 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -104,6 +104,12 @@ spec: - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem + # The atelet binary defaults these to api.ate-system.svc, which is + # correct only for the canonical render. The credential broker dials + # ateapi to mint actor certificates, so pass the chart-resolved + # Service and the matching name on its serving cert. + - --ateapi-address={{ include "substrate.ateApi.endpoint" . }} + - --ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc {{- with .Values.atelet.extraArgs }} {{ toYaml . | indent 8 }} {{- end }} diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 78976a39a6..347d53597e 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -210,6 +210,9 @@ spec: - "--port-extproc=50051" - "--extproc-address=127.0.0.1" - "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + # /statusz looks up this instance's own ClusterIP by Service name, and + # the chart prefixes that name for any release not called "substrate". + - "--router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }}" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" - "--status-port=4040" diff --git a/charts/substrate/templates/role.yaml b/charts/substrate/templates/role.yaml index 5a240f5baf..9226661d30 100644 --- a/charts/substrate/templates/role.yaml +++ b/charts/substrate/templates/role.yaml @@ -102,7 +102,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ include "substrate.fullname" (list "ate-controller" .) }} - namespace: ate-system + namespace: {{ .Release.Namespace }} rules: - apiGroups: - discovery.k8s.io diff --git a/cmd/ateapi/internal/actoridentity/actoridentity.go b/cmd/ateapi/internal/actoridentity/actoridentity.go index 7304c43d15..4a1abbc701 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity.go @@ -59,33 +59,26 @@ type Server struct { // is entitled to the actor it is asking for a credential for. store store.Interface workers *workercache.Cache + + // ateletSPIFFEID is the identity the calling atelet must present to mint + // actor credentials. + ateletSPIFFEID string } var _ ateapipb.ActorIdentityServer = (*Server)(nil) -func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache) *Server { +func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache, ateletSPIFFEID string) *Server { return &Server{ actorIdentityJWTIssuer: actorIdentityJWTIssuer, actorIDJWTPoolFile: actorIDJWTPoolFile, actorIDCAPool: actorIDCAPool, store: store, workers: workers, + ateletSPIFFEID: ateletSPIFFEID, } } -// The SPIFFE identity that atelet client certs carry, as minted by the -// podidentity signer (cmd/podcertcontroller/internal/podidentitysigner). -// -// These mirror the constants the atelet dialer verifies against in -// cmd/ateapi/internal/controlapi/dialer.go. They are duplicated rather than -// imported so that this package does not depend on controlapi for three -// strings; if a third pkg that need these constants appears, they should move to a shared package. -const ( - ateletTrustDomain = "cluster.local" - ateletNamespace = "ate-system" - ateletSA = "atelet" - actorCertificateLifetime = time.Hour -) +const actorCertificateLifetime = time.Hour func (s *Server) MintJWT(ctx context.Context, req *ateapipb.MintJWTRequest) (*ateapipb.MintJWTResponse, error) { caller, ok := principal.FromContext(ctx) @@ -148,7 +141,7 @@ func (s *Server) MintJWT(ctx context.Context, req *ateapipb.MintJWTRequest) (*at } func (s *Server) MintCert(ctx context.Context, req *ateapipb.MintCertRequest) (*ateapipb.MintCertResponse, error) { - caller, err := authenticateAtelet(ctx) + caller, err := s.authenticateAtelet(ctx) if err != nil { return nil, err } @@ -243,7 +236,7 @@ type ateletCaller struct { // pod-identity CA (see buildServerCreds in cmd/ateapi/main.go), so the // extensions read here are trustworthy: only the pod-identity signer can mint // a certificate carrying a given pod's node name. -func authenticateAtelet(ctx context.Context) (*ateletCaller, error) { +func (s *Server) authenticateAtelet(ctx context.Context) (*ateletCaller, error) { p, ok := peer.FromContext(ctx) if !ok { return nil, status.Errorf(codes.Unauthenticated, "no peer transport information found") @@ -262,11 +255,7 @@ func authenticateAtelet(ctx context.Context) (*ateletCaller, error) { // Only atelet may mint actor credentials. Everything else with a valid // pod-identity certificate — including the actor workloads themselves — is // rejected here. - expected := (&url.URL{ - Scheme: "spiffe", - Host: ateletTrustDomain, - Path: path.Join("ns", ateletNamespace, "sa", ateletSA), - }).String() + expected := s.ateletSPIFFEID if len(leaf.URIs) == 0 || leaf.URIs[0].String() != expected { slog.WarnContext(ctx, "ActorIdentity denied: caller is not atelet", slog.Any("uris", leaf.URIs), slog.String("expected", expected)) diff --git a/cmd/ateapi/internal/actoridentity/actoridentity_test.go b/cmd/ateapi/internal/actoridentity/actoridentity_test.go index d1282de67f..cc8b935ef3 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity_test.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity_test.go @@ -32,6 +32,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/principal" "github.com/agent-substrate/substrate/internal/resources" @@ -74,6 +75,19 @@ const ( // populates. Self-signing is sufficient because the code under test reads an // already transport-verified peer certificate and never re-validates the chain // itself. +// The atelet SPIFFE segments the tests build peer certificates from. They +// mirror what a default install mints, which is what the Server under test is +// configured with. +const ( + ateletTrustDomain = installdefaults.AteletTrustDomain + ateletNamespace = installdefaults.SystemNamespace + ateletSA = installdefaults.AteletServiceAccount +) + +// ateletSPIFFEID is the identity a default install's atelet presents, and what +// the Server under test is configured to accept. +var ateletSPIFFEID = installdefaults.SPIFFEID(ateletNamespace, ateletSA) + func newTestCert(t *testing.T, spiffePath string, podIdentity *substratex509.PodIdentity) *x509.Certificate { t.Helper() @@ -165,7 +179,7 @@ func newTestServer(t *testing.T, st store.Interface) *Server { t.Fatalf("start worker cache: %v", err) } } - return New("issuer", "", pool, st, workers) + return New("issuer", "", pool, st, workers, ateletSPIFFEID) } // staleWatchStore wraps a store with a WatchWorkers that never delivers, @@ -324,11 +338,11 @@ func newTestServerWithCache(t *testing.T, st store.Interface, workers *workercac t.Fatalf("generate CA: %v", err) } pool := &localca.ConcretePool{CAs: []*localca.CA{ca}} - return New("issuer", "", pool, st, workers) + return New("issuer", "", pool, st, workers, ateletSPIFFEID) } func TestMintJWTRequiresConfiguredJWTProvider(t *testing.T) { - srv := &Server{actorIdentityJWTIssuer: "https://kubernetes.example"} + srv := &Server{actorIdentityJWTIssuer: "https://kubernetes.example", ateletSPIFFEID: ateletSPIFFEID} for _, tt := range []struct { name string ctx context.Context @@ -918,7 +932,7 @@ func TestMintCertAuthorizesBeforeSigning(t *testing.T) { ActiveForSigning: "test-actor-ca", } - srv := New("issuer", "", pool, st, workers) + srv := New("issuer", "", pool, st, workers, ateletSPIFFEID) actor, err := st.GetActor(ctx, resources.ActorRef{Atespace: testAtespace, Name: testActorName}) if err != nil { @@ -1082,3 +1096,37 @@ func TestValidateMintCertRequest(t *testing.T) { }) } } + +// TestAuthenticateAteletHonorsConfiguredNamespace checks that the namespace +// the Server is configured with is the one it accepts, and that the canonical +// namespace is rejected when the install lives elsewhere. +// +// The table in TestMintCertAuthorization covers a caller from the wrong +// namespace, but its Server is always configured with the default, so it holds +// against a hardcoded "ate-system" too. Only the relocated case distinguishes +// "reads its configuration" from "happens to agree with the constant". +func TestAuthenticateAteletHonorsConfiguredNamespace(t *testing.T) { + const relocated = "substrate-test" + + srv := &Server{ateletSPIFFEID: installdefaults.SPIFFEID(relocated, ateletSA)} + + t.Run("accepts atelet from the configured namespace", func(t *testing.T) { + id := podIdentityOn(testNode) + id.Namespace = relocated + cert := newTestCert(t, path.Join("ns", relocated, "sa", ateletSA), id) + + if _, err := srv.authenticateAtelet(ctxWithCert(cert)); err != nil { + t.Errorf("authenticateAtelet() = %v, want success for an atelet in %q", err, relocated) + } + }) + + t.Run("rejects atelet from the canonical namespace", func(t *testing.T) { + id := podIdentityOn(testNode) + id.Namespace = installdefaults.SystemNamespace + cert := newTestCert(t, path.Join("ns", installdefaults.SystemNamespace, "sa", ateletSA), id) + + if _, err := srv.authenticateAtelet(ctxWithCert(cert)); status.Code(err) != codes.PermissionDenied { + t.Errorf("authenticateAtelet() code = %v, want PermissionDenied for an atelet in %q", status.Code(err), installdefaults.SystemNamespace) + } + }) +} diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 557d994aad..c6f6a638da 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -49,12 +49,7 @@ var ErrNoAteletOnNode = errors.New("no atelet pod found on node") // The SPIFFE identity that atelet serving certs carry, as minted by the // podidentity signer (cmd/podcertcontroller/internal/podidentitysigner). -// The namespace part is ateletNamespace, declared in informer.go. -const ( - trustDomainName = "cluster.local" - ateletSA = "atelet" -) - +// The namespace part is the dialer's ateletNamespace. // AteletDialer handles gRPC connections to Atelet pods. type AteletDialer struct { workerIndexer cache.Indexer @@ -83,13 +78,13 @@ func WithInsecureCredentials() DialerOption { // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet connection. -func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { +func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, ateletSPIFFEID, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { d := &AteletDialer{ workerIndexer: workerIndexer, ateletIndexer: ateletIndexer, ateletConns: newAteletConnCache(1024), dialCredentials: func(expectedPodUID string) (credentials.TransportCredentials, error) { - tlsConfig, err := buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID) + tlsConfig, err := buildTLSConfig(ateletSPIFFEID, clientBundlePath, serverCAPath, expectedPodUID) if err != nil { return nil, err } @@ -196,18 +191,18 @@ func (d *AteletDialer) DialForAteletOnNode(nodeName string) (*grpc.ClientConn, e return ateletConn, nil } -func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { - trustDomain, err := spiffeid.TrustDomainFromString(trustDomainName) +func buildTLSConfig(ateletSPIFFEID, clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { + trustDomain, err := spiffeid.TrustDomainFromString(installdefaults.AteletTrustDomain) if err != nil { - return nil, fmt.Errorf("while parsing trust domain %q: %w", trustDomainName, err) + return nil, fmt.Errorf("while parsing trust domain %q: %w", installdefaults.AteletTrustDomain, err) } bundle, err := x509bundle.Load(trustDomain, serverCAPath) if err != nil { return nil, fmt.Errorf("while loading CA bundle from %s: %w", serverCAPath, err) } - expectedID, err := spiffeid.FromSegments(trustDomain, "ns", installdefaults.NamespaceFromPodEnv(), "sa", ateletSA) + expectedID, err := spiffeid.FromString(ateletSPIFFEID) if err != nil { - return nil, fmt.Errorf("while building expected atelet SPIFFE ID: %w", err) + return nil, fmt.Errorf("while parsing expected atelet SPIFFE ID %q: %w", ateletSPIFFEID, err) } verify, err := verifyAteletServerCert(bundle, expectedID, expectedPodUID) diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 2fb103bc33..b26a71ef35 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -43,12 +43,12 @@ import ( const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" func TestAteletDialerInsecureRequiresOptIn(t *testing.T) { - secure := NewAteletDialer(nil, nil, "", "") + secure := NewAteletDialer(nil, nil, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") if _, err := secure.dialCredentials("pod-uid"); err == nil { t.Fatal("secure dialer accepted empty credential paths") } - insecureDialer := NewAteletDialer(nil, nil, "", "", WithInsecureCredentials()) + insecureDialer := NewAteletDialer(nil, nil, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", WithInsecureCredentials()) creds, err := insecureDialer.dialCredentials("pod-uid") if err != nil { t.Fatalf("insecure dial credentials: %v", err) @@ -380,7 +380,7 @@ func TestDialForAteletOnNode(t *testing.T) { } t.Run("no atelet on node", func(t *testing.T) { - d := NewAteletDialer(nil, newTestAteletIndexer(t), "", "") + d := NewAteletDialer(nil, newTestAteletIndexer(t), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") if _, err := d.DialForAteletOnNode("node1"); !errors.Is(err, ErrNoAteletOnNode) { t.Fatalf("DialForAteletOnNode = %v, want ErrNoAteletOnNode", err) } @@ -390,7 +390,7 @@ func TestDialForAteletOnNode(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), ateletPod("atelet-2", "uid-2", "node1", "10.0.0.2"), - ), "", "") + ), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") _, err := d.DialForAteletOnNode("node1") if err == nil || errors.Is(err, ErrNoAteletOnNode) { t.Fatalf("DialForAteletOnNode = %v, want a non-ErrNoAteletOnNode error", err) @@ -400,7 +400,7 @@ func TestDialForAteletOnNode(t *testing.T) { t.Run("dials and caches the node's atelet", func(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), - ), "", "") + ), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") var credsUID string d.dialCredentials = func(expectedPodUID string) (credentials.TransportCredentials, error) { credsUID = expectedPodUID @@ -427,7 +427,7 @@ func TestDialForAteletOnNode(t *testing.T) { d := NewAteletDialer(nil, newTestAteletIndexer(t, ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"), ateletPod("atelet-2", "uid-2", "node2", "10.0.0.2"), - ), "", "", WithDialCredentials(func(string) (credentials.TransportCredentials, error) { + ), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil })) d.ateletConns = newAteletConnCache(1) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 2c5afa464e..65cff1cdf6 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -152,7 +152,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu // Dial the fake atelet over insecure transport instead of per-atelet mTLS, // so DialForWorker's real lookup/dial/cache path is exercised under test. - dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), "", "", + dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", controlapi.WithDialCredentials(func(_ string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil })) diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index e7dfdf659a..f2893f1cc4 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -21,6 +21,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -218,7 +219,7 @@ func newDanglingDialer() *AteletDialer { byNamespaceAndName: func(obj any) ([]string, error) { return nil, nil }, byNode: func(obj any) ([]string, error) { return nil, nil }, }) - return NewAteletDialer(empty, empty, "", "") + return NewAteletDialer(empty, empty, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "") } func TestEnsureAteletSuspended_DanglingWorkerDoesNotRecordPhantomSnapshot(t *testing.T) { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 9c9de8567a..cfcbd75335 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -75,6 +75,7 @@ var ( actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") + ateletServiceAccount = pflag.String("atelet-service-account", installdefaults.AteletServiceAccount, "ServiceAccount atelet runs as. It is the service-account segment of the SPIFFE ID expected on atelet's certificate, so it has to match what the deployment actually creates; a Helm release that prefixes resource names needs it set.") ateletInsecure = pflag.Bool("atelet-insecure", false, "Dial atelet without transport security. Intended only for local clusters without Pod Certificates.") drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") @@ -163,7 +164,8 @@ func main() { // atelet shares ateapi's namespace in every supported deployment topology, // so we read it from Kubernetes' downward API rather than expose a flag. ateletNamespace := installdefaults.NamespaceFromPodEnv() - slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace)) + ateletSPIFFEID := installdefaults.SPIFFEID(ateletNamespace, *ateletServiceAccount) + slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace), slog.String("atelet-spiffe-id", ateletSPIFFEID)) workerPodInformerFactory, workerPodInformer := controlapi.WorkerPodInformer(clientset) ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset, ateletNamespace) @@ -199,7 +201,7 @@ func main() { if *ateletInsecure { dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) } - ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), ateletSPIFFEID, *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) controlSrv := controlapi.NewRPCService(persistence, workerCache, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, *actorWorkflowDeadline, volPlugins) // Drive stored ActorTemplates through the golden actor flow. @@ -211,7 +213,7 @@ func main() { serverboot.Fatal(ctx, "while loading the Actor ID CA", err) } - actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, actorIDCAPool, persistence, workerCache) + actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, actorIDCAPool, persistence, workerCache, ateletSPIFFEID) lisCfg := &net.ListenConfig{} lis, err := lisCfg.Listen(ctx, "tcp", *listenAddr) diff --git a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go index f2e8f87c04..c51f5a7ddd 100644 --- a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go +++ b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go @@ -40,13 +40,16 @@ import ( // CA pool. type EgressMITMTrustReconciler struct { client.Client + + // SystemNamespace is the namespace holding the egress MITM CA pool Secret. + SystemNamespace string } // EgressMITMCAPoolRef names the Secret holding the CA pool the egress gateway's // sdsmint sidecar signs per-SNI leaves with. -func EgressMITMCAPoolRef() types.NamespacedName { +func EgressMITMCAPoolRef(systemNamespace string) types.NamespacedName { const egressMITMCAPoolSecret = "egress-mitm-ca-pool" - return types.NamespacedName{Namespace: ateSystemNamespace, Name: egressMITMCAPoolSecret} + return types.NamespacedName{Namespace: systemNamespace, Name: egressMITMCAPoolSecret} } //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch @@ -162,7 +165,7 @@ func (r *EgressMITMTrustReconciler) deleteTrustBundle(ctx context.Context) error } func (r *EgressMITMTrustReconciler) SetupWithManager(mgr ctrl.Manager) error { - poolRef := EgressMITMCAPoolRef() + poolRef := EgressMITMCAPoolRef(r.SystemNamespace) // The pool Secret is the only object reconciled from. The bundle is watched // as well so that deleting or hand-editing the derived object is reverted diff --git a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go index 2d84330900..69dd952f9d 100644 --- a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go +++ b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller_test.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" ) @@ -66,7 +67,7 @@ func secretForPool(t *testing.T, pool *localca.ConcretePool) *corev1.Secret { if err != nil { t.Fatalf("marshal CA pool: %v", err) } - ref := EgressMITMCAPoolRef() + ref := EgressMITMCAPoolRef(installdefaults.SystemNamespace) return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Namespace: ref.Namespace, Name: ref.Name}, Data: map[string][]byte{"pool": wire}, @@ -86,8 +87,8 @@ func rootPEM(t *testing.T, pool *localca.ConcretePool) string { func reconcilePool(t *testing.T, c client.Client) error { t.Helper() - r := &EgressMITMTrustReconciler{Client: c} - _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: EgressMITMCAPoolRef()}) + r := &EgressMITMTrustReconciler{Client: c, SystemNamespace: installdefaults.SystemNamespace} + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: EgressMITMCAPoolRef(installdefaults.SystemNamespace)}) return err } @@ -170,7 +171,7 @@ func TestEgressMITMTrustFollowsPoolRotation(t *testing.T) { rotated, rotatedPool := caPoolSecret(t, "mitm", "mitm-next") current := &corev1.Secret{} - if err := c.Get(context.Background(), EgressMITMCAPoolRef(), current); err != nil { + if err := c.Get(context.Background(), EgressMITMCAPoolRef(installdefaults.SystemNamespace), current); err != nil { t.Fatalf("get pool secret: %v", err) } current.Data = rotated.Data @@ -279,7 +280,7 @@ func TestEgressMITMTrustKeepsLastGoodBundleOnBadPool(t *testing.T) { } current := &corev1.Secret{} - if err := c.Get(context.Background(), EgressMITMCAPoolRef(), current); err != nil { + if err := c.Get(context.Background(), EgressMITMCAPoolRef(installdefaults.SystemNamespace), current); err != nil { t.Fatalf("get pool secret: %v", err) } current.Data = tc.data diff --git a/cmd/atecontroller/internal/controllers/gen.go b/cmd/atecontroller/internal/controllers/gen.go index a7ed1b1cf5..8c784d79af 100644 --- a/cmd/atecontroller/internal/controllers/gen.go +++ b/cmd/atecontroller/internal/controllers/gen.go @@ -20,6 +20,8 @@ package controllers // - internal/k8sresolver watches ateapi's EndpointSlices to dial it. // //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch +// The marker must name a literal namespace, so it carries the canonical one. +// hack/gen-rbac.sh rewrites it to the chart's release namespace on the way out. //+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch,namespace=ate-system //go:generate bash ../../../../hack/gen-rbac.sh diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go index 928c053a92..22c65bbbdd 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller.go @@ -33,13 +33,17 @@ import ( const ( networkPolicyFieldOwner = "ate-networkpolicy" - ateSystemNamespace = "ate-system" atenetRouterAppName = "atenet-router" ) type NetworkPolicyReconciler struct { client.Client Scheme *runtime.Scheme + + // SystemNamespace is the namespace atenet-router runs in. The generated + // ingress policy admits only that namespace, so a value that does not + // match the running router blocks every request to the worker pool. + SystemNamespace string } //+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch @@ -74,7 +78,7 @@ func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques func (r *NetworkPolicyReconciler) reconcileImpl(ctx context.Context, wp *atev1alpha1.WorkerPool) error { log := log.FromContext(ctx) - npAC := buildNetworkPolicyApplyConfig(wp) + npAC := r.buildNetworkPolicyApplyConfig(wp) if err := r.Apply(ctx, npAC, client.FieldOwner(networkPolicyFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply NetworkPolicy %s:%s: %w", *npAC.Namespace, *npAC.Name, err) @@ -86,7 +90,7 @@ func (r *NetworkPolicyReconciler) reconcileImpl(ctx context.Context, wp *atev1al return nil } -func buildNetworkPolicyApplyConfig(wp *atev1alpha1.WorkerPool) *networkingv1ac.NetworkPolicyApplyConfiguration { +func (r *NetworkPolicyReconciler) buildNetworkPolicyApplyConfig(wp *atev1alpha1.WorkerPool) *networkingv1ac.NetworkPolicyApplyConfiguration { np := networkingv1ac.NetworkPolicy(resources.NetworkPolicyName(wp.Name), wp.Namespace). WithLabels(map[string]string{ "ate.dev/worker-pool": wp.Name, @@ -110,7 +114,7 @@ func buildNetworkPolicyApplyConfig(wp *atev1alpha1.WorkerPool) *networkingv1ac.N WithFrom( networkingv1ac.NetworkPolicyPeer(). WithNamespaceSelector(metav1ac.LabelSelector(). - WithMatchLabels(map[string]string{"kubernetes.io/metadata.name": ateSystemNamespace})). + WithMatchLabels(map[string]string{"kubernetes.io/metadata.name": r.SystemNamespace})). WithPodSelector(metav1ac.LabelSelector(). WithMatchLabels(map[string]string{"app": atenetRouterAppName})), ), diff --git a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go index 0296b873fe..f1ff854d10 100644 --- a/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go +++ b/cmd/atecontroller/internal/controllers/networkpolicy_controller_test.go @@ -21,6 +21,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/types" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/resources" ) @@ -75,7 +76,7 @@ func TestWorkerPoolCreatesNetworkPolicy(t *testing.T) { return false, nil } fromPeer := ingressRule.From[0] - if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != ateSystemNamespace { + if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != installdefaults.SystemNamespace { return false, nil } if fromPeer.PodSelector == nil || fromPeer.PodSelector.MatchLabels["app"] != atenetRouterAppName { @@ -90,3 +91,24 @@ func TestWorkerPoolCreatesNetworkPolicy(t *testing.T) { return true, nil }) } + +// TestBuildNetworkPolicyRelocatedNamespace pins the ingress peer to the +// reconciler's SystemNamespace rather than the canonical install namespace. +// The rest of the suite configures the reconciler with the default, so it +// passes just as well against a hardcoded "ate-system"; this is the case that +// catches that. A policy naming the wrong namespace admits nobody, and the CNI +// drops every request to the pool with no error from substrate itself. +func TestBuildNetworkPolicyRelocatedNamespace(t *testing.T) { + const relocated = "substrate-test" + + r := &NetworkPolicyReconciler{SystemNamespace: relocated} + np := r.buildNetworkPolicyApplyConfig(testWorkerPoolApplyConfig(nil)) + + if len(np.Spec.Ingress) != 1 || len(np.Spec.Ingress[0].From) != 1 { + t.Fatalf("expected exactly one ingress rule with one peer, got %+v", np.Spec.Ingress) + } + got := np.Spec.Ingress[0].From[0].NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] + if got != relocated { + t.Errorf("ingress namespace selector = %q, want %q", got, relocated) + } +} diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index c2fdf57773..ba6a61cbec 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -26,6 +26,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/deviceplugin" + "github.com/agent-substrate/substrate/internal/installdefaults" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -74,7 +75,7 @@ const ( // Deployment managed by a WorkerPool. Only fields owned by this controller // are declared here. otel, when it carries an endpoint, is propagated to the // ateom container so it pushes telemetry to that collector. -func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettings) *appsv1ac.DeploymentApplyConfiguration { +func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettings, systemNamespace, ateletServiceAccount, routerServiceAccount string) *appsv1ac.DeploymentApplyConfiguration { labels := map[string]string{} annotations := map[string]string{} if wp.Spec.Template != nil { @@ -96,6 +97,11 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin "--atunnel-connect-listen-address=:8443", "--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem", "--atunnel-trust-bundle="+atunnelIdentityMountPath+"/trust-bundle.pem", + // The peers atunnel authenticates live in substrate's namespace, + // not the worker's, so the controller passes their identities + // rather than letting ateom assume the default install. + "--atunnel-client-identity="+installdefaults.SPIFFEID(systemNamespace, routerServiceAccount), + "--atunnel-broker-identity="+installdefaults.SPIFFEID(systemNamespace, ateletServiceAccount), "--atunnel-egress-listen-address=0.0.0.0:15001", "--atunnel-egress-trust-bundle="+atunnelEgressTrustMountPath+"/trust-bundle.pem", ). diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 85bff190f4..6743ede58d 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -30,6 +30,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/deviceplugin" + "github.com/agent-substrate/substrate/internal/installdefaults" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -205,7 +206,7 @@ func TestBuildDeploymentApplyConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := buildDeploymentApplyConfig(tt.wp, ateomOTelSettings{}) + got := buildDeploymentApplyConfig(tt.wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount) if diff := cmp.Diff(tt.want, got); diff != "" { t.Fatalf("buildDeploymentApplyConfig() mismatch (-want +got):\n%s", diff) } @@ -225,7 +226,7 @@ func TestBuildDeploymentApplyConfigMetadata(t *testing.T) { }, }) - got := buildDeploymentApplyConfig(wp, ateomOTelSettings{}) + got := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount) wantLabels := map[string]string{ "project": "agent-substrate", "team": "compute", @@ -267,7 +268,7 @@ func TestMicroVMPodShape(t *testing.T) { t.Run(tt.name, func(t *testing.T) { wp := testWorkerPoolApplyConfig(nil) wp.Spec.SandboxClass = tt.class - ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec + ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec // /dev/kvm must come from the device plugin, never a hostPath: a // hostPath mount carries no cgroup device allow rule, and the @@ -360,7 +361,7 @@ func TestMicroVMDeviceRequestsPreserveTemplateResources(t *testing.T) { }, }) wp.Spec.SandboxClass = atev1alpha1.SandboxClassMicroVM - c := buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec.Containers[0] + c := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec.Containers[0] if got, ok := deviceLimit(c, string(corev1.ResourceMemory)); !ok || got != "2Gi" { t.Errorf("memory limit = %q (present=%v), want 2Gi", got, ok) @@ -425,7 +426,7 @@ func TestAteomSecurityContextByClass(t *testing.T) { // TestTerminationGracePeriodSeconds asserts the pod's grace period is hardcoded to 3600s. func TestTerminationGracePeriodSeconds(t *testing.T) { wp := testWorkerPoolApplyConfig(nil) - ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}).Spec.Template.Spec + ps := buildDeploymentApplyConfig(wp, ateomOTelSettings{}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount).Spec.Template.Spec if ps.TerminationGracePeriodSeconds == nil { t.Fatalf("TerminationGracePeriodSeconds not set") } @@ -449,7 +450,7 @@ func TestBuildDeploymentApplyConfigOTelEndpoint(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{Endpoint: tt.endpoint}). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{Endpoint: tt.endpoint}, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). Spec.Template.Spec.Containers[0] env := envByName(c.Env) @@ -535,7 +536,7 @@ func TestBuildDeploymentApplyConfigMetricExportTuning(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). Spec.Template.Spec.Containers[0] env := envByName(c.Env) for _, k := range []string{"OTEL_METRIC_EXPORT_INTERVAL", "OTEL_METRIC_EXPORT_TIMEOUT"} { @@ -592,7 +593,7 @@ func TestBuildDeploymentApplyConfigTracesSamplerPropagation(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel). + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel, installdefaults.SystemNamespace, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). Spec.Template.Spec.Containers[0] env := envByName(c.Env) for _, k := range []string{"OTEL_TRACES_SAMPLER", "OTEL_TRACES_SAMPLER_ARG"} { @@ -720,6 +721,8 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf "--atunnel-connect-listen-address=:8443", "--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem", "--atunnel-trust-bundle="+atunnelIdentityMountPath+"/trust-bundle.pem", + "--atunnel-client-identity="+installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + "--atunnel-broker-identity="+installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "--atunnel-egress-listen-address=0.0.0.0:15001", "--atunnel-egress-trust-bundle="+atunnelEgressTrustMountPath+"/trust-bundle.pem", ). @@ -799,3 +802,68 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithLabels(map[string]string{"ate.dev/worker-pool": wp.Name}). WithSpec(podSpecAC))) } + +// TestBuildDeploymentAtunnelIdentitiesRelocatedNamespace pins the SPIFFE +// identities handed to ateom to the controller's namespace. atunnel runs in +// the actor's pod, so it cannot derive atelet's or the router's namespace +// itself; if these carry the wrong one, the credential broker handshake and +// actor ingress both fail closed. Every other case here passes the canonical +// namespace and so would pass against a hardcoded value too. +func TestBuildDeploymentAtunnelIdentitiesRelocatedNamespace(t *testing.T) { + const relocated = "substrate-test" + + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{}, relocated, installdefaults.AteletServiceAccount, installdefaults.RouterServiceAccount). + Spec.Template.Spec.Containers[0] + + want := map[string]string{ + "--atunnel-client-identity=": "spiffe://cluster.local/ns/substrate-test/sa/atenet-router", + "--atunnel-broker-identity=": "spiffe://cluster.local/ns/substrate-test/sa/atelet", + } + for flag, wantVal := range want { + var got string + for _, arg := range c.Args { + if strings.HasPrefix(arg, flag) { + got = strings.TrimPrefix(arg, flag) + } + } + if got == "" { + t.Fatalf("no %s argument found in %v", flag, c.Args) + } + if got != wantVal { + t.Errorf("%s%s, want %s%s", flag, got, flag, wantVal) + } + } +} + +// TestBuildDeploymentAtunnelIdentitiesPrefixedServiceAccounts covers a release +// that renames the ServiceAccounts — what the Helm chart does for any release +// not called "substrate", which is every install that consumes substrate as a +// subchart. The SPIFFE ID embeds the ServiceAccount name, so identities built +// from the compiled-in defaults name accounts that do not exist and atunnel +// rejects the peer. +func TestBuildDeploymentAtunnelIdentitiesPrefixedServiceAccounts(t *testing.T) { + const ( + namespace = "kagent-system" + atelet = "kagent-atelet" + router = "kagent-atenet-router" + ) + + c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), ateomOTelSettings{}, namespace, atelet, router). + Spec.Template.Spec.Containers[0] + + want := map[string]string{ + "--atunnel-client-identity=": "spiffe://cluster.local/ns/kagent-system/sa/kagent-atenet-router", + "--atunnel-broker-identity=": "spiffe://cluster.local/ns/kagent-system/sa/kagent-atelet", + } + for flag, wantVal := range want { + var got string + for _, arg := range c.Args { + if strings.HasPrefix(arg, flag) { + got = strings.TrimPrefix(arg, flag) + } + } + if got != wantVal { + t.Errorf("%s%s, want %s%s", flag, got, flag, wantVal) + } + } +} diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller.go b/cmd/atecontroller/internal/controllers/workerpool_controller.go index 7c962d62dd..7f1aeda961 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller.go @@ -52,6 +52,15 @@ type WorkerPoolReconciler struct { // OTelTracesSamplerArg is the OTEL_TRACES_SAMPLER_ARG propagated to ateom // pods. Ignored unless OTelTracesSampler is set. OTelTracesSamplerArg string + // SystemNamespace is the namespace substrate's control plane runs in, and + // AteletServiceAccount / RouterServiceAccount are the ServiceAccounts those + // components run as. Together they name the SPIFFE identities that atunnel + // authenticates inside each worker, which is why the ServiceAccount names + // are configuration and not constants: a Helm release that prefixes + // resource names changes them. + SystemNamespace string + AteletServiceAccount string + RouterServiceAccount string desiredWorkers metric.Int64ObservableUpDownCounter readyWorkers metric.Int64ObservableUpDownCounter @@ -116,7 +125,7 @@ func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alp MetricExportTimeout: r.OTelMetricExportTimeout, TracesSampler: r.OTelTracesSampler, TracesSamplerArg: r.OTelTracesSamplerArg, - }) + }, r.SystemNamespace, r.AteletServiceAccount, r.RouterServiceAccount) if err := r.Apply(ctx, depAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply Deployment: %w", err) } diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index c1f3b8b6f7..5c7ea51366 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -43,6 +43,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/testenv" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" ) @@ -86,8 +87,9 @@ func TestMain(m *testing.M) { } if err := (&NetworkPolicyReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SystemNamespace: installdefaults.SystemNamespace, }).SetupWithManager(mgr); err != nil { fmt.Fprintf(os.Stderr, "netpolicy controller setup failed: %v\n", err) os.Exit(1) diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index 7ef107ec1a..f50152aece 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -21,6 +21,7 @@ import ( "github.com/agent-substrate/substrate/cmd/atecontroller/internal/controllers" "github.com/agent-substrate/substrate/cmd/atecontroller/internal/workersync" "github.com/agent-substrate/substrate/internal/ateapiauth" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/serverboot" clientv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" @@ -70,6 +71,9 @@ var ( otelTracesSamplerArg = pflag.String("otel-traces-sampler-arg", os.Getenv("OTEL_TRACES_SAMPLER_ARG"), "Trace sampler argument set on ateom worker pods, ignored unless --otel-traces-sampler is set. Defaults to the controller's own OTEL_TRACES_SAMPLER_ARG.") + ateletServiceAccount = pflag.String("atelet-service-account", installdefaults.AteletServiceAccount, "ServiceAccount atelet runs as. It is the service-account segment of the SPIFFE ID each worker's atunnel expects on the credential broker, so it has to match what the deployment actually creates.") + routerServiceAccount = pflag.String("router-service-account", installdefaults.RouterServiceAccount, "ServiceAccount atenet-router runs as. It is the service-account segment of the SPIFFE ID each worker's atunnel accepts on actor ingress, so it has to match what the deployment actually creates.") + ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.") ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") ateapiClientCert = pflag.String("ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required.") @@ -152,7 +156,8 @@ func main() { ateapiClient := ateapipb.NewControlClient(ateapiConn) // EgressMITMTrustReconciler watches the Secret `egress-mitm-ca-pool`. - egressMITMCAPool := controllers.EgressMITMCAPoolRef() + systemNamespace := installdefaults.NamespaceFromPodEnv() + egressMITMCAPool := controllers.EgressMITMCAPoolRef(systemNamespace) mgr, err := ctrl.NewManager(k8sConfig, ctrl.Options{ Scheme: scheme, Cache: cache.Options{ @@ -180,21 +185,26 @@ func main() { OTelMetricExportTimeout: *otelMetricExportTimeout, OTelTracesSampler: *otelTracesSampler, OTelTracesSamplerArg: *otelTracesSamplerArg, + SystemNamespace: systemNamespace, + AteletServiceAccount: *ateletServiceAccount, + RouterServiceAccount: *routerServiceAccount, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WorkerPool") os.Exit(1) } if err = (&controllers.NetworkPolicyReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SystemNamespace: systemNamespace, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NetPolicy") os.Exit(1) } if err = (&controllers.EgressMITMTrustReconciler{ - Client: mgr.GetClient(), + Client: mgr.GetClient(), + SystemNamespace: systemNamespace, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "EgressMITMTrust") os.Exit(1) diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 3c70cb92cc..ba22d18060 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/cobra" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" + "github.com/agent-substrate/substrate/internal/installdefaults" ) func NewRouterCmd() *cobra.Command { @@ -46,6 +47,7 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().StringVar(&cfg.MetricsAddr, "metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") cmd.Flags().StringVar(&cfg.AtenetRouter, "atenet-router", string(atenetRouterEnvoy), "Router dataplane: envoy or agentgateway") cmd.Flags().StringVar(&cfg.Namespace, "namespace", "default", "Target operations namespace") + cmd.Flags().StringVar(&cfg.RouterServiceName, "router-service-name", installdefaults.RouterServiceName, "Service name of this atenet-router in the operations namespace. Override when the deployment renames the Service.") cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") cmd.Flags().StringVar(&cfg.AteapiAddr, "ateapi-address", "k8s:///api.ate-system.svc:443", "gRPC dial target for the cluster ateapi Control instance.") cmd.Flags().IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router") diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 16297ee68a..059b342ad1 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -74,18 +74,22 @@ type authConfig struct { // routerConfig holds deployment setup and endpoint options for the router node instance. type routerConfig struct { // Mode restricts the instance to one traffic direction. Empty means ModeAll. - Mode Mode - AtenetRouter string - Namespace string - Kubeconfig string - AteapiAddr string - HttpPort int - XdsPort int - ExtprocPort int - ExtprocAddr string - StatusPort int - HealthInterval time.Duration - HttpsPort int + Mode Mode + AtenetRouter string + Namespace string + // RouterServiceName is the Service name of this atenet-router in the + // operations namespace, used by /statusz to look up its own ClusterIP. + // Defaults to installdefaults.RouterServiceName. + RouterServiceName string + Kubeconfig string + AteapiAddr string + HttpPort int + XdsPort int + ExtprocPort int + ExtprocAddr string + StatusPort int + HealthInterval time.Duration + HttpsPort int // ConnectPlainTextPort and ConnectTLSPort are the plaintext and TLS // listener ports for CONNECT-tunneled traffic. Non-positive disables the // corresponding listener. diff --git a/cmd/atenet/internal/router/status.go b/cmd/atenet/internal/router/status.go index 6878755580..9fe196908f 100644 --- a/cmd/atenet/internal/router/status.go +++ b/cmd/atenet/internal/router/status.go @@ -68,7 +68,7 @@ func (s *RouterServer) getRouterIP(ctx context.Context) string { return "Offline Mode (No Cluster IP)" } - svc, err := s.clientset.CoreV1().Services(s.cfg.Namespace).Get(ctx, "atenet-router", metav1.GetOptions{}) + svc, err := s.clientset.CoreV1().Services(s.cfg.Namespace).Get(ctx, s.cfg.RouterServiceName, metav1.GetOptions{}) if err != nil { return fmt.Sprintf("Lookup Failed: %v", err) } diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 90cbfaec9c..62f547cc61 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -44,6 +44,7 @@ import ( "github.com/agent-substrate/substrate/internal/childreap" "github.com/agent-substrate/substrate/internal/contextlogging" "github.com/agent-substrate/substrate/internal/imagecache" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/otlprelay" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" @@ -72,7 +73,8 @@ var ( atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", ":8443", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = pflag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") podIdentityTrustBundle = pflag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet") - atunnelClientIdentity = pflag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS") + atunnelClientIdentity = pflag.String("atunnel-client-identity", installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity allowed to call actor ingress HTTPS") + ateletIdentity = pflag.String("atunnel-broker-identity", installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity the node-local atelet must present on the credential broker connection. Override when atelet runs outside the default namespace.") atunnelEgressListenAddress = pflag.String("atunnel-egress-listen-address", "0.0.0.0:15001", "Address for transparently intercepted actor egress TCP") egressGatewayTrustBundle = pflag.String("atunnel-egress-trust-bundle", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "Service DNS trust bundle for the remote egress gateway") readinessListenAddress = pflag.String("readiness-listen-address", "0.0.0.0:8080", "Address for HTTP readiness checks") @@ -210,7 +212,7 @@ func do(ctx context.Context) error { return err } - ateomService := NewService(interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + ateomService := NewService(interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle, *ateletIdentity) svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -369,6 +371,10 @@ type AteomService struct { podIdentityTrustBundlePath string // egressGatewayTrustBundlePath verifies the remote gateway's serving cert. egressGatewayTrustBundlePath string + // ateletSPIFFEID is the identity the node-local atelet must present on the + // credential broker connection. It names atelet's namespace, not this + // worker's, so it is configured rather than derived from the downward API. + ateletSPIFFEID string // activeActor is the actor whose workload this ateom is currently running, // or nil when it is "available". An ateom serves one actor at a time, so a @@ -420,7 +426,7 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath, ateletSPIFFEID string) *AteomService { return &AteomService{ lock: newCancelableMutex(), interiorNetNS: interiorNetNS, @@ -431,6 +437,7 @@ func NewService(interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, workerCredentialBundlePath: workerCredentialBundlePath, podIdentityTrustBundlePath: podIdentityTrustBundlePath, egressGatewayTrustBundlePath: egressGatewayTrustBundlePath, + ateletSPIFFEID: ateletSPIFFEID, cgroupRoot: defaultCgroupRoot, } } @@ -1048,6 +1055,7 @@ func (s *AteomService) prepareActorEgress(ctx context.Context, actorUID string, CredentialBundlePath: s.workerCredentialBundlePath, TrustBundlePath: s.podIdentityTrustBundlePath, ExpectedActorUID: actorUID, + AteletSPIFFEID: s.ateletSPIFFEID, }) if err != nil { return nil, fmt.Errorf("while configuring actor certificate broker: %w", err) diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 751881e37f..0e0fcf9230 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -45,6 +45,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateomnet" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/otlprelay" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" @@ -77,7 +78,8 @@ var ( atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", ":8443", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") podIdentityTrustBundle = flag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet") - atunnelClientIdentity = flag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS") + atunnelClientIdentity = flag.String("atunnel-client-identity", installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity allowed to call actor ingress HTTPS") + ateletIdentity = flag.String("atunnel-broker-identity", installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "SPIFFE identity the node-local atelet must present on the credential broker connection. Override when atelet runs outside the default namespace.") atunnelEgressListenAddress = flag.String("atunnel-egress-listen-address", "0.0.0.0:15001", "Address for transparently intercepted actor egress TCP") egressGatewayTrustBundle = flag.String("atunnel-egress-trust-bundle", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "Service DNS trust bundle for the remote egress gateway") readinessListenAddress = flag.String("readiness-listen-address", "0.0.0.0:8080", "Address for HTTP readiness checks") @@ -256,7 +258,7 @@ func do(ctx context.Context) error { }() slog.InfoContext(ctx, "atunnel egress serving", slog.String("address", *atunnelEgressListenAddress)) - ateomService := NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle) + ateomService := NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle, *ateletIdentity) svr := grpc.NewServer( grpc.StatsHandler(otelgrpc.NewServerHandler()), @@ -423,6 +425,10 @@ type AteomService struct { podIdentityTrustBundlePath string // egressGatewayTrustBundlePath verifies the remote gateway's serving cert. egressGatewayTrustBundlePath string + // ateletSPIFFEID is the identity the node-local atelet must present on the + // credential broker connection. It names atelet's namespace, not this + // worker's, so it is configured rather than derived from the downward API. + ateletSPIFFEID string // running maps actor UID -> the live micro-VM, kept so CheckpointWorkload can // pause+snapshot+teardown the same sandbox (and RestoreWorkload can track the @@ -476,7 +482,7 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath, ateletSPIFFEID string) *AteomService { return &AteomService{ lock: newCancelableMutex(), podUID: podUID, @@ -492,6 +498,7 @@ func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveM workerCredentialBundlePath: workerCredentialBundlePath, podIdentityTrustBundlePath: podIdentityTrustBundlePath, egressGatewayTrustBundlePath: egressGatewayTrustBundlePath, + ateletSPIFFEID: ateletSPIFFEID, running: map[string]*runningActor{}, } } @@ -520,6 +527,7 @@ func (s *AteomService) prepareActorEgress(ctx context.Context, actorUID string, CredentialBundlePath: s.workerCredentialBundlePath, TrustBundlePath: s.podIdentityTrustBundlePath, ExpectedActorUID: actorUID, + AteletSPIFFEID: s.ateletSPIFFEID, }) if err != nil { return nil, fmt.Errorf("while configuring actor certificate broker: %w", err) diff --git a/hack/gen-rbac.sh b/hack/gen-rbac.sh index baa22fa517..b7083b00e7 100755 --- a/hack/gen-rbac.sh +++ b/hack/gen-rbac.sh @@ -35,3 +35,9 @@ bash "${ROOT}/hack/run-tool.sh" controller-gen \ # Templatize the ClusterRole name. controller-gen emits ` name: ate-controller` # at column 0; the substitution is exact-match to stay robust. sed -i 's|^ name: ate-controller$| name: {{ include "substrate.fullname" (list "ate-controller" .) }}|' "${OUT}" + +# Templatize the namespaced Role's namespace. A kubebuilder rbac marker has to +# name a literal namespace, so controller-gen emits the canonical one; the +# chart installs into whatever namespace the release targets, and a Role left +# in ate-system would leave ate-controller without the permission there. +sed -i 's|^ namespace: ate-system$| namespace: {{ .Release.Namespace }}|' "${OUT}" diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 2e9fd0c03c..5a997053de 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -35,6 +35,16 @@ if [[ -z "${KUBECTL_CONTEXT:-}" ]]; then fi # otherwise just use the current cluster in KUBECONFIG ... +# Namespace the substrate control plane is installed into. Defaults to the +# canonical ate-system so existing flows are unaffected; override it to install +# a relocated release (the chart's --namespace must match). +ATE_NAMESPACE="${ATE_NAMESPACE:-ate-system}" + +# Service name fronting ateapi. It is the audience the API authentication config +# accepts, so it has to match the Service the deployment actually creates; a +# Helm release that prefixes resource names needs it set. +ATE_API_SERVICE_NAME="${ATE_API_SERVICE_NAME:-api}" + # ATE_DEMOS is an array that registers the prefix name of the demo functions. ATE_DEMOS=() @@ -224,8 +234,20 @@ rollout_timeout() { echo "${timeout}" } +# ensure_ate_namespace creates ATE_NAMESPACE and waits for it to go Active. +ensure_ate_namespace() { + if [[ "${ATE_NAMESPACE}" == "ate-system" ]]; then + run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml + else + run_kubectl create namespace "${ATE_NAMESPACE}" --dry-run=client -o yaml \ + | run_kubectl apply -f - + fi + run_kubectl wait --for=jsonpath='{.status.phase}'=Active \ + "namespace/${ATE_NAMESPACE}" --timeout=60s +} + default_postgres_connection_string() { - echo "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + echo "postgresql://postgres@postgres.${ATE_NAMESPACE}.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" } use_bundled_postgres() { @@ -361,14 +383,14 @@ apply_atenet_egress() { # bootstrap arrives as a ConfigMap change, and an otherwise unchanged # Deployment will not pick that up on its own. local running=false - if run_kubectl -n ate-system get deployment/atenet-egress >/dev/null 2>&1; then + if run_kubectl -n "${ATE_NAMESPACE}" get deployment/atenet-egress >/dev/null 2>&1; then running=true fi echo "${manifests}" | run_kubectl apply -f - if [[ "${running}" == "true" ]] && additional_egress_extproc_enabled; then - run_kubectl -n ate-system rollout restart deployment/atenet-egress + run_kubectl -n "${ATE_NAMESPACE}" rollout restart deployment/atenet-egress fi } @@ -417,28 +439,28 @@ apply_otel_endpoint_override() { fi local current="" - current="$(run_kubectl -n ate-system get configmap ate-otel-config \ + current="$(run_kubectl -n "${ATE_NAMESPACE}" get configmap ate-otel-config \ -o jsonpath='{.data.OTEL_EXPORTER_OTLP_ENDPOINT}' 2>/dev/null || true)" if [[ "${current}" == "${ATE_OTLP_ENDPOINT}" ]]; then return 0 fi echo "Overriding OTEL_EXPORTER_OTLP_ENDPOINT with ${ATE_OTLP_ENDPOINT}" - run_kubectl -n ate-system patch configmap ate-otel-config --type=merge \ + run_kubectl -n "${ATE_NAMESPACE}" patch configmap ate-otel-config --type=merge \ -p "{\"data\":{\"OTEL_EXPORTER_OTLP_ENDPOINT\":\"${ATE_OTLP_ENDPOINT}\"}}" local workload for workload in deployment/ate-api-server deployment/ate-controller \ deployment/atenet-router; do - if run_kubectl -n ate-system get "${workload}" >/dev/null 2>&1; then - run_kubectl -n ate-system rollout restart "${workload}" + if run_kubectl -n "${ATE_NAMESPACE}" get "${workload}" >/dev/null 2>&1; then + run_kubectl -n "${ATE_NAMESPACE}" rollout restart "${workload}" fi done # atelet DaemonSet names carry a version suffix; restart whichever versions # are installed. local ds="" - for ds in $(run_kubectl -n ate-system get daemonset -l app=atelet -o name 2>/dev/null); do - run_kubectl -n ate-system rollout restart "${ds}" + for ds in $(run_kubectl -n "${ATE_NAMESPACE}" get daemonset -l app=atelet -o name 2>/dev/null); do + run_kubectl -n "${ATE_NAMESPACE}" rollout restart "${ds}" done } @@ -460,7 +482,7 @@ create_jwt_authority_pool_secret() { run_kubectl_ate admin make-jwt-pool \ --key-id="1" \ --name="actor-id-jwt-pool" \ - --secret-namespace=ate-system + --secret-namespace="${ATE_NAMESPACE}" } create_actor_id_ca_pool_secret() { @@ -468,7 +490,7 @@ create_actor_id_ca_pool_secret() { run_kubectl_ate admin make-ca-pool \ --ca-id="1" \ --name="actor-id-ca-pool" \ - --secret-namespace=ate-system + --secret-namespace="${ATE_NAMESPACE}" } # The egress gateway has to verify actor client certificates, which means it @@ -482,7 +504,7 @@ create_actor_id_ca_certs_secret() { # inside the create-secret argument list, which would silently produce an # empty trust bundle and an egress gateway that rejects every actor. local actorid_root="" - actorid_root=$(ca_pool_root_pem actor-id-ca-pool ate-system) + actorid_root=$(ca_pool_root_pem actor-id-ca-pool "${ATE_NAMESPACE}") if [[ -z "${actorid_root}" ]]; then echo "error: failed to extract the actor-identity CA root for actor-id-ca-certs" >&2 return 1 @@ -490,7 +512,7 @@ create_actor_id_ca_certs_secret() { run_kubectl create secret generic actor-id-ca-certs \ --from-literal=ca.crt="${actorid_root}" \ - -n ate-system \ + -n "${ATE_NAMESPACE}" \ --dry-run=client -o yaml \ | run_kubectl apply -f - } @@ -504,7 +526,7 @@ create_egress_mitm_ca_pool_secret() { run_kubectl_ate admin make-ca-pool \ --ca-id="1" \ --name="egress-mitm-ca-pool" \ - --secret-namespace=ate-system \ + --secret-namespace="${ATE_NAMESPACE}" \ --key-type=ECDSAP256 } @@ -515,7 +537,7 @@ ensure_egress_mitm_ca_pool_secret() { if [[ "${ATE_EXPERIMENTAL_USE_SDSMINT:-false}" != "true" ]]; then return 0 fi - run_kubectl get secret -n ate-system egress-mitm-ca-pool >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" egress-mitm-ca-pool >/dev/null 2>&1 \ || create_egress_mitm_ca_pool_secret } @@ -544,7 +566,7 @@ wait_for_podcertificate_trust_bundles() { create_api_server_env_vars() { log_step "create_api_server_env_vars" - run_kubectl create namespace ate-system --dry-run=client -o yaml \ + run_kubectl create namespace "${ATE_NAMESPACE}" --dry-run=client -o yaml \ | run_kubectl apply -f - local postgres_connection_string="${ATE_API_POSTGRES_CONNECTION_STRING:-}" @@ -555,7 +577,7 @@ create_api_server_env_vars() { echo "POSTGRES_CONNECTION_STRING: ${postgres_connection_string}" - run_kubectl create configmap -n ate-system ate-api-server-envvars \ + run_kubectl create configmap -n "${ATE_NAMESPACE}" ate-api-server-envvars \ --from-literal=ATE_API_POSTGRES_CONNECTION_STRING="${postgres_connection_string}" \ --from-literal=ATE_API_POSTGRES_SCHEMA="${postgres_schema}" \ --dry-run=client -o yaml \ @@ -584,7 +606,7 @@ apply_podcert_workers_override() { create_api_authentication_config() { log_step "create_api_authentication_config" - run_kubectl create namespace ate-system --dry-run=client -o yaml \ + run_kubectl create namespace "${ATE_NAMESPACE}" --dry-run=client -o yaml \ | run_kubectl apply -f - local jwt_issuer="" @@ -604,8 +626,8 @@ create_api_authentication_config() { ;; esac local authentication_config - authentication_config=$(printf 'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n issuer: %s\n audiences: [api.ate-system.svc]\n%s' "${jwt_issuer}" "${discovery_config}") - run_kubectl create configmap -n ate-system ate-api-authentication \ + authentication_config=$(printf 'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n issuer: %s\n audiences: [%s.%s.svc]\n%s' "${jwt_issuer}" "${ATE_API_SERVICE_NAME}" "${ATE_NAMESPACE}" "${discovery_config}") + run_kubectl create configmap -n "${ATE_NAMESPACE}" ate-api-authentication \ --from-literal=authentication.yaml="${authentication_config}" \ --dry-run=client -o yaml \ | run_kubectl apply -f - @@ -666,8 +688,7 @@ deploy_ate_system() { ensure_substrate_version # Ensure namespace exists before applying RBAC or CRDs - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace # The atelet DaemonSet applied below and the demo WorkerPools' worker pods # schedule only to version-labeled nodes. @@ -729,13 +750,13 @@ deploy_ate_system() { log_step "Waiting for ATE system components to be ready..." if use_bundled_postgres; then - run_kubectl rollout status statefulset/postgres -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status statefulset/postgres -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" fi - run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/ate-controller -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/ate-api-server -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/ate-controller -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-router -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-egress -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" # After the bundle, which carries its own copy of ate-otel-config. apply_otel_endpoint_override @@ -744,18 +765,18 @@ deploy_ate_system() { # Ensure secrets and configmaps required by ate-apiserver ensure_apiserver_prerequisites() { log_step "ensure_apiserver_prerequisites" - run_kubectl get secret -n ate-system actor-id-jwt-pool >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" actor-id-jwt-pool >/dev/null 2>&1 \ || create_jwt_authority_pool_secret - run_kubectl get secret -n ate-system actor-id-ca-pool >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" actor-id-ca-pool >/dev/null 2>&1 \ || create_actor_id_ca_pool_secret # Derived from actor-id-ca-pool above, so it must come after it. - run_kubectl get secret -n ate-system actor-id-ca-certs >/dev/null 2>&1 \ + run_kubectl get secret -n "${ATE_NAMESPACE}" actor-id-ca-certs >/dev/null 2>&1 \ || create_actor_id_ca_certs_secret run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool >/dev/null 2>&1 \ || create_podcertificate_controller_cas # Always reconcile the PostgreSQL connection settings. create_api_server_env_vars - run_kubectl get configmap -n ate-system ate-api-authentication >/dev/null 2>&1 \ + run_kubectl get configmap -n "${ATE_NAMESPACE}" ate-api-authentication >/dev/null 2>&1 \ || create_api_authentication_config } @@ -765,15 +786,14 @@ deploy_ate_apiserver() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace ensure_apiserver_prerequisites apply_otel_config apply_otel_endpoint_override run_ko apply -f manifests/ate-install/ate-api-server.yaml - run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/ate-api-server -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" } deploy_atelet() { @@ -782,8 +802,7 @@ deploy_atelet() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace label_nodes_substrate_version apply_otel_config @@ -798,7 +817,7 @@ deploy_atelet() { manifest=$(run_ko resolve -f manifests/ate-install/atelet.yaml | substitute_version) fi echo "${manifest}" | run_kubectl apply -f - - run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status "daemonset/$(atelet_daemonset_name)" -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" } deploy_atenet() { @@ -806,8 +825,7 @@ deploy_atenet() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ - && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + ensure_ate_namespace apply_otel_config apply_otel_endpoint_override @@ -819,9 +837,9 @@ deploy_atenet() { ensure_egress_mitm_ca_pool_secret apply_atenet_egress run_ko apply -f manifests/ate-install/atenet-dns.yaml - run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/dns -n ate-system --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-router -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/atenet-egress -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" + run_kubectl rollout status deployment/dns -n "${ATE_NAMESPACE}" --timeout="$(rollout_timeout)" } # get_actor_state echoes the actor's state enum (e.g. ACTOR_STATE_SUSPENDED). @@ -889,7 +907,7 @@ delete_demo_actors_substrate() { return 1 fi - if ! run_kubectl get deployment/ate-api-server -n ate-system >/dev/null 2>&1; then + if ! run_kubectl get deployment/ate-api-server -n "${ATE_NAMESPACE}" >/dev/null 2>&1; then log_step "ate-api-server not found; skipping actor cleanup" return 0 fi @@ -1070,7 +1088,7 @@ delete_ate_system() { run_kubectl delete --ignore-not-found -f manifests/ate-install fi - run_kubectl delete --ignore-not-found -n ate-system daemonset -l app=atelet + run_kubectl delete --ignore-not-found -n "${ATE_NAMESPACE}" daemonset -l app=atelet run_kubectl delete --ignore-not-found \ -f manifests/ate-install/components/agentgateway/configmap.yaml run_kubectl delete --ignore-not-found -f manifests/ate-install/postgres.yaml diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 6765db009d..c3991e96eb 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -44,9 +44,53 @@ import ( metricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned" ) +// NamespaceEnv overrides the namespace the client looks for substrate in. The +// client runs outside the cluster, so it has no downward API to read and no +// pod namespace to fall back on; it matches the ATE_NAMESPACE that +// hack/install-ate.sh installed with. +const NamespaceEnv = "ATE_NAMESPACE" + +// APIServiceEnv and ClientServiceAccountEnv override the ateapi Service and the +// ServiceAccount the client mints its token from. The Helm chart prefixes both +// names for a release not called "substrate", and the client has no way to +// discover that from outside the cluster. const ( - apiServerName = "api.ate-system.svc" + APIServiceEnv = "ATE_API_SERVICE_NAME" + ClientServiceAccountEnv = "ATE_CLIENT_SERVICE_ACCOUNT" +) + +// apiServiceName is the Service that fronts ateapi. +func apiServiceName() string { + if n := os.Getenv(APIServiceEnv); n != "" { + return n + } + return installdefaults.APIServiceName +} + +// clientServiceAccount is the ServiceAccount the bearer token is minted from. +func clientServiceAccount() string { + if n := os.Getenv(ClientServiceAccountEnv); n != "" { + return n + } + return installdefaults.ClientServiceAccount +} +// systemNamespace is the namespace the client expects ateapi to be running in. +func systemNamespace() string { + if ns := os.Getenv(NamespaceEnv); ns != "" { + return ns + } + return installdefaults.SystemNamespace +} + +// apiServerName is the in-cluster DNS name of the ateapi Service. It is both +// the SNI presented on the connection and the audience of the minted token, so +// it has to track the namespace ateapi actually runs in. +func apiServerName() string { + return fmt.Sprintf("%s.%s.svc", apiServiceName(), systemNamespace()) +} + +const ( // serviceDNSSignerName and liveBundleSelector mirror the // clusterTrustBundle projected-volume sources that in-cluster clients // mount to verify ateapi's serving cert. @@ -84,7 +128,7 @@ func (c *Client) Close() { } // NewClient creates a new Ate API client. If endpoint is empty, it automatically port-forwards -// to the ate-api-server pod in the ate-system namespace. +// to the ate-api-server pod in substrate's namespace. func NewClient(ctx context.Context, kubeconfigPath, k8sContext, endpoint, tokenFile string, traceEnabled bool) (*Client, error) { tp, err := initTracing(ctx, traceEnabled) if err != nil { @@ -168,7 +212,7 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext, tokenFile // TODO: Should we special-case a LoadBalancer "api" Service and dial its // address directly instead of port-forwarding? - localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, installdefaults.SystemNamespace, installdefaults.APIServiceName, 443) + localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, systemNamespace(), apiServiceName(), 443) if err != nil { return nil, err } @@ -233,7 +277,7 @@ func serverTLSConfig(ctx context.Context, clientset kubernetes.Interface) (*tls. return &tls.Config{ MinVersion: tls.VersionTLS13, RootCAs: pool, - ServerName: apiServerName, + ServerName: apiServerName(), }, nil } @@ -253,11 +297,11 @@ func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset, expirationSeconds := int64(3600) tokenRequest := &authv1.TokenRequest{ Spec: authv1.TokenRequestSpec{ - Audiences: []string{apiServerName}, + Audiences: []string{apiServerName()}, ExpirationSeconds: &expirationSeconds, }, } - token, err := clientset.CoreV1().ServiceAccounts(installdefaults.SystemNamespace).CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) + token, err := clientset.CoreV1().ServiceAccounts(systemNamespace()).CreateToken(ctx, clientServiceAccount(), tokenRequest, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to request ateapi bearer token: %w", err) } diff --git a/internal/atunnel/credential.go b/internal/atunnel/credential.go index 65090af769..6b17aa26fc 100644 --- a/internal/atunnel/credential.go +++ b/internal/atunnel/credential.go @@ -23,9 +23,7 @@ import ( "crypto/x509" "fmt" "net" - "net/url" "os" - "path" "slices" "sync" "time" @@ -61,6 +59,11 @@ type BrokerConfig struct { // ExpectedActorUID prevents a mint started for an old activation from // receiving the newly assigned actor's certificate. ExpectedActorUID string + // AteletSPIFFEID is the SPIFFE ID atelet must present on the credential + // broker connection. It carries the namespace atelet runs in, which is not + // this worker's own namespace, so it is supplied by the caller rather than + // read from the downward API. See installdefaults.AteletSPIFFEID. + AteletSPIFFEID string } // NewBrokerCertificateSource creates one actor key for this activation. The key @@ -70,6 +73,9 @@ func NewBrokerCertificateSource(cfg BrokerConfig) (*BrokerCertificateSource, err if cfg.SocketPath == "" || cfg.CredentialBundlePath == "" || cfg.TrustBundlePath == "" || cfg.ExpectedActorUID == "" { return nil, fmt.Errorf("atunnel: credential broker socket, credentials, trust bundle, and expected actor UID are required") } + if cfg.AteletSPIFFEID == "" { + return nil, fmt.Errorf("atunnel: expected atelet SPIFFE ID is required") + } localCert, err := credbundle.Parse(cfg.CredentialBundlePath) if err != nil { return nil, fmt.Errorf("atunnel: load worker identity: %w", err) @@ -90,7 +96,7 @@ func NewBrokerCertificateSource(cfg BrokerConfig) (*BrokerCertificateSource, err if err != nil { return nil, fmt.Errorf("atunnel: generate actor private key: %w", err) } - expectedURI := (&url.URL{Scheme: "spiffe", Host: "cluster.local", Path: path.Join("ns", "ate-system", "sa", "atelet")}).String() + expectedURI := cfg.AteletSPIFFEID tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS13, InsecureSkipVerify: true, // Verification below supports SPIFFE Pod certificates without a DNS name. diff --git a/internal/atunnel/credential_test.go b/internal/atunnel/credential_test.go index 057b6fe9e8..8e598043e4 100644 --- a/internal/atunnel/credential_test.go +++ b/internal/atunnel/credential_test.go @@ -29,6 +29,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/substratex509" "google.golang.org/grpc" @@ -184,6 +185,7 @@ func newTestBrokerCertificateSource(t *testing.T, ateletIdentity *substratex509. CredentialBundlePath: credentialPath, TrustBundlePath: trustPath, ExpectedActorUID: "actor-uid", + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), }) if err != nil { t.Fatal(err) diff --git a/internal/e2e/env.go b/internal/e2e/env.go index ccc2e7f6b6..f0136102bd 100644 --- a/internal/e2e/env.go +++ b/internal/e2e/env.go @@ -17,6 +17,8 @@ package e2e import ( "fmt" "os" + + "github.com/agent-substrate/substrate/internal/installdefaults" ) // CheckEnv checks the list of env vars exist and returns their value. @@ -32,3 +34,28 @@ func CheckEnv(keys ...string) (map[string]string, error) { } return env, nil } + +// SystemNamespaceEnv names the namespace the substrate control plane under test +// was installed into. It mirrors the ATE_NAMESPACE hack/install-ate.sh +// installed with, and the --namespace the chart was released into. +const SystemNamespaceEnv = "E2E_SYSTEM_NAMESPACE" + +// SystemNamespace returns the namespace the control plane under test runs in, +// falling back to the canonical install namespace. +func SystemNamespace() string { + if ns := os.Getenv(SystemNamespaceEnv); ns != "" { + return ns + } + return installdefaults.SystemNamespace +} + +// ResourcePrefixEnv is the prefix the install under test puts on substrate's +// resource names. The Helm chart prefixes them with the release name for any +// release not called "substrate", which is what a subchart install produces, +// and the harness addresses several of those resources by name. +const ResourcePrefixEnv = "E2E_RESOURCE_PREFIX" + +// ResourceName returns name as the install under test renders it. +func ResourceName(name string) string { + return os.Getenv(ResourcePrefixEnv) + name +} diff --git a/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl b/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl index 97b78a7137..9ab467ec5b 100644 --- a/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl +++ b/internal/e2e/fixtures/testserver/egressprobe.yaml.tmpl @@ -15,7 +15,7 @@ # The egress probe used by internal/e2e/suites/egressauthz: testserver's probe # subcommand, with the credential volumes that suite mints. It is not a # ServerPod because it needs those volumes and a namespace the suite populates -# first, so it stays a bespoke manifest. ${NAMESPACE} is substituted by the +# first, so it stays a bespoke manifest. ${NAMESPACE} and ${SYSTEM_NAMESPACE} are substituted by the # suite with the randomized namespace it created, so the probe is torn down with # that namespace and leaves nothing behind. apiVersion: v1 @@ -33,6 +33,10 @@ spec: args: - "egressprobe" - "--listen=:8080" + # The gateway lives in substrate's namespace, which the probe cannot infer + # from inside the sandbox. ${SYSTEM_NAMESPACE} is substituted alongside + # ${NAMESPACE} by the suite. + - "--gateway-address=atenet-egress.${SYSTEM_NAMESPACE}.svc:443" ports: - name: http containerPort: 8080 diff --git a/internal/e2e/preflight.go b/internal/e2e/preflight.go index 14cae171d3..938bef93d8 100644 --- a/internal/e2e/preflight.go +++ b/internal/e2e/preflight.go @@ -38,10 +38,10 @@ func PreflightChecks() error { // Check deployments. deployments := []string{ - "ate-controller", - "ate-api-server", + ResourceName("ate-controller"), + ResourceName("ate-api-server"), } - namespace := "ate-system" + namespace := SystemNamespace() for _, depName := range deployments { dep, err := clients.K8s.AppsV1().Deployments(namespace).Get(ctx, depName, metav1.GetOptions{}) if err != nil { diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 7a9e536079..d139514cbf 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -36,8 +36,6 @@ import ( ) const ( - routerNamespace = "ate-system" - routerService = "atenet-router" // routerConnectServicePort is atenet-router's Service port for // CONNECT-tunneled traffic (see manifests/ate-install/atenet-router.yaml). // It is a distinct listener from the plain HTTP one Get/PostJSON use: @@ -79,7 +77,7 @@ func NewRouterClient(ctx context.Context) (*RouterClient, error) { return nil, fmt.Errorf("creating k8s client: %w", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, routerNamespace, routerService, 80) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, SystemNamespace(), ResourceName("atenet-router"), 80) if err != nil { return nil, err } @@ -188,7 +186,7 @@ func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, // in one test don't each pay for a fresh port-forward. func (c *RouterClient) ensureConnectPortForward(ctx context.Context) error { c.connectOnce.Do(func() { - localPort, stop, err := portforward.ServicePortForward(ctx, c.config, c.clientset, routerNamespace, routerService, routerConnectServicePort) + localPort, stop, err := portforward.ServicePortForward(ctx, c.config, c.clientset, SystemNamespace(), ResourceName("atenet-router"), routerConnectServicePort) if err != nil { c.connectErr = fmt.Errorf("port-forwarding to the router's CONNECT listener: %w", err) return diff --git a/internal/e2e/statusz.go b/internal/e2e/statusz.go index a04a07141d..4e3516aed4 100644 --- a/internal/e2e/statusz.go +++ b/internal/e2e/statusz.go @@ -51,7 +51,7 @@ func NewStatuszClient(ctx context.Context) (*StatuszClient, error) { return nil, fmt.Errorf("creating k8s client: %w", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, routerNamespace, routerService, routerStatusPort) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, SystemNamespace(), ResourceName("atenet-router"), routerStatusPort) if err != nil { return nil, err } diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index cd6fcd237d..dcbb0c1b94 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -1264,13 +1264,13 @@ func callActorPathOnce(t *testing.T, actorRef resources.ActorRef, method, path s t.Helper() clients := e2e.GetClients() - svc, err := clients.K8s.CoreV1().Services("ate-system").Get(context.Background(), "atenet-router", metav1.GetOptions{}) + svc, err := clients.K8s.CoreV1().Services(e2e.SystemNamespace()).Get(context.Background(), e2e.ResourceName("atenet-router"), metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("failed to get atenet-router service: %w", err) } selector := labels.SelectorFromSet(svc.Spec.Selector).String() - pods, err := clients.K8s.CoreV1().Pods("ate-system").List(context.Background(), metav1.ListOptions{LabelSelector: selector}) + pods, err := clients.K8s.CoreV1().Pods(e2e.SystemNamespace()).List(context.Background(), metav1.ListOptions{LabelSelector: selector}) if err != nil { return "", fmt.Errorf("failed to list atenet-router pods: %w", err) } diff --git a/internal/e2e/suites/egressauthz/actoridentity_test.go b/internal/e2e/suites/egressauthz/actoridentity_test.go index ffe82e5e2a..b2bcce6c36 100644 --- a/internal/e2e/suites/egressauthz/actoridentity_test.go +++ b/internal/e2e/suites/egressauthz/actoridentity_test.go @@ -73,16 +73,16 @@ const ( // the secret ateapi signs with. func actorIdentityCA(t *testing.T, ctx context.Context) *localca.CA { t.Helper() - secret, err := e2e.GetClients().K8s.CoreV1().Secrets(egressNamespace).Get(ctx, actorIDCASecret, metav1.GetOptions{}) + secret, err := e2e.GetClients().K8s.CoreV1().Secrets(e2e.SystemNamespace()).Get(ctx, actorIDCASecret, metav1.GetOptions{}) if err != nil { - t.Fatalf("reading actor-identity CA pool secret %s/%s: %v", egressNamespace, actorIDCASecret, err) + t.Fatalf("reading actor-identity CA pool secret %s/%s: %v", e2e.SystemNamespace(), actorIDCASecret, err) } pool, err := localca.Unmarshal(secret.Data[actorIDCASecretKey]) if err != nil { - t.Fatalf("parsing actor-identity CA pool from %s/%s key %q: %v", egressNamespace, actorIDCASecret, actorIDCASecretKey, err) + t.Fatalf("parsing actor-identity CA pool from %s/%s key %q: %v", e2e.SystemNamespace(), actorIDCASecret, actorIDCASecretKey, err) } if len(pool.CAs) == 0 { - t.Fatalf("actor-identity CA pool %s/%s contains no CA", egressNamespace, actorIDCASecret) + t.Fatalf("actor-identity CA pool %s/%s contains no CA", e2e.SystemNamespace(), actorIDCASecret) } // CAs[0] is the one that signs: ateapi's MintCert makes the same choice. return pool.CAs[0] diff --git a/internal/e2e/suites/egressauthz/egressauthz_test.go b/internal/e2e/suites/egressauthz/egressauthz_test.go index 1fa04ffaff..3b39da7162 100644 --- a/internal/e2e/suites/egressauthz/egressauthz_test.go +++ b/internal/e2e/suites/egressauthz/egressauthz_test.go @@ -55,7 +55,6 @@ import ( const ( // Where the gateway's CA lives, fixed by hack/install-ate.sh. - egressNamespace = "ate-system" probeName = "egressprobe" ) @@ -163,6 +162,7 @@ func startProbe(t *testing.T, ctx context.Context) *probeClient { } manifest := filepath.Join(t.TempDir(), "egressprobe.yaml") rendered := strings.ReplaceAll(string(tmpl), "${NAMESPACE}", ns) + rendered = strings.ReplaceAll(rendered, "${SYSTEM_NAMESPACE}", e2e.SystemNamespace()) if err := os.WriteFile(manifest, []byte(rendered), 0o644); err != nil { t.Fatalf("writing rendered egressprobe manifest: %v", err) } diff --git a/internal/e2e/suites/metrics/metrics_test.go b/internal/e2e/suites/metrics/metrics_test.go index 38d61dba00..d03923df26 100644 --- a/internal/e2e/suites/metrics/metrics_test.go +++ b/internal/e2e/suites/metrics/metrics_test.go @@ -43,7 +43,7 @@ func TestPlatformMetricsEmitted(t *testing.T) { tmpl := e2e.SubstrateCounterFixture() actorID := fmt.Sprintf("metrics-probe-%d", time.Now().UnixNano()) metricPrefixes := append([]string(nil), e2e.PlatformMetricPrefixes...) - router, err := clients.K8s.AppsV1().Deployments("ate-system").Get(ctx, "atenet-router", metav1.GetOptions{}) + router, err := clients.K8s.AppsV1().Deployments(e2e.SystemNamespace()).Get(ctx, e2e.ResourceName("atenet-router"), metav1.GetOptions{}) if err != nil { t.Fatalf("Get atenet-router deployment: %v", err) } diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 331cfdd61a..024abb963f 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -311,7 +311,7 @@ func routerAddress(t *testing.T, ctx context.Context) string { if err != nil { t.Fatalf("creating k8s client: %v", err) } - localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, "ate-system", "atenet-router", 80) + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, e2e.SystemNamespace(), e2e.ResourceName("atenet-router"), 80) if err != nil { t.Fatalf("port-forwarding to the router: %v", err) } diff --git a/internal/e2e/suites/networkpolicy/networkpolicy_test.go b/internal/e2e/suites/networkpolicy/networkpolicy_test.go index 54a929c3ce..22b7357f9c 100644 --- a/internal/e2e/suites/networkpolicy/networkpolicy_test.go +++ b/internal/e2e/suites/networkpolicy/networkpolicy_test.go @@ -36,7 +36,6 @@ import ( ) const ( - ateSystemNamespace = "ate-system" atenetRouterAppName = "atenet-router" ) @@ -103,8 +102,8 @@ func TestNetworkPolicyLifecycleAndReconciliation(t *testing.T) { t.Fatalf("expected exactly 1 ingress from peer, got %d", len(ingressRule.From)) } fromPeer := ingressRule.From[0] - if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != ateSystemNamespace { - t.Errorf("expected namespace selector for %s, got %v", ateSystemNamespace, fromPeer.NamespaceSelector) + if fromPeer.NamespaceSelector == nil || fromPeer.NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"] != e2e.SystemNamespace() { + t.Errorf("expected namespace selector for %s, got %v", e2e.SystemNamespace(), fromPeer.NamespaceSelector) } if fromPeer.PodSelector == nil || fromPeer.PodSelector.MatchLabels["app"] != atenetRouterAppName { t.Errorf("expected pod selector for %s, got %v", atenetRouterAppName, fromPeer.PodSelector) diff --git a/internal/e2e/trustbundle.go b/internal/e2e/trustbundle.go index 948cf31015..045e5d2643 100644 --- a/internal/e2e/trustbundle.go +++ b/internal/e2e/trustbundle.go @@ -39,9 +39,7 @@ const ( // EgressTrustBundleObjectName is the reconciler-owned ClusterTrustBundle. EgressTrustBundleObjectName = "egress-mitm.ate.dev:mitm:primary-bundle" - egressCAPoolNamespace = "ate-system" - egressCAPoolSecretName = "egress-mitm-ca-pool" - egressCAPoolSecretKey = "pool" + egressCAPoolSecretKey = "pool" ) // EnsureEgressTrustBundle makes sure the egress trust bundle exists, then @@ -68,12 +66,12 @@ func ReplaceEgressTrustPool(t *testing.T, ctx context.Context, clients *Clients, if !createEgressTrustPool(t, ctx, clients, secret) { // Took over an existing pool: overwrite its contents without adopting // its cleanup, since whoever created it registered one already. - existing, err := clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Get(ctx, egressCAPoolSecretName, metav1.GetOptions{}) + existing, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Get(ctx, ResourceName("egress-mitm-ca-pool"), metav1.GetOptions{}) if err != nil { t.Fatalf("reading existing CA pool secret: %v", err) } existing.Data = secret.Data - if _, err := clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Update(ctx, existing, metav1.UpdateOptions{}); err != nil { + if _, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Update(ctx, existing, metav1.UpdateOptions{}); err != nil { t.Fatalf("updating CA pool secret: %v", err) } } @@ -103,7 +101,7 @@ func newEgressTrustPool(t *testing.T) (*corev1.Secret, string) { t.Fatalf("encoding the egress CA private key: %v", err) } return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: egressCAPoolNamespace, Name: egressCAPoolSecretName}, + ObjectMeta: metav1.ObjectMeta{Namespace: SystemNamespace(), Name: ResourceName("egress-mitm-ca-pool")}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{ egressCAPoolSecretKey: poolBytes, @@ -120,14 +118,14 @@ func newEgressTrustPool(t *testing.T) (*corev1.Secret, string) { // behind, and no caller removes a pool it merely found. func createEgressTrustPool(t *testing.T, ctx context.Context, clients *Clients, secret *corev1.Secret) bool { t.Helper() - if _, err := clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err := clients.K8s.CoreV1().Secrets(SystemNamespace()).Create(ctx, secret, metav1.CreateOptions{}); err != nil { if !apierrors.IsAlreadyExists(err) { - t.Fatalf("creating CA pool secret %s/%s: %v", egressCAPoolNamespace, egressCAPoolSecretName, err) + t.Fatalf("creating CA pool secret %s/%s: %v", SystemNamespace(), ResourceName("egress-mitm-ca-pool"), err) } return false } t.Cleanup(func() { - _ = clients.K8s.CoreV1().Secrets(egressCAPoolNamespace).Delete(context.Background(), egressCAPoolSecretName, metav1.DeleteOptions{}) + _ = clients.K8s.CoreV1().Secrets(SystemNamespace()).Delete(context.Background(), ResourceName("egress-mitm-ca-pool"), metav1.DeleteOptions{}) }) return true } diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go index 8f47d84f0b..7bc83a0796 100644 --- a/internal/installdefaults/installdefaults.go +++ b/internal/installdefaults/installdefaults.go @@ -18,7 +18,11 @@ // the canonical layout pass actual values via the corresponding flags. package installdefaults -import "os" +import ( + "net/url" + "os" + "path" +) const ( // SystemNamespace is the namespace where substrate's control-plane @@ -30,6 +34,19 @@ const ( RouterServiceName = "atenet-router" // DNSServiceName is the Service name of substrate's CoreDNS. DNSServiceName = "dns" + // ClientServiceAccount is the ServiceAccount an out-of-cluster client mints + // its ateapi bearer token from. + ClientServiceAccount = "ate-client" + + // AteletTrustDomain, AteletServiceAccount and RouterServiceAccount are the + // trust-domain and service-account segments of the SPIFFE IDs that atelet + // and atenet-router Pod certificates carry, as minted by the podidentity + // signer (cmd/podcertcontroller/internal/podidentitysigner). The namespace + // segment is the namespace they run in, which callers resolve themselves + // rather than assume. + AteletTrustDomain = "cluster.local" + AteletServiceAccount = "atelet" + RouterServiceAccount = "atenet-router" // PodNamespaceEnv is the conventional env var name for the namespace // a pod is running in, exposed via Kubernetes' downward API. @@ -45,3 +62,23 @@ func NamespaceFromPodEnv() string { } return SystemNamespace } + +// SPIFFEID returns the SPIFFE ID that Pod certificates for serviceAccount in +// namespace carry. Peers authenticate by comparing against this exact string. +func SPIFFEID(namespace, serviceAccount string) string { + return (&url.URL{ + Scheme: "spiffe", + Host: AteletTrustDomain, + Path: path.Join("ns", namespace, "sa", serviceAccount), + }).String() +} + +// AteletSPIFFEID returns the SPIFFE ID atelet presents when it runs in namespace. +func AteletSPIFFEID(namespace string) string { + return SPIFFEID(namespace, AteletServiceAccount) +} + +// RouterSPIFFEID returns the SPIFFE ID atenet-router presents when it runs in namespace. +func RouterSPIFFEID(namespace string) string { + return SPIFFEID(namespace, RouterServiceAccount) +} diff --git a/internal/installdefaults/installdefaults_test.go b/internal/installdefaults/installdefaults_test.go new file mode 100644 index 0000000000..f45bbb7c33 --- /dev/null +++ b/internal/installdefaults/installdefaults_test.go @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installdefaults + +import "testing" + +func TestAteletSPIFFEID(t *testing.T) { + tests := []struct { + name string + namespace string + want string + }{ + { + // The canonical install. Peers reject any other string, so this + // value is effectively wire format: changing it breaks the atelet + // mTLS handshake for every existing deployment. + name: "default namespace", + namespace: SystemNamespace, + want: "spiffe://cluster.local/ns/ate-system/sa/atelet", + }, + { + name: "namespace the install was relocated to", + namespace: "team-a-substrate", + want: "spiffe://cluster.local/ns/team-a-substrate/sa/atelet", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := AteletSPIFFEID(tt.namespace); got != tt.want { + t.Errorf("AteletSPIFFEID(%q) = %q, want %q", tt.namespace, got, tt.want) + } + }) + } +} + +func TestRouterSPIFFEID(t *testing.T) { + // Matches the --atunnel-client-identity default the ateom binaries ship + // with, which is what actor ingress authenticates the router against. + const want = "spiffe://cluster.local/ns/ate-system/sa/atenet-router" + if got := RouterSPIFFEID(SystemNamespace); got != want { + t.Errorf("RouterSPIFFEID(%q) = %q, want %q", SystemNamespace, got, want) + } +} + +func TestNamespaceFromPodEnv(t *testing.T) { + t.Run("falls back to the install default when unset", func(t *testing.T) { + t.Setenv(PodNamespaceEnv, "") + if got := NamespaceFromPodEnv(); got != SystemNamespace { + t.Errorf("NamespaceFromPodEnv() = %q, want %q", got, SystemNamespace) + } + }) + + t.Run("prefers the downward API value", func(t *testing.T) { + t.Setenv(PodNamespaceEnv, "team-a-substrate") + if got := NamespaceFromPodEnv(); got != "team-a-substrate" { + t.Errorf("NamespaceFromPodEnv() = %q, want %q", got, "team-a-substrate") + } + }) +}