Skip to content
3 changes: 3 additions & 0 deletions charts/substrate/templates/ate-api-server.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ spec:
- "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json"
- "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json"
- "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443"
# ateapi verifies atelet's SPIFFE ID, which embeds the ServiceAccount
# name this chart prefixes for a renamed release.
- "--atelet-service-account={{ include "substrate.fullname" (list "atelet" .) }}"
- "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem"
- "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem"
- "--drain-delay=13s"
Expand Down
16 changes: 15 additions & 1 deletion charts/substrate/templates/ate-controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,23 @@ spec:
# dials the right backend when substrate is installed as a subchart.
- "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443"
- "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem"
# A SPIFFE ID names a ServiceAccount, and this chart prefixes those for
# any release not called "substrate". The controller stamps these
# identities onto every worker, so it has to be told the names actually
# rendered rather than assume the canonical ones.
- "--atelet-service-account={{ include "substrate.fullname" (list "atelet" .) }}"
- "--router-service-account={{ include "substrate.fullname" (list "atenet-router" .) }}"
- "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem"
{{- if .Values.otel.endpoint }}
env:
# ate-controller resolves substrate's namespace from the downward API.
# It names the atelet and atenet-router SPIFFE identities handed to
# each worker's atunnel, which live in substrate's namespace and not
# the worker's, so the controller cannot infer them any other way.
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- if .Values.otel.endpoint }}
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: {{ .Values.otel.endpoint | quote }}
{{- end }}
Expand Down
6 changes: 6 additions & 0 deletions charts/substrate/templates/atelet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,12 @@ spec:
- --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem
- --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem
- --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem
# The atelet binary defaults these to api.ate-system.svc, which is
# correct only for the canonical render. The credential broker dials
# ateapi to mint actor certificates, so pass the chart-resolved
# Service and the matching name on its serving cert.
- --ateapi-address={{ include "substrate.ateApi.endpoint" . }}
- --ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc
{{- with .Values.atelet.extraArgs }}
{{ toYaml . | indent 8 }}
{{- end }}
Expand Down
3 changes: 3 additions & 0 deletions charts/substrate/templates/atenet-router.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,9 @@ spec:
- "--port-extproc=50051"
- "--extproc-address=127.0.0.1"
- "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443"
# /statusz looks up this instance's own ClusterIP by Service name, and
# the chart prefixes that name for any release not called "substrate".
- "--router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }}"
- "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem"
- "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem"
- "--status-port=4040"
Expand Down
2 changes: 1 addition & 1 deletion charts/substrate/templates/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ include "substrate.fullname" (list "ate-controller" .) }}
namespace: ate-system
namespace: {{ .Release.Namespace }}
rules:
- apiGroups:
- discovery.k8s.io
Expand Down
31 changes: 10 additions & 21 deletions cmd/ateapi/internal/actoridentity/actoridentity.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,33 +59,26 @@ type Server struct {
// is entitled to the actor it is asking for a credential for.
store store.Interface
workers *workercache.Cache

// ateletSPIFFEID is the identity the calling atelet must present to mint
// actor credentials.
ateletSPIFFEID string
}

var _ ateapipb.ActorIdentityServer = (*Server)(nil)

func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache) *Server {
func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache, ateletSPIFFEID string) *Server {
return &Server{
actorIdentityJWTIssuer: actorIdentityJWTIssuer,
actorIDJWTPoolFile: actorIDJWTPoolFile,
actorIDCAPool: actorIDCAPool,
store: store,
workers: workers,
ateletSPIFFEID: ateletSPIFFEID,
}
}

// The SPIFFE identity that atelet client certs carry, as minted by the
// podidentity signer (cmd/podcertcontroller/internal/podidentitysigner).
//
// These mirror the constants the atelet dialer verifies against in
// cmd/ateapi/internal/controlapi/dialer.go. They are duplicated rather than
// imported so that this package does not depend on controlapi for three
// strings; if a third pkg that need these constants appears, they should move to a shared package.
const (
ateletTrustDomain = "cluster.local"
ateletNamespace = "ate-system"
ateletSA = "atelet"
actorCertificateLifetime = time.Hour
)
const actorCertificateLifetime = time.Hour

func (s *Server) MintJWT(ctx context.Context, req *ateapipb.MintJWTRequest) (*ateapipb.MintJWTResponse, error) {
caller, ok := principal.FromContext(ctx)
Expand Down Expand Up @@ -148,7 +141,7 @@ func (s *Server) MintJWT(ctx context.Context, req *ateapipb.MintJWTRequest) (*at
}

func (s *Server) MintCert(ctx context.Context, req *ateapipb.MintCertRequest) (*ateapipb.MintCertResponse, error) {
caller, err := authenticateAtelet(ctx)
caller, err := s.authenticateAtelet(ctx)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -243,7 +236,7 @@ type ateletCaller struct {
// pod-identity CA (see buildServerCreds in cmd/ateapi/main.go), so the
// extensions read here are trustworthy: only the pod-identity signer can mint
// a certificate carrying a given pod's node name.
func authenticateAtelet(ctx context.Context) (*ateletCaller, error) {
func (s *Server) authenticateAtelet(ctx context.Context) (*ateletCaller, error) {
p, ok := peer.FromContext(ctx)
if !ok {
return nil, status.Errorf(codes.Unauthenticated, "no peer transport information found")
Expand All @@ -262,11 +255,7 @@ func authenticateAtelet(ctx context.Context) (*ateletCaller, error) {
// Only atelet may mint actor credentials. Everything else with a valid
// pod-identity certificate — including the actor workloads themselves — is
// rejected here.
expected := (&url.URL{
Scheme: "spiffe",
Host: ateletTrustDomain,
Path: path.Join("ns", ateletNamespace, "sa", ateletSA),
}).String()
expected := s.ateletSPIFFEID
if len(leaf.URIs) == 0 || leaf.URIs[0].String() != expected {
slog.WarnContext(ctx, "ActorIdentity denied: caller is not atelet",
slog.Any("uris", leaf.URIs), slog.String("expected", expected))
Expand Down
56 changes: 52 additions & 4 deletions cmd/ateapi/internal/actoridentity/actoridentity_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest"
"github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache"
"github.com/agent-substrate/substrate/internal/installdefaults"
"github.com/agent-substrate/substrate/internal/localca"
"github.com/agent-substrate/substrate/internal/principal"
"github.com/agent-substrate/substrate/internal/resources"
Expand Down Expand Up @@ -74,6 +75,19 @@ const (
// populates. Self-signing is sufficient because the code under test reads an
// already transport-verified peer certificate and never re-validates the chain
// itself.
// The atelet SPIFFE segments the tests build peer certificates from. They
// mirror what a default install mints, which is what the Server under test is
// configured with.
const (
ateletTrustDomain = installdefaults.AteletTrustDomain
ateletNamespace = installdefaults.SystemNamespace
ateletSA = installdefaults.AteletServiceAccount
)

// ateletSPIFFEID is the identity a default install's atelet presents, and what
// the Server under test is configured to accept.
var ateletSPIFFEID = installdefaults.SPIFFEID(ateletNamespace, ateletSA)

func newTestCert(t *testing.T, spiffePath string, podIdentity *substratex509.PodIdentity) *x509.Certificate {
t.Helper()

Expand Down Expand Up @@ -165,7 +179,7 @@ func newTestServer(t *testing.T, st store.Interface) *Server {
t.Fatalf("start worker cache: %v", err)
}
}
return New("issuer", "", pool, st, workers)
return New("issuer", "", pool, st, workers, ateletSPIFFEID)
}

// staleWatchStore wraps a store with a WatchWorkers that never delivers,
Expand Down Expand Up @@ -324,11 +338,11 @@ func newTestServerWithCache(t *testing.T, st store.Interface, workers *workercac
t.Fatalf("generate CA: %v", err)
}
pool := &localca.ConcretePool{CAs: []*localca.CA{ca}}
return New("issuer", "", pool, st, workers)
return New("issuer", "", pool, st, workers, ateletSPIFFEID)
}

func TestMintJWTRequiresConfiguredJWTProvider(t *testing.T) {
srv := &Server{actorIdentityJWTIssuer: "https://kubernetes.example"}
srv := &Server{actorIdentityJWTIssuer: "https://kubernetes.example", ateletSPIFFEID: ateletSPIFFEID}
for _, tt := range []struct {
name string
ctx context.Context
Expand Down Expand Up @@ -918,7 +932,7 @@ func TestMintCertAuthorizesBeforeSigning(t *testing.T) {
ActiveForSigning: "test-actor-ca",
}

srv := New("issuer", "", pool, st, workers)
srv := New("issuer", "", pool, st, workers, ateletSPIFFEID)

actor, err := st.GetActor(ctx, resources.ActorRef{Atespace: testAtespace, Name: testActorName})
if err != nil {
Expand Down Expand Up @@ -1082,3 +1096,37 @@ func TestValidateMintCertRequest(t *testing.T) {
})
}
}

// TestAuthenticateAteletHonorsConfiguredNamespace checks that the namespace
// the Server is configured with is the one it accepts, and that the canonical
// namespace is rejected when the install lives elsewhere.
//
// The table in TestMintCertAuthorization covers a caller from the wrong
// namespace, but its Server is always configured with the default, so it holds
// against a hardcoded "ate-system" too. Only the relocated case distinguishes
// "reads its configuration" from "happens to agree with the constant".
func TestAuthenticateAteletHonorsConfiguredNamespace(t *testing.T) {
const relocated = "substrate-test"

srv := &Server{ateletSPIFFEID: installdefaults.SPIFFEID(relocated, ateletSA)}

t.Run("accepts atelet from the configured namespace", func(t *testing.T) {
id := podIdentityOn(testNode)
id.Namespace = relocated
cert := newTestCert(t, path.Join("ns", relocated, "sa", ateletSA), id)

if _, err := srv.authenticateAtelet(ctxWithCert(cert)); err != nil {
t.Errorf("authenticateAtelet() = %v, want success for an atelet in %q", err, relocated)
}
})

t.Run("rejects atelet from the canonical namespace", func(t *testing.T) {
id := podIdentityOn(testNode)
id.Namespace = installdefaults.SystemNamespace
cert := newTestCert(t, path.Join("ns", installdefaults.SystemNamespace, "sa", ateletSA), id)

if _, err := srv.authenticateAtelet(ctxWithCert(cert)); status.Code(err) != codes.PermissionDenied {
t.Errorf("authenticateAtelet() code = %v, want PermissionDenied for an atelet in %q", status.Code(err), installdefaults.SystemNamespace)
}
})
}
21 changes: 8 additions & 13 deletions cmd/ateapi/internal/controlapi/dialer.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,7 @@ var ErrNoAteletOnNode = errors.New("no atelet pod found on node")

// The SPIFFE identity that atelet serving certs carry, as minted by the
// podidentity signer (cmd/podcertcontroller/internal/podidentitysigner).
// The namespace part is ateletNamespace, declared in informer.go.
const (
trustDomainName = "cluster.local"
ateletSA = "atelet"
)

// The namespace part is the dialer's ateletNamespace.
// AteletDialer handles gRPC connections to Atelet pods.
type AteletDialer struct {
workerIndexer cache.Indexer
Expand Down Expand Up @@ -83,13 +78,13 @@ func WithInsecureCredentials() DialerOption {

// NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath
// are used to build the per-atelet mTLS credentials used for every atelet connection.
func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer {
func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, ateletSPIFFEID, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer {
d := &AteletDialer{
workerIndexer: workerIndexer,
ateletIndexer: ateletIndexer,
ateletConns: newAteletConnCache(1024),
dialCredentials: func(expectedPodUID string) (credentials.TransportCredentials, error) {
tlsConfig, err := buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID)
tlsConfig, err := buildTLSConfig(ateletSPIFFEID, clientBundlePath, serverCAPath, expectedPodUID)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -196,18 +191,18 @@ func (d *AteletDialer) DialForAteletOnNode(nodeName string) (*grpc.ClientConn, e
return ateletConn, nil
}

func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) {
trustDomain, err := spiffeid.TrustDomainFromString(trustDomainName)
func buildTLSConfig(ateletSPIFFEID, clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) {
trustDomain, err := spiffeid.TrustDomainFromString(installdefaults.AteletTrustDomain)
if err != nil {
return nil, fmt.Errorf("while parsing trust domain %q: %w", trustDomainName, err)
return nil, fmt.Errorf("while parsing trust domain %q: %w", installdefaults.AteletTrustDomain, err)
}
bundle, err := x509bundle.Load(trustDomain, serverCAPath)
if err != nil {
return nil, fmt.Errorf("while loading CA bundle from %s: %w", serverCAPath, err)
}
expectedID, err := spiffeid.FromSegments(trustDomain, "ns", installdefaults.NamespaceFromPodEnv(), "sa", ateletSA)
expectedID, err := spiffeid.FromString(ateletSPIFFEID)
if err != nil {
return nil, fmt.Errorf("while building expected atelet SPIFFE ID: %w", err)
return nil, fmt.Errorf("while parsing expected atelet SPIFFE ID %q: %w", ateletSPIFFEID, err)
}

verify, err := verifyAteletServerCert(bundle, expectedID, expectedPodUID)
Expand Down
12 changes: 6 additions & 6 deletions cmd/ateapi/internal/controlapi/dialer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,12 @@ import (
const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet"

func TestAteletDialerInsecureRequiresOptIn(t *testing.T) {
secure := NewAteletDialer(nil, nil, "", "")
secure := NewAteletDialer(nil, nil, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "")
if _, err := secure.dialCredentials("pod-uid"); err == nil {
t.Fatal("secure dialer accepted empty credential paths")
}

insecureDialer := NewAteletDialer(nil, nil, "", "", WithInsecureCredentials())
insecureDialer := NewAteletDialer(nil, nil, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", WithInsecureCredentials())
creds, err := insecureDialer.dialCredentials("pod-uid")
if err != nil {
t.Fatalf("insecure dial credentials: %v", err)
Expand Down Expand Up @@ -380,7 +380,7 @@ func TestDialForAteletOnNode(t *testing.T) {
}

t.Run("no atelet on node", func(t *testing.T) {
d := NewAteletDialer(nil, newTestAteletIndexer(t), "", "")
d := NewAteletDialer(nil, newTestAteletIndexer(t), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "")
if _, err := d.DialForAteletOnNode("node1"); !errors.Is(err, ErrNoAteletOnNode) {
t.Fatalf("DialForAteletOnNode = %v, want ErrNoAteletOnNode", err)
}
Expand All @@ -390,7 +390,7 @@ func TestDialForAteletOnNode(t *testing.T) {
d := NewAteletDialer(nil, newTestAteletIndexer(t,
ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"),
ateletPod("atelet-2", "uid-2", "node1", "10.0.0.2"),
), "", "")
), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "")
_, err := d.DialForAteletOnNode("node1")
if err == nil || errors.Is(err, ErrNoAteletOnNode) {
t.Fatalf("DialForAteletOnNode = %v, want a non-ErrNoAteletOnNode error", err)
Expand All @@ -400,7 +400,7 @@ func TestDialForAteletOnNode(t *testing.T) {
t.Run("dials and caches the node's atelet", func(t *testing.T) {
d := NewAteletDialer(nil, newTestAteletIndexer(t,
ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"),
), "", "")
), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "")
var credsUID string
d.dialCredentials = func(expectedPodUID string) (credentials.TransportCredentials, error) {
credsUID = expectedPodUID
Expand All @@ -427,7 +427,7 @@ func TestDialForAteletOnNode(t *testing.T) {
d := NewAteletDialer(nil, newTestAteletIndexer(t,
ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"),
ateletPod("atelet-2", "uid-2", "node2", "10.0.0.2"),
), "", "", WithDialCredentials(func(string) (credentials.TransportCredentials, error) {
), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "", WithDialCredentials(func(string) (credentials.TransportCredentials, error) {
return insecure.NewCredentials(), nil
}))
d.ateletConns = newAteletConnCache(1)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu

// Dial the fake atelet over insecure transport instead of per-atelet mTLS,
// so DialForWorker's real lookup/dial/cache path is exercised under test.
dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), "", "",
dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "",
controlapi.WithDialCredentials(func(_ string) (credentials.TransportCredentials, error) {
return insecure.NewCredentials(), nil
}))
Expand Down
3 changes: 2 additions & 1 deletion cmd/ateapi/internal/controlapi/workflow_suspend_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest"
"github.com/agent-substrate/substrate/internal/installdefaults"
"github.com/agent-substrate/substrate/internal/resources"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"google.golang.org/grpc/codes"
Expand Down Expand Up @@ -218,7 +219,7 @@ func newDanglingDialer() *AteletDialer {
byNamespaceAndName: func(obj any) ([]string, error) { return nil, nil },
byNode: func(obj any) ([]string, error) { return nil, nil },
})
return NewAteletDialer(empty, empty, "", "")
return NewAteletDialer(empty, empty, installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), "", "")
}

func TestEnsureAteletSuspended_DanglingWorkerDoesNotRecordPhantomSnapshot(t *testing.T) {
Expand Down
Loading
Loading