From 8684213e12b2555c8b7699d1b6ec312639c7f5fd Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:37:31 +0000 Subject: [PATCH 1/3] Secure mediated VM egress --- lib/egressproxy/README.md | 4 +- lib/egressproxy/dial.go | 70 ++++++++++++++++++ lib/egressproxy/dial_test.go | 71 +++++++++++++++++++ lib/egressproxy/enforce_linux.go | 53 ++++++++------ lib/egressproxy/enforce_linux_test.go | 30 ++++++++ lib/egressproxy/enforce_other.go | 2 +- lib/egressproxy/service.go | 20 +++++- lib/egressproxy/service_test.go | 16 ++++- lib/egressproxy/types.go | 4 +- lib/instances/egress_proxy.go | 1 - .../egress_proxy_integration_test.go | 1 + 11 files changed, 242 insertions(+), 30 deletions(-) create mode 100644 lib/egressproxy/dial.go create mode 100644 lib/egressproxy/dial_test.go create mode 100644 lib/egressproxy/enforce_linux_test.go diff --git a/lib/egressproxy/README.md b/lib/egressproxy/README.md index 0796ef4a6..7267c489e 100644 --- a/lib/egressproxy/README.md +++ b/lib/egressproxy/README.md @@ -63,7 +63,9 @@ Operationally, this is intended for key rotation, revocation/reissue flows, and - Real secret values are persisted in the normal instance `env` metadata, which is already host-side state. - TLS interception requires guest trust of the proxy CA; hypeman installs this CA in the guest when proxy mode is enabled. -- Egress enforcement is applied per instance TAP device and removed when the instance stops/standbys/deletes. +- Proxy requests are accepted only from source addresses registered to running instances. +- Upstream resolution rejects private, loopback, link-local, carrier-grade NAT, documentation, benchmark, multicast, and reserved addresses. +- Egress enforcement matches each instance source address and is removed when the instance stops/standbys/deletes. - Enforcement intentionally targets TCP egress only. DNS/other non-TCP traffic is not rewritten and is not blocked by `all` mode. ## Limits of enforcement diff --git a/lib/egressproxy/dial.go b/lib/egressproxy/dial.go new file mode 100644 index 000000000..cdf349380 --- /dev/null +++ b/lib/egressproxy/dial.go @@ -0,0 +1,70 @@ +package egressproxy + +import ( + "context" + "fmt" + "net" + "net/netip" +) + +type ipResolver interface { + LookupNetIP(context.Context, string, string) ([]netip.Addr, error) +} + +type dialContextFunc func(context.Context, string, string) (net.Conn, error) + +var blockedEgressPrefixes = []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/8"), + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("192.0.0.0/24"), + netip.MustParsePrefix("192.0.2.0/24"), + netip.MustParsePrefix("198.18.0.0/15"), + netip.MustParsePrefix("198.51.100.0/24"), + netip.MustParsePrefix("203.0.113.0/24"), + netip.MustParsePrefix("240.0.0.0/4"), + netip.MustParsePrefix("2001:db8::/32"), +} + +func publicDialContext(resolver ipResolver, dial dialContextFunc) dialContextFunc { + return func(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + addresses := make([]netip.Addr, 0, 1) + if parsed, parseErr := netip.ParseAddr(host); parseErr == nil { + addresses = append(addresses, parsed) + } else { + addresses, err = resolver.LookupNetIP(ctx, "ip", host) + if err != nil { + return nil, err + } + } + if len(addresses) == 0 { + return nil, fmt.Errorf("egress destination did not resolve") + } + selected := addresses[0] + for _, candidate := range addresses { + if !isPublicEgressIP(candidate) { + return nil, fmt.Errorf("egress destination resolves to a blocked address") + } + if candidate.Is4() { + selected = candidate + } + } + return dial(ctx, network, net.JoinHostPort(selected.String(), port)) + } +} + +func isPublicEgressIP(address netip.Addr) bool { + address = address.Unmap() + if !address.IsValid() || !address.IsGlobalUnicast() || address.IsPrivate() || address.IsLoopback() || address.IsLinkLocalUnicast() || address.IsLinkLocalMulticast() || address.IsMulticast() || address.IsUnspecified() { + return false + } + for _, prefix := range blockedEgressPrefixes { + if prefix.Contains(address) { + return false + } + } + return true +} diff --git a/lib/egressproxy/dial_test.go b/lib/egressproxy/dial_test.go new file mode 100644 index 000000000..8e171de78 --- /dev/null +++ b/lib/egressproxy/dial_test.go @@ -0,0 +1,71 @@ +package egressproxy + +import ( + "context" + "net" + "net/netip" + "testing" + + "github.com/stretchr/testify/require" +) + +type fixedResolver map[string][]netip.Addr + +func (r fixedResolver) LookupNetIP(_ context.Context, _, host string) ([]netip.Addr, error) { + return r[host], nil +} + +func TestIsPublicEgressIP(t *testing.T) { + t.Parallel() + + tests := map[string]bool{ + "8.8.8.8": true, + "2606:4700:4700::1111": true, + "10.0.0.1": false, + "100.64.0.1": false, + "127.0.0.1": false, + "169.254.169.254": false, + "192.0.2.1": false, + "198.18.0.1": false, + "224.0.0.1": false, + "::1": false, + "fc00::1": false, + "fe80::1": false, + "2001:db8::1": false, + } + for address, want := range tests { + t.Run(address, func(t *testing.T) { + require.Equal(t, want, isPublicEgressIP(netip.MustParseAddr(address))) + }) + } +} + +func TestPublicDialContext(t *testing.T) { + t.Parallel() + + resolver := fixedResolver{ + "public.example": {netip.MustParseAddr("2606:4700:4700::1111"), netip.MustParseAddr("8.8.8.8")}, + "private.example": {netip.MustParseAddr("10.0.0.1")}, + "mixed.example": {netip.MustParseAddr("8.8.8.8"), netip.MustParseAddr("127.0.0.1")}, + } + var dialed string + dial := publicDialContext(resolver, func(_ context.Context, _, address string) (net.Conn, error) { + dialed = address + client, server := net.Pipe() + server.Close() + return client, nil + }) + + connection, err := dial(context.Background(), "tcp", "public.example:443") + require.NoError(t, err) + require.Equal(t, "8.8.8.8:443", dialed) + require.NoError(t, connection.Close()) + + for _, address := range []string{"private.example:443", "mixed.example:443", "127.0.0.1:443"} { + dialed = "" + connection, err := dial(context.Background(), "tcp", address) + require.Error(t, err) + require.Nil(t, connection) + require.Empty(t, dialed) + } +} diff --git a/lib/egressproxy/enforce_linux.go b/lib/egressproxy/enforce_linux.go index 898f84758..f03880310 100644 --- a/lib/egressproxy/enforce_linux.go +++ b/lib/egressproxy/enforce_linux.go @@ -4,6 +4,7 @@ package egressproxy import ( "fmt" + "net" "os/exec" "strings" "unicode" @@ -15,8 +16,8 @@ const ( enforcementSuffixAllTCP = "all-tcp" ) -func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { - if instanceID == "" || tapDevice == "" || gatewayIP == "" || proxyPort <= 0 { +func applyEgressEnforcement(instanceID, sourceIP, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { + if instanceID == "" || net.ParseIP(sourceIP).To4() == nil || net.ParseIP(gatewayIP).To4() == nil || proxyPort <= 0 { return fmt.Errorf("invalid egress enforcement inputs") } @@ -30,16 +31,16 @@ func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, proxyPort i _ = removeRuleByComment(commentAllTCP) if blockAllTCPEgress { - if err := insertRejectAllTCPRule(tapDevice, gatewayIP, commentAllTCP); err != nil { + if err := insertRejectAllTCPRule(sourceIP, commentAllTCP); err != nil { return fmt.Errorf("insert all-tcp egress enforcement: %w", err) } return nil } - if err := insertRejectRule(tapDevice, gatewayIP, 80, comment80); err != nil { + if err := insertRejectRule(sourceIP, 80, comment80); err != nil { return fmt.Errorf("insert port 80 egress enforcement: %w", err) } - if err := insertRejectRule(tapDevice, gatewayIP, 443, comment443); err != nil { + if err := insertRejectRule(sourceIP, 443, comment443); err != nil { _ = removeRuleByComment(comment80) return fmt.Errorf("insert port 443 egress enforcement: %w", err) } @@ -60,35 +61,41 @@ func removeEgressEnforcement(instanceID string) error { return nil } -func insertRejectRule(tapDevice, gatewayIP string, port int, comment string) error { - cmd := exec.Command( - "iptables", "-I", "FORWARD", "1", - "-i", tapDevice, +func insertRejectRule(sourceIP string, port int, comment string) error { + cmd := exec.Command("iptables", rejectRuleArgs(sourceIP, port, comment)...) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + +func insertRejectAllTCPRule(sourceIP, comment string) error { + cmd := exec.Command("iptables", rejectAllTCPRuleArgs(sourceIP, comment)...) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + +func rejectRuleArgs(sourceIP string, port int, comment string) []string { + return []string{ + "-I", "FORWARD", "1", + "-s", sourceIP, "-p", "tcp", "--dport", fmt.Sprintf("%d", port), - "!", "-d", gatewayIP, "-m", "comment", "--comment", comment, "-j", "REJECT", - ) - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out))) } - return nil } -func insertRejectAllTCPRule(tapDevice, gatewayIP, comment string) error { - cmd := exec.Command( - "iptables", "-I", "FORWARD", "1", - "-i", tapDevice, +func rejectAllTCPRuleArgs(sourceIP, comment string) []string { + return []string{ + "-I", "FORWARD", "1", + "-s", sourceIP, "-p", "tcp", - "!", "-d", gatewayIP, "-m", "comment", "--comment", comment, "-j", "REJECT", - ) - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out))) } - return nil } func removeRuleByComment(comment string) error { diff --git a/lib/egressproxy/enforce_linux_test.go b/lib/egressproxy/enforce_linux_test.go new file mode 100644 index 000000000..f2bb8f5ad --- /dev/null +++ b/lib/egressproxy/enforce_linux_test.go @@ -0,0 +1,30 @@ +//go:build linux + +package egressproxy + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRejectRuleArgsMatchSourceAddress(t *testing.T) { + t.Parallel() + + require.Equal(t, []string{ + "-I", "FORWARD", "1", + "-s", "10.102.1.2", + "-p", "tcp", + "--dport", "443", + "-m", "comment", "--comment", "hypeman-egress-instance-443", + "-j", "REJECT", + }, rejectRuleArgs("10.102.1.2", 443, "hypeman-egress-instance-443")) + + require.Equal(t, []string{ + "-I", "FORWARD", "1", + "-s", "10.102.1.2", + "-p", "tcp", + "-m", "comment", "--comment", "hypeman-egress-instance-all-tcp", + "-j", "REJECT", + }, rejectAllTCPRuleArgs("10.102.1.2", "hypeman-egress-instance-all-tcp")) +} diff --git a/lib/egressproxy/enforce_other.go b/lib/egressproxy/enforce_other.go index 3eb72c042..f122bba3c 100644 --- a/lib/egressproxy/enforce_other.go +++ b/lib/egressproxy/enforce_other.go @@ -2,7 +2,7 @@ package egressproxy -func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { +func applyEgressEnforcement(instanceID, sourceIP, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { _ = blockAllTCPEgress return nil } diff --git a/lib/egressproxy/service.go b/lib/egressproxy/service.go index acd6172c8..f48576846 100644 --- a/lib/egressproxy/service.go +++ b/lib/egressproxy/service.go @@ -83,9 +83,14 @@ func NewServiceWithOptions(dataDir string, listenPort int, opts ServiceOptions) return nil, err } + dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} + dialContext := publicDialContext(net.DefaultResolver, dialer.DialContext) + if opts.DialContext != nil { + dialContext = opts.DialContext + } transport := &http.Transport{ Proxy: nil, - DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + DialContext: dialContext, ForceAttemptHTTP2: false, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, @@ -212,7 +217,7 @@ func (s *Service) RegisterInstance(ctx context.Context, gatewayIP string, cfg In return GuestConfig{}, err } - if err := applyEgressEnforcement(cfg.InstanceID, cfg.TAPDevice, gatewayIP, s.listenPort, cfg.BlockAllTCPEgress); err != nil { + if err := applyEgressEnforcement(cfg.InstanceID, cfg.SourceIP, gatewayIP, s.listenPort, cfg.BlockAllTCPEgress); err != nil { log.WarnContext(ctx, "failed to apply egress proxy enforcement", "instance_id", cfg.InstanceID, "error", err) opErr = err return GuestConfig{}, err @@ -358,6 +363,10 @@ func (s *Service) proxyURLLocked() string { func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { sourceIP := sourceIPFromRemoteAddr(r.RemoteAddr) + if !s.isRegisteredSourceIP(sourceIP) { + http.Error(w, "proxy source is not registered", http.StatusForbidden) + return + } if r.Method == http.MethodConnect { s.handleConnect(w, r, sourceIP) return @@ -365,6 +374,13 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.handleHTTPProxyRequest(w, r, sourceIP, false) } +func (s *Service) isRegisteredSourceIP(sourceIP string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + _, ok := s.policiesBySourceIP[sourceIP] + return ok +} + func (s *Service) handleHTTPProxyRequest(w http.ResponseWriter, r *http.Request, sourceIP string, insideTunnel bool) { protocol := "http" if insideTunnel { diff --git a/lib/egressproxy/service_test.go b/lib/egressproxy/service_test.go index 84aa487fa..f05a5ba00 100644 --- a/lib/egressproxy/service_test.go +++ b/lib/egressproxy/service_test.go @@ -93,6 +93,19 @@ func TestApplyHeaderInjectionsHTTPSOnlyAndDomainGated(t *testing.T) { require.Equal(t, "Bearer mock-OUTBOUND_OPENAI_KEY", httpAllowedDomain.Get("Authorization")) } +func TestServeHTTPRejectsUnregisteredSource(t *testing.T) { + t.Parallel() + + svc := &Service{policiesBySourceIP: map[string]sourcePolicy{}} + req := httptest.NewRequest(http.MethodGet, "http://api.example.com/", nil) + req.RemoteAddr = "10.0.0.3:1234" + rec := httptest.NewRecorder() + + svc.ServeHTTP(rec, req) + + require.Equal(t, http.StatusForbidden, rec.Code) +} + func TestHandleHTTPProxyRequest_DoesNotLeakUpstreamErrorDetails(t *testing.T) { t.Parallel() @@ -103,11 +116,12 @@ func TestHandleHTTPProxyRequest_DoesNotLeakUpstreamErrorDetails(t *testing.T) { return nil, sentinelErr }, }, - policiesBySourceIP: map[string]sourcePolicy{}, + policiesBySourceIP: map[string]sourcePolicy{"10.0.0.2": {}}, sourceIPByInstance: map[string]string{}, } req := httptest.NewRequest(http.MethodGet, "http://api.example.com/v1/chat/completions", nil) + req.RemoteAddr = "10.0.0.2:1234" rec := httptest.NewRecorder() svc.ServeHTTP(rec, req) diff --git a/lib/egressproxy/types.go b/lib/egressproxy/types.go index 3a2e59fa6..c1e847c73 100644 --- a/lib/egressproxy/types.go +++ b/lib/egressproxy/types.go @@ -1,8 +1,10 @@ package egressproxy import ( + "context" "errors" "log/slog" + "net" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" @@ -20,7 +22,6 @@ var ( type InstanceConfig struct { InstanceID string SourceIP string - TAPDevice string BlockAllTCPEgress bool HeaderInjectRules []HeaderInjectRuleConfig } @@ -35,6 +36,7 @@ type HeaderInjectRuleConfig struct { // ServiceOptions customizes service construction (primarily for tests). type ServiceOptions struct { AdditionalRootCAPEM []string + DialContext func(context.Context, string, string) (net.Conn, error) Logger *slog.Logger Meter metric.Meter Tracer trace.Tracer diff --git a/lib/instances/egress_proxy.go b/lib/instances/egress_proxy.go index 82a3c4e1b..751a7e517 100644 --- a/lib/instances/egress_proxy.go +++ b/lib/instances/egress_proxy.go @@ -293,7 +293,6 @@ func (m *manager) maybeRegisterEgressProxy(ctx context.Context, stored *StoredMe guestCfg, err := svc.RegisterInstance(ctx, netConfig.Gateway, egressproxy.InstanceConfig{ InstanceID: stored.Id, SourceIP: netConfig.IP, - TAPDevice: netConfig.TAPDevice, BlockAllTCPEgress: stored.NetworkEgress.EnforcementMode != EgressEnforcementModeHTTPHTTPSOnly, HeaderInjectRules: rules, }) diff --git a/lib/instances/egress_proxy_integration_test.go b/lib/instances/egress_proxy_integration_test.go index e494743ad..2576461d3 100644 --- a/lib/instances/egress_proxy_integration_test.go +++ b/lib/instances/egress_proxy_integration_test.go @@ -36,6 +36,7 @@ func TestEgressProxyRewritesHTTPSHeaders(t *testing.T) { caPEM, cert := mustGenerateTLSChain(t, []string{"localhost"}) manager.egressProxyServiceOptions = egressproxy.ServiceOptions{ AdditionalRootCAPEM: []string{caPEM}, + DialContext: (&net.Dialer{}).DialContext, } target := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From d6cf2e327edd983002fb416195d2315ea0d71778 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:09:56 +0000 Subject: [PATCH 2/3] Block direct guest access to host services --- lib/egressproxy/enforce_linux.go | 116 ++++++++++++++++++-------- lib/egressproxy/enforce_linux_test.go | 21 +++++ 2 files changed, 104 insertions(+), 33 deletions(-) diff --git a/lib/egressproxy/enforce_linux.go b/lib/egressproxy/enforce_linux.go index f03880310..5ba621413 100644 --- a/lib/egressproxy/enforce_linux.go +++ b/lib/egressproxy/enforce_linux.go @@ -11,9 +11,12 @@ import ( ) const ( - enforcementSuffixPort80 = "80" - enforcementSuffixPort443 = "443" - enforcementSuffixAllTCP = "all-tcp" + enforcementSuffixPort80 = "80" + enforcementSuffixPort443 = "443" + enforcementSuffixAllTCP = "all-tcp" + enforcementSuffixHostTCP = "host-tcp" + enforcementSuffixHostDNS = "host-dns" + enforcementSuffixHostProxy = "host-proxy" ) func applyEgressEnforcement(instanceID, sourceIP, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { @@ -21,56 +24,78 @@ func applyEgressEnforcement(instanceID, sourceIP, gatewayIP string, proxyPort in return fmt.Errorf("invalid egress enforcement inputs") } - comment80 := enforcementComment(instanceID, enforcementSuffixPort80) - comment443 := enforcementComment(instanceID, enforcementSuffixPort443) - commentAllTCP := enforcementComment(instanceID, enforcementSuffixAllTCP) - - // Clean old rules first so updates are idempotent across restarts and mode changes. - _ = removeRuleByComment(comment80) - _ = removeRuleByComment(comment443) - _ = removeRuleByComment(commentAllTCP) - + removeEgressRules(instanceID) if blockAllTCPEgress { - if err := insertRejectAllTCPRule(sourceIP, commentAllTCP); err != nil { + if err := insertRejectAllTCPRule(sourceIP, enforcementComment(instanceID, enforcementSuffixAllTCP)); err != nil { return fmt.Errorf("insert all-tcp egress enforcement: %w", err) } + if err := insertRejectHostTCPRule(sourceIP, enforcementComment(instanceID, enforcementSuffixHostTCP)); err != nil { + removeEgressRules(instanceID) + return fmt.Errorf("insert host-tcp egress enforcement: %w", err) + } + if err := insertAcceptHostTCPRule(sourceIP, 53, enforcementComment(instanceID, enforcementSuffixHostDNS)); err != nil { + removeEgressRules(instanceID) + return fmt.Errorf("insert host DNS allowance: %w", err) + } + if err := insertAcceptHostTCPRule(sourceIP, proxyPort, enforcementComment(instanceID, enforcementSuffixHostProxy)); err != nil { + removeEgressRules(instanceID) + return fmt.Errorf("insert host proxy allowance: %w", err) + } return nil } + comment80 := enforcementComment(instanceID, enforcementSuffixPort80) if err := insertRejectRule(sourceIP, 80, comment80); err != nil { return fmt.Errorf("insert port 80 egress enforcement: %w", err) } - if err := insertRejectRule(sourceIP, 443, comment443); err != nil { - _ = removeRuleByComment(comment80) + if err := insertRejectRule(sourceIP, 443, enforcementComment(instanceID, enforcementSuffixPort443)); err != nil { + removeEgressRules(instanceID) return fmt.Errorf("insert port 443 egress enforcement: %w", err) } - return nil } func removeEgressEnforcement(instanceID string) error { - if instanceID == "" { - return nil + if instanceID != "" { + removeEgressRules(instanceID) } - comment80 := enforcementComment(instanceID, enforcementSuffixPort80) - comment443 := enforcementComment(instanceID, enforcementSuffixPort443) - commentAllTCP := enforcementComment(instanceID, enforcementSuffixAllTCP) - _ = removeRuleByComment(comment80) - _ = removeRuleByComment(comment443) - _ = removeRuleByComment(commentAllTCP) return nil } -func insertRejectRule(sourceIP string, port int, comment string) error { - cmd := exec.Command("iptables", rejectRuleArgs(sourceIP, port, comment)...) - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out))) +func removeEgressRules(instanceID string) { + for _, rule := range []struct { + chain string + suffix string + }{ + {chain: "FORWARD", suffix: enforcementSuffixPort80}, + {chain: "FORWARD", suffix: enforcementSuffixPort443}, + {chain: "FORWARD", suffix: enforcementSuffixAllTCP}, + {chain: "INPUT", suffix: enforcementSuffixHostTCP}, + {chain: "INPUT", suffix: enforcementSuffixHostDNS}, + {chain: "INPUT", suffix: enforcementSuffixHostProxy}, + } { + _ = removeRuleByComment(rule.chain, enforcementComment(instanceID, rule.suffix)) } - return nil +} + +func insertRejectRule(sourceIP string, port int, comment string) error { + return insertIptablesRule(rejectRuleArgs(sourceIP, port, comment)) } func insertRejectAllTCPRule(sourceIP, comment string) error { - cmd := exec.Command("iptables", rejectAllTCPRuleArgs(sourceIP, comment)...) + return insertIptablesRule(rejectAllTCPRuleArgs(sourceIP, comment)) +} + +func insertRejectHostTCPRule(sourceIP, comment string) error { + return insertIptablesRule(rejectHostTCPRuleArgs(sourceIP, comment)) +} + +func insertAcceptHostTCPRule(sourceIP string, port int, comment string) error { + return insertIptablesRule(acceptHostTCPRuleArgs(sourceIP, port, comment)) +} + +func insertIptablesRule(arguments []string) error { + cmd := exec.Command("iptables", arguments...) if out, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out))) } @@ -83,6 +108,7 @@ func rejectRuleArgs(sourceIP string, port int, comment string) []string { "-s", sourceIP, "-p", "tcp", "--dport", fmt.Sprintf("%d", port), + "-m", "conntrack", "--ctstate", "NEW", "-m", "comment", "--comment", comment, "-j", "REJECT", } @@ -93,13 +119,37 @@ func rejectAllTCPRuleArgs(sourceIP, comment string) []string { "-I", "FORWARD", "1", "-s", sourceIP, "-p", "tcp", + "-m", "conntrack", "--ctstate", "NEW", "-m", "comment", "--comment", comment, "-j", "REJECT", } } -func removeRuleByComment(comment string) error { - listCmd := exec.Command("iptables", "-L", "FORWARD", "--line-numbers", "-n") +func rejectHostTCPRuleArgs(sourceIP, comment string) []string { + return []string{ + "-I", "INPUT", "1", + "-s", sourceIP, + "-p", "tcp", + "-m", "conntrack", "--ctstate", "NEW", + "-m", "comment", "--comment", comment, + "-j", "REJECT", + } +} + +func acceptHostTCPRuleArgs(sourceIP string, port int, comment string) []string { + return []string{ + "-I", "INPUT", "1", + "-s", sourceIP, + "-p", "tcp", + "--dport", fmt.Sprintf("%d", port), + "-m", "conntrack", "--ctstate", "NEW", + "-m", "comment", "--comment", comment, + "-j", "ACCEPT", + } +} + +func removeRuleByComment(chain, comment string) error { + listCmd := exec.Command("iptables", "-L", chain, "--line-numbers", "-n") output, err := listCmd.Output() if err != nil { return err @@ -119,7 +169,7 @@ func removeRuleByComment(comment string) error { } for i := len(ruleNums) - 1; i >= 0; i-- { - delCmd := exec.Command("iptables", "-D", "FORWARD", ruleNums[i]) + delCmd := exec.Command("iptables", "-D", chain, ruleNums[i]) _ = delCmd.Run() } return nil diff --git a/lib/egressproxy/enforce_linux_test.go b/lib/egressproxy/enforce_linux_test.go index f2bb8f5ad..1bc3923df 100644 --- a/lib/egressproxy/enforce_linux_test.go +++ b/lib/egressproxy/enforce_linux_test.go @@ -16,6 +16,7 @@ func TestRejectRuleArgsMatchSourceAddress(t *testing.T) { "-s", "10.102.1.2", "-p", "tcp", "--dport", "443", + "-m", "conntrack", "--ctstate", "NEW", "-m", "comment", "--comment", "hypeman-egress-instance-443", "-j", "REJECT", }, rejectRuleArgs("10.102.1.2", 443, "hypeman-egress-instance-443")) @@ -24,7 +25,27 @@ func TestRejectRuleArgsMatchSourceAddress(t *testing.T) { "-I", "FORWARD", "1", "-s", "10.102.1.2", "-p", "tcp", + "-m", "conntrack", "--ctstate", "NEW", "-m", "comment", "--comment", "hypeman-egress-instance-all-tcp", "-j", "REJECT", }, rejectAllTCPRuleArgs("10.102.1.2", "hypeman-egress-instance-all-tcp")) + + require.Equal(t, []string{ + "-I", "INPUT", "1", + "-s", "10.102.1.2", + "-p", "tcp", + "-m", "conntrack", "--ctstate", "NEW", + "-m", "comment", "--comment", "hypeman-egress-instance-host-tcp", + "-j", "REJECT", + }, rejectHostTCPRuleArgs("10.102.1.2", "hypeman-egress-instance-host-tcp")) + + require.Equal(t, []string{ + "-I", "INPUT", "1", + "-s", "10.102.1.2", + "-p", "tcp", + "--dport", "18080", + "-m", "conntrack", "--ctstate", "NEW", + "-m", "comment", "--comment", "hypeman-egress-instance-host-proxy", + "-j", "ACCEPT", + }, acceptHostTCPRuleArgs("10.102.1.2", 18080, "hypeman-egress-instance-host-proxy")) } From e69387c18b701aae5ff0ddda07da472373a31035 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:11:45 +0000 Subject: [PATCH 3/3] Reject spoofed VM source addresses --- lib/egressproxy/enforce_linux.go | 41 +++++++++++++++++++++------ lib/egressproxy/enforce_linux_test.go | 8 ++++++ lib/egressproxy/enforce_other.go | 2 +- lib/egressproxy/service.go | 2 +- lib/egressproxy/types.go | 1 + lib/instances/egress_proxy.go | 1 + 6 files changed, 45 insertions(+), 10 deletions(-) diff --git a/lib/egressproxy/enforce_linux.go b/lib/egressproxy/enforce_linux.go index 5ba621413..656d2bdc0 100644 --- a/lib/egressproxy/enforce_linux.go +++ b/lib/egressproxy/enforce_linux.go @@ -11,20 +11,29 @@ import ( ) const ( - enforcementSuffixPort80 = "80" - enforcementSuffixPort443 = "443" - enforcementSuffixAllTCP = "all-tcp" - enforcementSuffixHostTCP = "host-tcp" - enforcementSuffixHostDNS = "host-dns" - enforcementSuffixHostProxy = "host-proxy" + enforcementSuffixPort80 = "80" + enforcementSuffixPort443 = "443" + enforcementSuffixAllTCP = "all-tcp" + enforcementSuffixHostTCP = "host-tcp" + enforcementSuffixHostDNS = "host-dns" + enforcementSuffixHostProxy = "host-proxy" + enforcementSuffixForwardSpoof = "forward-spoof" + enforcementSuffixHostSpoof = "host-spoof" ) -func applyEgressEnforcement(instanceID, sourceIP, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { - if instanceID == "" || net.ParseIP(sourceIP).To4() == nil || net.ParseIP(gatewayIP).To4() == nil || proxyPort <= 0 { +func applyEgressEnforcement(instanceID, sourceIP, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { + if instanceID == "" || net.ParseIP(sourceIP).To4() == nil || tapDevice == "" || net.ParseIP(gatewayIP).To4() == nil || proxyPort <= 0 { return fmt.Errorf("invalid egress enforcement inputs") } removeEgressRules(instanceID) + if err := insertAntiSpoofRule("FORWARD", sourceIP, tapDevice, enforcementComment(instanceID, enforcementSuffixForwardSpoof)); err != nil { + return fmt.Errorf("insert forwarded anti-spoof enforcement: %w", err) + } + if err := insertAntiSpoofRule("INPUT", sourceIP, tapDevice, enforcementComment(instanceID, enforcementSuffixHostSpoof)); err != nil { + removeEgressRules(instanceID) + return fmt.Errorf("insert host anti-spoof enforcement: %w", err) + } if blockAllTCPEgress { if err := insertRejectAllTCPRule(sourceIP, enforcementComment(instanceID, enforcementSuffixAllTCP)); err != nil { return fmt.Errorf("insert all-tcp egress enforcement: %w", err) @@ -73,6 +82,8 @@ func removeEgressRules(instanceID string) { {chain: "INPUT", suffix: enforcementSuffixHostTCP}, {chain: "INPUT", suffix: enforcementSuffixHostDNS}, {chain: "INPUT", suffix: enforcementSuffixHostProxy}, + {chain: "FORWARD", suffix: enforcementSuffixForwardSpoof}, + {chain: "INPUT", suffix: enforcementSuffixHostSpoof}, } { _ = removeRuleByComment(rule.chain, enforcementComment(instanceID, rule.suffix)) } @@ -94,6 +105,10 @@ func insertAcceptHostTCPRule(sourceIP string, port int, comment string) error { return insertIptablesRule(acceptHostTCPRuleArgs(sourceIP, port, comment)) } +func insertAntiSpoofRule(chain, sourceIP, tapDevice, comment string) error { + return insertIptablesRule(antiSpoofRuleArgs(chain, sourceIP, tapDevice, comment)) +} + func insertIptablesRule(arguments []string) error { cmd := exec.Command("iptables", arguments...) if out, err := cmd.CombinedOutput(); err != nil { @@ -148,6 +163,16 @@ func acceptHostTCPRuleArgs(sourceIP string, port int, comment string) []string { } } +func antiSpoofRuleArgs(chain, sourceIP, tapDevice, comment string) []string { + return []string{ + "-I", chain, "1", + "-m", "physdev", "--physdev-in", tapDevice, + "!", "-s", sourceIP, + "-m", "comment", "--comment", comment, + "-j", "DROP", + } +} + func removeRuleByComment(chain, comment string) error { listCmd := exec.Command("iptables", "-L", chain, "--line-numbers", "-n") output, err := listCmd.Output() diff --git a/lib/egressproxy/enforce_linux_test.go b/lib/egressproxy/enforce_linux_test.go index 1bc3923df..55fe0b163 100644 --- a/lib/egressproxy/enforce_linux_test.go +++ b/lib/egressproxy/enforce_linux_test.go @@ -48,4 +48,12 @@ func TestRejectRuleArgsMatchSourceAddress(t *testing.T) { "-m", "comment", "--comment", "hypeman-egress-instance-host-proxy", "-j", "ACCEPT", }, acceptHostTCPRuleArgs("10.102.1.2", 18080, "hypeman-egress-instance-host-proxy")) + + require.Equal(t, []string{ + "-I", "FORWARD", "1", + "-m", "physdev", "--physdev-in", "hype-instance", + "!", "-s", "10.102.1.2", + "-m", "comment", "--comment", "hypeman-egress-instance-forward-spoof", + "-j", "DROP", + }, antiSpoofRuleArgs("FORWARD", "10.102.1.2", "hype-instance", "hypeman-egress-instance-forward-spoof")) } diff --git a/lib/egressproxy/enforce_other.go b/lib/egressproxy/enforce_other.go index f122bba3c..1a5fb0967 100644 --- a/lib/egressproxy/enforce_other.go +++ b/lib/egressproxy/enforce_other.go @@ -2,7 +2,7 @@ package egressproxy -func applyEgressEnforcement(instanceID, sourceIP, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { +func applyEgressEnforcement(instanceID, sourceIP, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { _ = blockAllTCPEgress return nil } diff --git a/lib/egressproxy/service.go b/lib/egressproxy/service.go index f48576846..69bbf34d9 100644 --- a/lib/egressproxy/service.go +++ b/lib/egressproxy/service.go @@ -217,7 +217,7 @@ func (s *Service) RegisterInstance(ctx context.Context, gatewayIP string, cfg In return GuestConfig{}, err } - if err := applyEgressEnforcement(cfg.InstanceID, cfg.SourceIP, gatewayIP, s.listenPort, cfg.BlockAllTCPEgress); err != nil { + if err := applyEgressEnforcement(cfg.InstanceID, cfg.SourceIP, cfg.TAPDevice, gatewayIP, s.listenPort, cfg.BlockAllTCPEgress); err != nil { log.WarnContext(ctx, "failed to apply egress proxy enforcement", "instance_id", cfg.InstanceID, "error", err) opErr = err return GuestConfig{}, err diff --git a/lib/egressproxy/types.go b/lib/egressproxy/types.go index c1e847c73..c2130a887 100644 --- a/lib/egressproxy/types.go +++ b/lib/egressproxy/types.go @@ -22,6 +22,7 @@ var ( type InstanceConfig struct { InstanceID string SourceIP string + TAPDevice string BlockAllTCPEgress bool HeaderInjectRules []HeaderInjectRuleConfig } diff --git a/lib/instances/egress_proxy.go b/lib/instances/egress_proxy.go index 751a7e517..82a3c4e1b 100644 --- a/lib/instances/egress_proxy.go +++ b/lib/instances/egress_proxy.go @@ -293,6 +293,7 @@ func (m *manager) maybeRegisterEgressProxy(ctx context.Context, stored *StoredMe guestCfg, err := svc.RegisterInstance(ctx, netConfig.Gateway, egressproxy.InstanceConfig{ InstanceID: stored.Id, SourceIP: netConfig.IP, + TAPDevice: netConfig.TAPDevice, BlockAllTCPEgress: stored.NetworkEgress.EnforcementMode != EgressEnforcementModeHTTPHTTPSOnly, HeaderInjectRules: rules, })