Secure mediated VM egress

lib/egressproxy/README.md

@@ -63,7 +63,9 @@ Operationally, this is intended for key rotation, revocation/reissue flows, and
 
 - Real secret values are persisted in the normal instance `env` metadata, which is already host-side state.
 - TLS interception requires guest trust of the proxy CA; hypeman installs this CA in the guest when proxy mode is enabled.
-- Egress enforcement is applied per instance TAP device and removed when the instance stops/standbys/deletes.
+- Proxy requests are accepted only from source addresses registered to running instances.
+- Upstream resolution rejects private, loopback, link-local, carrier-grade NAT, documentation, benchmark, multicast, and reserved addresses.
+- Egress enforcement matches each instance source address and is removed when the instance stops/standbys/deletes.
 - Enforcement intentionally targets TCP egress only. DNS/other non-TCP traffic is not rewritten and is not blocked by `all` mode.
 
 ## Limits of enforcement

lib/egressproxy/dial.go

@@ -0,0 +1,70 @@
+package egressproxy
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+)
+
+type ipResolver interface {
+	LookupNetIP(context.Context, string, string) ([]netip.Addr, error)
+}
+
+type dialContextFunc func(context.Context, string, string) (net.Conn, error)
+
+var blockedEgressPrefixes = []netip.Prefix{
+	netip.MustParsePrefix("0.0.0.0/8"),
+	netip.MustParsePrefix("100.64.0.0/10"),
+	netip.MustParsePrefix("192.0.0.0/24"),
+	netip.MustParsePrefix("192.0.2.0/24"),
+	netip.MustParsePrefix("198.18.0.0/15"),
+	netip.MustParsePrefix("198.51.100.0/24"),
+	netip.MustParsePrefix("203.0.113.0/24"),
+	netip.MustParsePrefix("240.0.0.0/4"),
+	netip.MustParsePrefix("2001:db8::/32"),
+}
+
+func publicDialContext(resolver ipResolver, dial dialContextFunc) dialContextFunc {
+	return func(ctx context.Context, network, address string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(address)
+		if err != nil {
+			return nil, err
+		}
+		addresses := make([]netip.Addr, 0, 1)
+		if parsed, parseErr := netip.ParseAddr(host); parseErr == nil {
+			addresses = append(addresses, parsed)
+		} else {
+			addresses, err = resolver.LookupNetIP(ctx, "ip", host)
+			if err != nil {
+				return nil, err
+			}
+		}
+		if len(addresses) == 0 {
+			return nil, fmt.Errorf("egress destination did not resolve")
+		}
+		selected := addresses[0]
+		for _, candidate := range addresses {
+			if !isPublicEgressIP(candidate) {
+				return nil, fmt.Errorf("egress destination resolves to a blocked address")
+			}
+			if candidate.Is4() {
+				selected = candidate
+			}
+		}
+		return dial(ctx, network, net.JoinHostPort(selected.String(), port))
+	}
+}
+
+func isPublicEgressIP(address netip.Addr) bool {
+	address = address.Unmap()
+	if !address.IsValid() || !address.IsGlobalUnicast() || address.IsPrivate() || address.IsLoopback() || address.IsLinkLocalUnicast() || address.IsLinkLocalMulticast() || address.IsMulticast() || address.IsUnspecified() {
+		return false
+	}
+	for _, prefix := range blockedEgressPrefixes {
+		if prefix.Contains(address) {
+			return false
+		}
+	}
+	return true
+}

lib/egressproxy/dial_test.go

@@ -0,0 +1,71 @@
+package egressproxy
+
+import (
+	"context"
+	"net"
+	"net/netip"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type fixedResolver map[string][]netip.Addr
+
+func (r fixedResolver) LookupNetIP(_ context.Context, _, host string) ([]netip.Addr, error) {
+	return r[host], nil
+}
+
+func TestIsPublicEgressIP(t *testing.T) {
+	t.Parallel()
+
+	tests := map[string]bool{
+		"8.8.8.8":              true,
+		"2606:4700:4700::1111": true,
+		"10.0.0.1":             false,
+		"100.64.0.1":           false,
+		"127.0.0.1":            false,
+		"169.254.169.254":      false,
+		"192.0.2.1":            false,
+		"198.18.0.1":           false,
+		"224.0.0.1":            false,
+		"::1":                  false,
+		"fc00::1":              false,
+		"fe80::1":              false,
+		"2001:db8::1":          false,
+	}
+	for address, want := range tests {
+		t.Run(address, func(t *testing.T) {
+			require.Equal(t, want, isPublicEgressIP(netip.MustParseAddr(address)))
+		})
+	}
+}
+
+func TestPublicDialContext(t *testing.T) {
+	t.Parallel()
+
+	resolver := fixedResolver{
+		"public.example":  {netip.MustParseAddr("2606:4700:4700::1111"), netip.MustParseAddr("8.8.8.8")},
+		"private.example": {netip.MustParseAddr("10.0.0.1")},
+		"mixed.example":   {netip.MustParseAddr("8.8.8.8"), netip.MustParseAddr("127.0.0.1")},
+	}
+	var dialed string
+	dial := publicDialContext(resolver, func(_ context.Context, _, address string) (net.Conn, error) {
+		dialed = address
+		client, server := net.Pipe()
+		server.Close()
+		return client, nil
+	})
+
+	connection, err := dial(context.Background(), "tcp", "public.example:443")
+	require.NoError(t, err)
+	require.Equal(t, "8.8.8.8:443", dialed)
+	require.NoError(t, connection.Close())
+
+	for _, address := range []string{"private.example:443", "mixed.example:443", "127.0.0.1:443"} {
+		dialed = ""
+		connection, err := dial(context.Background(), "tcp", address)
+		require.Error(t, err)
+		require.Nil(t, connection)
+		require.Empty(t, dialed)
+	}
+}

lib/egressproxy/enforce_linux.go

@@ -4,95 +4,177 @@ package egressproxy
 
 import (
 	"fmt"
+	"net"
 	"os/exec"
 	"strings"
 	"unicode"
 )
 
 const (
-	enforcementSuffixPort80  = "80"
-	enforcementSuffixPort443 = "443"
-	enforcementSuffixAllTCP  = "all-tcp"
+	enforcementSuffixPort80       = "80"
+	enforcementSuffixPort443      = "443"
+	enforcementSuffixAllTCP       = "all-tcp"
+	enforcementSuffixHostTCP      = "host-tcp"
+	enforcementSuffixHostDNS      = "host-dns"
+	enforcementSuffixHostProxy    = "host-proxy"
+	enforcementSuffixForwardSpoof = "forward-spoof"
+	enforcementSuffixHostSpoof    = "host-spoof"
 )
 
-func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error {
-	if instanceID == "" || tapDevice == "" || gatewayIP == "" || proxyPort <= 0 {
+func applyEgressEnforcement(instanceID, sourceIP, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error {
+	if instanceID == "" || net.ParseIP(sourceIP).To4() == nil || tapDevice == "" || net.ParseIP(gatewayIP).To4() == nil || proxyPort <= 0 {
 		return fmt.Errorf("invalid egress enforcement inputs")
 	}
 
-	comment80 := enforcementComment(instanceID, enforcementSuffixPort80)
-	comment443 := enforcementComment(instanceID, enforcementSuffixPort443)
-	commentAllTCP := enforcementComment(instanceID, enforcementSuffixAllTCP)
-
-	// Clean old rules first so updates are idempotent across restarts and mode changes.
-	_ = removeRuleByComment(comment80)
-	_ = removeRuleByComment(comment443)
-	_ = removeRuleByComment(commentAllTCP)
-
+	removeEgressRules(instanceID)
+	if err := insertAntiSpoofRule("FORWARD", sourceIP, tapDevice, enforcementComment(instanceID, enforcementSuffixForwardSpoof)); err != nil {
+		return fmt.Errorf("insert forwarded anti-spoof enforcement: %w", err)
+	}
+	if err := insertAntiSpoofRule("INPUT", sourceIP, tapDevice, enforcementComment(instanceID, enforcementSuffixHostSpoof)); err != nil {
+		removeEgressRules(instanceID)
+		return fmt.Errorf("insert host anti-spoof enforcement: %w", err)
+	}
 	if blockAllTCPEgress {
-		if err := insertRejectAllTCPRule(tapDevice, gatewayIP, commentAllTCP); err != nil {
+		if err := insertRejectAllTCPRule(sourceIP, enforcementComment(instanceID, enforcementSuffixAllTCP)); err != nil {
 			return fmt.Errorf("insert all-tcp egress enforcement: %w", err)
 		}
+		if err := insertRejectHostTCPRule(sourceIP, enforcementComment(instanceID, enforcementSuffixHostTCP)); err != nil {
+			removeEgressRules(instanceID)
+			return fmt.Errorf("insert host-tcp egress enforcement: %w", err)
+		}
+		if err := insertAcceptHostTCPRule(sourceIP, 53, enforcementComment(instanceID, enforcementSuffixHostDNS)); err != nil {
+			removeEgressRules(instanceID)
+			return fmt.Errorf("insert host DNS allowance: %w", err)
+		}
+		if err := insertAcceptHostTCPRule(sourceIP, proxyPort, enforcementComment(instanceID, enforcementSuffixHostProxy)); err != nil {
+			removeEgressRules(instanceID)
+			return fmt.Errorf("insert host proxy allowance: %w", err)
+		}
 		return nil
 	}
 
-	if err := insertRejectRule(tapDevice, gatewayIP, 80, comment80); err != nil {
+	comment80 := enforcementComment(instanceID, enforcementSuffixPort80)
+	if err := insertRejectRule(sourceIP, 80, comment80); err != nil {
 		return fmt.Errorf("insert port 80 egress enforcement: %w", err)
 	}
-	if err := insertRejectRule(tapDevice, gatewayIP, 443, comment443); err != nil {
-		_ = removeRuleByComment(comment80)
+	if err := insertRejectRule(sourceIP, 443, enforcementComment(instanceID, enforcementSuffixPort443)); err != nil {
+		removeEgressRules(instanceID)
 		return fmt.Errorf("insert port 443 egress enforcement: %w", err)
 	}
-
 	return nil
 }
 
 func removeEgressEnforcement(instanceID string) error {
-	if instanceID == "" {
-		return nil
+	if instanceID != "" {
+		removeEgressRules(instanceID)
 	}
-	comment80 := enforcementComment(instanceID, enforcementSuffixPort80)
-	comment443 := enforcementComment(instanceID, enforcementSuffixPort443)
-	commentAllTCP := enforcementComment(instanceID, enforcementSuffixAllTCP)
-	_ = removeRuleByComment(comment80)
-	_ = removeRuleByComment(comment443)
-	_ = removeRuleByComment(commentAllTCP)
 	return nil
 }
 
-func insertRejectRule(tapDevice, gatewayIP string, port int, comment string) error {
-	cmd := exec.Command(
-		"iptables", "-I", "FORWARD", "1",
-		"-i", tapDevice,
+func removeEgressRules(instanceID string) {
+	for _, rule := range []struct {
+		chain  string
+		suffix string
+	}{
+		{chain: "FORWARD", suffix: enforcementSuffixPort80},
+		{chain: "FORWARD", suffix: enforcementSuffixPort443},
+		{chain: "FORWARD", suffix: enforcementSuffixAllTCP},
+		{chain: "INPUT", suffix: enforcementSuffixHostTCP},
+		{chain: "INPUT", suffix: enforcementSuffixHostDNS},
+		{chain: "INPUT", suffix: enforcementSuffixHostProxy},
+		{chain: "FORWARD", suffix: enforcementSuffixForwardSpoof},
+		{chain: "INPUT", suffix: enforcementSuffixHostSpoof},
+	} {
+		_ = removeRuleByComment(rule.chain, enforcementComment(instanceID, rule.suffix))
+	}
+}
+
+func insertRejectRule(sourceIP string, port int, comment string) error {
+	return insertIptablesRule(rejectRuleArgs(sourceIP, port, comment))
+}
+
+func insertRejectAllTCPRule(sourceIP, comment string) error {
+	return insertIptablesRule(rejectAllTCPRuleArgs(sourceIP, comment))
+}
+
+func insertRejectHostTCPRule(sourceIP, comment string) error {
+	return insertIptablesRule(rejectHostTCPRuleArgs(sourceIP, comment))
+}
+
+func insertAcceptHostTCPRule(sourceIP string, port int, comment string) error {
+	return insertIptablesRule(acceptHostTCPRuleArgs(sourceIP, port, comment))
+}
+
+func insertAntiSpoofRule(chain, sourceIP, tapDevice, comment string) error {
+	return insertIptablesRule(antiSpoofRuleArgs(chain, sourceIP, tapDevice, comment))
+}
+
+func insertIptablesRule(arguments []string) error {
+	cmd := exec.Command("iptables", arguments...)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out)))
+	}
+	return nil
+}
+
+func rejectRuleArgs(sourceIP string, port int, comment string) []string {
+	return []string{
+		"-I", "FORWARD", "1",
+		"-s", sourceIP,
 		"-p", "tcp",
 		"--dport", fmt.Sprintf("%d", port),
-		"!", "-d", gatewayIP,
+		"-m", "conntrack", "--ctstate", "NEW",
 		"-m", "comment", "--comment", comment,
 		"-j", "REJECT",
-	)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out)))
 	}
-	return nil
 }
 
-func insertRejectAllTCPRule(tapDevice, gatewayIP, comment string) error {
-	cmd := exec.Command(
-		"iptables", "-I", "FORWARD", "1",
-		"-i", tapDevice,
+func rejectAllTCPRuleArgs(sourceIP, comment string) []string {
+	return []string{
+		"-I", "FORWARD", "1",
+		"-s", sourceIP,
 		"-p", "tcp",
-		"!", "-d", gatewayIP,
+		"-m", "conntrack", "--ctstate", "NEW",
 		"-m", "comment", "--comment", comment,
 		"-j", "REJECT",
-	)
-	if out, err := cmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out)))
 	}
-	return nil
 }
 
-func removeRuleByComment(comment string) error {
-	listCmd := exec.Command("iptables", "-L", "FORWARD", "--line-numbers", "-n")
+func rejectHostTCPRuleArgs(sourceIP, comment string) []string {
+	return []string{
+		"-I", "INPUT", "1",
+		"-s", sourceIP,
+		"-p", "tcp",
+		"-m", "conntrack", "--ctstate", "NEW",
+		"-m", "comment", "--comment", comment,
+		"-j", "REJECT",
+	}
+}
+
+func acceptHostTCPRuleArgs(sourceIP string, port int, comment string) []string {
+	return []string{
+		"-I", "INPUT", "1",
+		"-s", sourceIP,
+		"-p", "tcp",
+		"--dport", fmt.Sprintf("%d", port),
+		"-m", "conntrack", "--ctstate", "NEW",
+		"-m", "comment", "--comment", comment,
+		"-j", "ACCEPT",
+	}
+}
+
+func antiSpoofRuleArgs(chain, sourceIP, tapDevice, comment string) []string {
+	return []string{
+		"-I", chain, "1",
+		"-m", "physdev", "--physdev-in", tapDevice,
+		"!", "-s", sourceIP,
+		"-m", "comment", "--comment", comment,
+		"-j", "DROP",
+	}
+}
+
+func removeRuleByComment(chain, comment string) error {
+	listCmd := exec.Command("iptables", "-L", chain, "--line-numbers", "-n")
 	output, err := listCmd.Output()
 	if err != nil {
 		return err
@@ -112,7 +194,7 @@ func removeRuleByComment(comment string) error {
 	}
 
 	for i := len(ruleNums) - 1; i >= 0; i-- {
-		delCmd := exec.Command("iptables", "-D", "FORWARD", ruleNums[i])
+		delCmd := exec.Command("iptables", "-D", chain, ruleNums[i])
 		_ = delCmd.Run()
 	}
 	return nil