Add option to skip subnets (#53)

Add a flag `--subnets-to-ignore` to skip subnets
from redirecting to linkerd proxy

The goal is to support running linkerd with dind containers

Fixes linkerd/linkerd2#6758

Signed-off-by: Michael Lin <[email protected]>

Author: michaellzc

cmd/root.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"net"
 	"os/exec"
 
 	log "github.com/sirupsen/logrus"
@@ -19,6 +20,7 @@ type RootOptions struct {
 	PortsToRedirect       []int
 	InboundPortsToIgnore  []string
 	OutboundPortsToIgnore []string
+	SubnetsToIgnore       []string
 	SimulateOnly          bool
 	NetNs                 string
 	UseWaitFlag           bool
@@ -35,6 +37,7 @@ func newRootOptions() *RootOptions {
 		PortsToRedirect:       make([]int, 0),
 		InboundPortsToIgnore:  make([]string, 0),
 		OutboundPortsToIgnore: make([]string, 0),
+		SubnetsToIgnore:       make([]string, 0),
 		SimulateOnly:          false,
 		NetNs:                 "",
 		UseWaitFlag:           false,
@@ -87,6 +90,7 @@ func NewRootCmd() *cobra.Command {
 	cmd.PersistentFlags().IntSliceVarP(&options.PortsToRedirect, "ports-to-redirect", "r", options.PortsToRedirect, "Port to redirect to proxy, if no port is specified then ALL ports are redirected")
 	cmd.PersistentFlags().StringSliceVar(&options.InboundPortsToIgnore, "inbound-ports-to-ignore", options.InboundPortsToIgnore, "Inbound ports and/or port ranges (inclusive) to ignore and not redirect to proxy. This has higher precedence than any other parameters.")
 	cmd.PersistentFlags().StringSliceVar(&options.OutboundPortsToIgnore, "outbound-ports-to-ignore", options.OutboundPortsToIgnore, "Outbound ports and/or port ranges (inclusive) to ignore and not redirect to proxy. This has higher precedence than any other parameters.")
+	cmd.PersistentFlags().StringSliceVar(&options.SubnetsToIgnore, "subnets-to-ignore", options.SubnetsToIgnore, "Subnets to ignore and not redirect to proxy. This has higher precedence than any other parameters.")
 	cmd.PersistentFlags().BoolVar(&options.SimulateOnly, "simulate", options.SimulateOnly, "Don't execute any command, just print what would be executed")
 	cmd.PersistentFlags().StringVar(&options.NetNs, "netns", options.NetNs, "Optional network namespace in which to run the iptables commands")
 	cmd.PersistentFlags().BoolVarP(&options.UseWaitFlag, "use-wait-flag", "w", options.UseWaitFlag, "Appends the \"-w\" flag to the iptables commands")
@@ -106,13 +110,21 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu
 		return nil, fmt.Errorf("--outgoing-proxy-port must be a valid TCP port number")
 	}
 
+	for _, subnet := range options.SubnetsToIgnore {
+		_, _, err := net.ParseCIDR(subnet)
+		if err != nil {
+			return nil, fmt.Errorf("%s is not a valid CIDR address", subnet)
+		}
+	}
+
 	firewallConfiguration := &iptables.FirewallConfiguration{
 		ProxyInboundPort:       options.IncomingProxyPort,
 		ProxyOutgoingPort:      options.OutgoingProxyPort,
 		ProxyUID:               options.ProxyUserID,
 		PortsToRedirectInbound: options.PortsToRedirect,
 		InboundPortsToIgnore:   options.InboundPortsToIgnore,
 		OutboundPortsToIgnore:  options.OutboundPortsToIgnore,
+		SubnetsToIgnore:        options.SubnetsToIgnore,
 		SimulateOnly:           options.SimulateOnly,
 		NetNs:                  options.NetNs,
 		UseWaitFlag:            options.UseWaitFlag,

cmd/root_test.go

@@ -17,6 +17,7 @@ func TestBuildFirewallConfiguration(t *testing.T) {
 			PortsToRedirectInbound: make([]int, 0),
 			InboundPortsToIgnore:   make([]string, 0),
 			OutboundPortsToIgnore:  make([]string, 0),
+			SubnetsToIgnore:        make([]string, 0),
 			ProxyInboundPort:       expectedIncomingProxyPort,
 			ProxyOutgoingPort:      expectedOutgoingProxyPort,
 			ProxyUID:               expectedProxyUserID,
@@ -72,6 +73,12 @@ func TestBuildFirewallConfiguration(t *testing.T) {
 				},
 				errorMessage: "--outgoing-proxy-port must be a valid TCP port number",
 			},
+			{
+				options: &RootOptions{
+					SubnetsToIgnore: []string{"1.1.1.1/24", "0.0.0.0"},
+				},
+				errorMessage: "0.0.0.0 is not a valid CIDR address",
+			},
 		} {
 			_, err := BuildFirewallConfiguration(tt.options)
 			if err == nil {

integration_test/iptables/http_test.go

@@ -177,6 +177,30 @@ func TestPodMakesOutboundConnection(t *testing.T) {
 	})
 }
 
+func TestPodWithSomeSubnetsIgnored(t *testing.T) {
+	t.Parallel()
+
+	podIgnoredSomeSubnetsIP := os.Getenv("POD_IGNORES_SUBNETS_IP")
+
+	t.Run("connecting to a not-a-proxy-container should bypass proxy container", func(t *testing.T) {
+		response := expectSuccessfulGetRequestTo(t, podIgnoredSomeSubnetsIP, notTheProxyContainerPort)
+
+		expectedResponse := fmt.Sprintf("pod-ignores-subnets:%s", notTheProxyContainerPort)
+		if !strings.Contains(response, expectedResponse) {
+			t.Fatalf("Expected response to be bypassed, expected %s but it was %s", expectedResponse, response)
+		}
+	})
+
+	t.Run("connecting directly to the proxy container pod should still work", func(t *testing.T) {
+		response := expectSuccessfulGetRequestTo(t, podIgnoredSomeSubnetsIP, proxyContainerPort)
+
+		expectedResponse := "proxy"
+		if !strings.Contains(response, expectedResponse) {
+			t.Fatalf("Expected response from the proxy container, expected %s but it was %s", expectedResponse, response)
+		}
+	})
+}
+
 func makeCallFromContainerToAnother(t *testing.T, fromPodNamed string, fromContainerAtPort string, podIWantToReachName string, containerPortIWantToReach string) string {
 	downstreamURL := fmt.Sprintf("http://%s:%s", podIWantToReachName, containerPortIWantToReach)
 

integration_test/iptables/iptablestest-lab.yaml

@@ -274,3 +274,56 @@ spec:
   volumes:
   - emptyDir: {}
     name: linkerd-proxy-init-xtables-lock
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-ignores-subnets
+  labels:
+    app: pod-ignores-subnets
+spec:
+  containers:
+  - name: proxy-stub
+    image: ghcr.io/linkerd/iptables-tester:v1
+    env:
+    - name: PORT
+      value: "8080"
+    - name: AM_I_THE_PROXY
+      value: "yes"
+    command: ["go", "run", "/go/test_service/test_service.go"]
+    ports:
+    - name: http
+      containerPort: 8080
+    securityContext:
+      privileged: false
+      runAsUser: 2102
+  - name: other-container
+    image: ghcr.io/linkerd/iptables-tester:v1
+    env:
+    - name: PORT
+      value: "9090"
+    command: ["go", "run", "/go/test_service/test_service.go"]
+    ports:
+    - name: http
+      containerPort: 9090
+  initContainers:
+  - name: linkerd-init
+    image: ghcr.io/linkerd/proxy-init:latest
+    imagePullPolicy: Never
+    args: ["-p", "8080",  "-o", "8080", "-u", "2102", "--subnets-to-ignore", "0.0.0.0/0"]
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        add:
+        - NET_ADMIN
+        - NET_RAW
+      privileged: false
+      readOnlyRootFilesystem: true
+      runAsNonRoot: false
+      runAsUser: 0
+    volumeMounts:
+    - mountPath: /run
+      name: linkerd-proxy-init-xtables-lock
+  volumes:
+  - emptyDir: {}
+    name: linkerd-proxy-init-xtables-lock

integration_test/run_tests.sh

@@ -60,6 +60,9 @@ log "POD_REDIRECTS_WHITELISTED_IP=${POD_REDIRECTS_WHITELISTED_IP}"
 POD_DOEST_REDIRECT_BLACKLISTED_IP=$(get_ip_for_pod 'pod-doesnt-redirect-blacklisted')
 log "POD_DOEST_REDIRECT_BLACKLISTED_IP=${POD_DOEST_REDIRECT_BLACKLISTED_IP}"
 
+POD_IGNORES_SUBNETS_IP=$(get_ip_for_pod 'pod-ignores-subnets')
+log "POD_IGNORES_SUBNETS_IP=${POD_IGNORES_SUBNETS_IP}"
+
 header 'Running tester...'
 cat <<EOF | kubectl create -f -
 apiVersion: batch/v1
@@ -85,6 +88,8 @@ spec:
             value: ${POD_REDIRECTS_WHITELISTED_IP}
           - name: POD_DOEST_REDIRECT_BLACKLISTED_IP
             value: ${POD_DOEST_REDIRECT_BLACKLISTED_IP}
+          - name: POD_IGNORES_SUBNETS_IP
+            value: ${POD_IGNORES_SUBNETS_IP}
       restartPolicy: Never
 EOF
 

iptables/iptables.go

@@ -48,6 +48,7 @@ type FirewallConfiguration struct {
 	PortsToRedirectInbound []int
 	InboundPortsToIgnore   []string
 	OutboundPortsToIgnore  []string
+	SubnetsToIgnore        []string
 	ProxyInboundPort       int
 	ProxyOutgoingPort      int
 	ProxyUID               int
@@ -133,6 +134,7 @@ func addOutgoingTrafficRules(commands []*exec.Cmd, firewallConfiguration Firewal
 func addIncomingTrafficRules(commands []*exec.Cmd, firewallConfiguration FirewallConfiguration) []*exec.Cmd {
 	commands = append(commands, makeCreateNewChain(redirectChainName, "redirect-common-chain"))
 	commands = addRulesForIgnoredPorts(firewallConfiguration.InboundPortsToIgnore, redirectChainName, commands)
+	commands = addRulesForIgnoredSubnets(firewallConfiguration.SubnetsToIgnore, redirectChainName, commands)
 	commands = addRulesForInboundPortRedirect(firewallConfiguration, redirectChainName, commands)
 
 	// Redirect all remaining inbound traffic to the proxy.
@@ -179,6 +181,15 @@ func addRulesForIgnoredPorts(portsToIgnore []string, chainName string, commands
 	return commands
 }
 
+func addRulesForIgnoredSubnets(subnetsToIgnore []string, chainName string, commands []*exec.Cmd) []*exec.Cmd {
+	for _, subnet := range subnetsToIgnore {
+		log.Infof("Will ignore subnet %s on chain %s", subnet, chainName)
+
+		commands = append(commands, makeIgnoreSubnet(chainName, subnet, fmt.Sprintf("ignore-subnet-%s", subnet)))
+	}
+	return commands
+}
+
 func makeMultiportDestinations(portsToIgnore []string) [][]string {
 	destinationSlices := make([][]string, 0)
 	destinationPortCount := 0
@@ -297,6 +308,17 @@ func makeIgnorePorts(chainName string, destinations []string, comment string) *e
 		"--comment", formatComment(comment))
 }
 
+func makeIgnoreSubnet(chainName string, subnet string, comment string) *exec.Cmd {
+	return exec.Command("iptables",
+		"-t", "nat",
+		"-A", chainName,
+		"-p", "all",
+		"-j", "RETURN",
+		"-s", subnet,
+		"-m", "comment",
+		"--comment", formatComment(comment))
+}
+
 func makeIgnoreLoopback(chainName string, comment string) *exec.Cmd {
 	return exec.Command("iptables",
 		"-t", "nat",