ensure ingress-nginx node label is applied (#377)

Signed-off-by: Manabu McCloskey <[email protected]>

pkg/kind/cluster.go

@@ -19,6 +19,11 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+const (
+	ingressNginxNodeLabelKey   = "ingress-ready"
+	ingressNginxNodeLabelValue = "true"
+)
+
 var (
 	setupLog = log.Log.WithName("setup")
 )
@@ -241,28 +246,61 @@ func (c *Cluster) ensureCorrectConfig(in []byte) (kindv1alpha4.Cluster, error) {
 	if err != nil {
 		return kindv1alpha4.Cluster{}, fmt.Errorf("parsing kind config: %w", err)
 	}
-
+	// the port and ingress-nginx label must be on the same node to ensure nginx runs on the node with the right port.
 	appendNecessaryPort := true
+	appendIngressNodeLabel := true
+	// pick the first node for the ingress-nginx if we need to configure node port.
+	nodePosition := 0
+
+	if parsedCluster.Nodes == nil || len(parsedCluster.Nodes) == 0 {
+		return kindv1alpha4.Cluster{}, fmt.Errorf("provided kind config does not have the node field defined")
+	}
+
 nodes:
 	for i := range parsedCluster.Nodes {
 		node := parsedCluster.Nodes[i]
 		for _, pm := range node.ExtraPortMappings {
 			if strconv.Itoa(int(pm.HostPort)) == c.cfg.Port {
 				appendNecessaryPort = false
+				nodePosition = i
+				if node.Labels != nil {
+					v, ok := node.Labels[ingressNginxNodeLabelKey]
+					if ok && v == ingressNginxNodeLabelValue {
+						appendIngressNodeLabel = false
+					}
+				}
+				break nodes
+			}
+		}
+		if node.Labels != nil {
+			v, ok := node.Labels[ingressNginxNodeLabelKey]
+			if ok && v == ingressNginxNodeLabelValue {
+				appendIngressNodeLabel = false
+				nodePosition = i
 				break nodes
 			}
 		}
 	}
 
-	if appendNecessaryPort && len(parsedCluster.Nodes) != 0 {
+	if appendNecessaryPort {
 		hp, err := strconv.Atoi(c.cfg.Port)
 		if err != nil {
 			return kindv1alpha4.Cluster{}, fmt.Errorf("converting port, %s, to int: %w", c.cfg.Port, err)
 		}
 		// either "80" or "443". No need to check for err
 		cp, _ := strconv.Atoi(containerPort)
 
-		parsedCluster.Nodes[0].ExtraPortMappings = append(parsedCluster.Nodes[0].ExtraPortMappings, kindv1alpha4.PortMapping{ContainerPort: int32(cp), HostPort: int32(hp)})
+		if parsedCluster.Nodes[nodePosition].ExtraPortMappings == nil {
+			parsedCluster.Nodes[nodePosition].ExtraPortMappings = make([]kindv1alpha4.PortMapping, 0, 1)
+		}
+		parsedCluster.Nodes[nodePosition].ExtraPortMappings =
+			append(parsedCluster.Nodes[nodePosition].ExtraPortMappings, kindv1alpha4.PortMapping{ContainerPort: int32(cp), HostPort: int32(hp), Protocol: "TCP"})
+	}
+	if appendIngressNodeLabel {
+		if parsedCluster.Nodes[nodePosition].Labels == nil {
+			parsedCluster.Nodes[nodePosition].Labels = make(map[string]string)
+		}
+		parsedCluster.Nodes[nodePosition].Labels[ingressNginxNodeLabelKey] = ingressNginxNodeLabelValue
 	}
 
 	return parsedCluster, nil

pkg/kind/cluster_test.go

@@ -98,37 +98,57 @@ func TestGetConfigCustom(t *testing.T) {
 		inputPath  string
 		outputPath string
 		hostPort   string
-		Protocol   string
+		protocol   string
+		error      bool
 	}
 
 	cases := []testCase{
 		{
-			inputPath:  "testdata/no-necessary-port.yaml",
-			outputPath: "testdata/expected/no-necessary-port.yaml",
+			inputPath:  "testdata/no-port.yaml",
+			outputPath: "testdata/expected/no-port.yaml",
 			hostPort:   "8443",
-			Protocol:   "https",
+			protocol:   "https",
 		},
 		{
-			inputPath:  "testdata/necessary-port-present.yaml",
-			outputPath: "testdata/expected/necessary-port-present.yaml",
+			inputPath:  "testdata/port-only.yaml",
+			outputPath: "testdata/expected/port-only.yaml",
 			hostPort:   "80",
-			Protocol:   "http",
+			protocol:   "http",
+		},
+		{
+			inputPath:  "testdata/no-port-multi.yaml",
+			outputPath: "testdata/expected/no-port-multi.yaml",
+			hostPort:   "8443",
+			protocol:   "https",
+		},
+		{
+			inputPath:  "testdata/label-only.yaml",
+			outputPath: "testdata/expected/label-only.yaml",
+			hostPort:   "8443",
+			protocol:   "https",
+		},
+		{
+			inputPath: "testdata/no-node",
+			error:     true,
 		},
 	}
 
 	for _, v := range cases {
 		c, _ := NewCluster("testcase", "v1.26.3", "", v.inputPath, "", util.CorePackageTemplateConfig{
 			Host:     "cnoe.localtest.me",
 			Port:     v.hostPort,
-			Protocol: v.Protocol,
+			Protocol: v.protocol,
 		})
 
 		b, err := c.getConfig()
+		if v.error {
+			assert.Error(t, err)
+			continue
+		}
 		assert.NoError(t, err)
 		expected, _ := os.ReadFile(v.outputPath)
 		assert.YAMLEq(t, string(expected), string(b))
 	}
-
 }
 
 // Mock provider for testing

pkg/kind/testdata/expected/label-only.yaml

@@ -0,0 +1,21 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking: {}
+nodes:
+  - role: control-plane
+    extraPortMappings:
+      - containerPort: 31337
+        hostPort: 31337
+      - containerPort: 31340
+        hostPort: 31340
+      - containerPort: 31333
+        hostPort: 31333
+  - role: worker
+    image: "abc"
+    labels:
+      ingress-ready: "true"
+    extraPortMappings:
+      - containerPort: 443
+        hostPort: 8443
+        protocol: TCP
+

pkg/kind/testdata/expected/no-port-multi.yaml

@@ -0,0 +1,19 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking: {}
+nodes:
+  - role: control-plane
+    labels:
+      ingress-ready: "true"
+    extraPortMappings:
+      - containerPort: 31337
+        hostPort: 31337
+      - containerPort: 31340
+        hostPort: 31340
+      - containerPort: 31333
+        hostPort: 31333
+      - containerPort: 443
+        hostPort: 8443
+        protocol: TCP
+  - role: worker
+    image: "abc"

pkg/kind/testdata/expected/no-necessary-port.yaml → pkg/kind/testdata/expected/no-port.yaml

@@ -3,6 +3,8 @@ apiVersion: kind.x-k8s.io/v1alpha4
 networking: {}
 nodes:
   - role: control-plane
+    labels:
+      ingress-ready: "true"
     extraMounts:
       - containerPath: /var/lib/kubelet/config.json
         hostPath: ~/.docker/config.json
@@ -14,4 +16,5 @@ nodes:
       - containerPort: 31333
         hostPort: 31333
       - containerPort: 443
-        hostPort: 8443
+        hostPort: 8443
+        protocol: TCP

pkg/kind/testdata/expected/necessary-port-present.yaml → pkg/kind/testdata/expected/port-only.yaml

@@ -3,15 +3,17 @@ apiVersion: kind.x-k8s.io/v1alpha4
 networking: {}
 nodes:
   - role: control-plane
-    extraMounts:
-      - containerPath: /var/lib/kubelet/config.json
-        hostPath: ~/.docker/config.json
     extraPortMappings:
       - containerPort: 31337
         hostPort: 31337
       - containerPort: 31340
         hostPort: 31340
       - containerPort: 31333
         hostPort: 31333
+  - role: worker
+    labels:
+      ingress-ready: "true"
+    extraPortMappings:
       - containerPort: 80
         hostPort: 80
+        protocol: TCP

pkg/kind/testdata/label-only.yaml

@@ -0,0 +1,15 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraPortMappings:
+      - containerPort: 31337
+        hostPort: 31337
+      - containerPort: 31340
+        hostPort: 31340
+      - containerPort: 31333
+        hostPort: 31333
+  - role: worker
+    image: "abc"
+    labels:
+      ingress-ready: "true"

pkg/kind/testdata/no-node.yaml

@@ -0,0 +1,2 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4

pkg/kind/testdata/no-port-multi.yaml

@@ -0,0 +1,13 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraPortMappings:
+      - containerPort: 31337
+        hostPort: 31337
+      - containerPort: 31340
+        hostPort: 31340
+      - containerPort: 31333
+        hostPort: 31333
+  - role: worker
+    image: "abc"

pkg/kind/testdata/necessary-port-present.yaml → pkg/kind/testdata/port-only.yaml

@@ -2,15 +2,15 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
   - role: control-plane
-    extraMounts:
-      - containerPath: /var/lib/kubelet/config.json
-        hostPath: ~/.docker/config.json
     extraPortMappings:
       - containerPort: 31337
         hostPort: 31337
       - containerPort: 31340
         hostPort: 31340
       - containerPort: 31333
         hostPort: 31333
+  - role: worker
+    extraPortMappings:
       - containerPort: 80
         hostPort: 80
+        protocol: TCP