Using bridge network for nodes

cmd/kind/create/cluster/createcluster.go

@@ -88,6 +88,10 @@ func runE(flags *flagpole, cmd *cobra.Command, args []string) error {
 
 	// create a cluster context and create the cluster
 	ctx := cluster.NewContext(flags.Name)
+	// using cluster name as bridge network name
+	if cfg.Networking.BridgeName == "" {
+		cfg.Networking.BridgeName = flags.Name
+	}
 	if flags.ImageName != "" {
 		// Apply image override to all the Nodes defined in Config
 		// TODO(fabrizio pandini): this should be reconsidered when implementing

pkg/cluster/config/types.go

@@ -102,4 +102,7 @@ type Networking struct {
 	// If DisableDefaultCNI is true, kind will not install the default CNI setup.
 	// Instead the user should install their own CNI after creating the cluster.
 	DisableDefaultCNI bool
+	// BridgeName is the bridge network name.
+	// Defaults to same as cluster name.
+	BridgeName string
 }

pkg/cluster/config/v1alpha3/types.go

@@ -102,4 +102,7 @@ type Networking struct {
 	// If DisableDefaultCNI is true, kind will not install the default CNI setup.
 	// Instead the user should install their own CNI after creating the cluster.
 	DisableDefaultCNI bool `json:"disableDefaultCNI,omitempty"`
+	// Name is the bridge network name.
+	// Defaults to same as cluster name.
+	BridgeName string `json:"bridgeName,omitempty"`
 }

pkg/cluster/internal/create/actions/loadbalancer/loadbalancer.go

@@ -71,11 +71,7 @@ func (a *Action) Execute(ctx *actions.ActionContext) error {
 		return err
 	}
 	for _, n := range controlPlaneNodes {
-		controlPlaneIP, err := n.IP()
-		if err != nil {
-			return errors.Wrapf(err, "failed to get IP for node %s", n.Name())
-		}
-		backendServers[n.Name()] = fmt.Sprintf("%s:%d", controlPlaneIP, kubeadm.APIServerPort)
+		backendServers[n.Name()] = fmt.Sprintf("%s:%d", n.Name(), kubeadm.APIServerPort)
 	}
 
 	// create loadbalancer config data

pkg/cluster/internal/create/nodes.go

@@ -29,6 +29,7 @@ import (
 	"sigs.k8s.io/kind/pkg/cluster/nodes"
 	"sigs.k8s.io/kind/pkg/concurrent"
 	"sigs.k8s.io/kind/pkg/container/cri"
+	"sigs.k8s.io/kind/pkg/container/docker"
 	logutil "sigs.k8s.io/kind/pkg/log"
 )
 
@@ -80,6 +81,14 @@ func provisionNodes(
 ) error {
 	defer status.End(false)
 
+	// check if the network is exist.
+	if !docker.IsNetworkExist(cfg.Networking.BridgeName) {
+		// create bridge network for nodes.
+		if err := createNetwork(status, cfg.Networking.BridgeName); err != nil {
+			return err
+		}
+	}
+
 	if err := createNodeContainers(status, cfg, clusterName, clusterLabel); err != nil {
 		return err
 	}
@@ -88,6 +97,18 @@ func provisionNodes(
 	return nil
 }
 
+func createNetwork(status *logutil.Status, clusterName string) error {
+	defer status.End(false)
+
+	status.Start("Creating bridge network 🌉")
+	if err := docker.CreateNetwork(clusterName); err != nil {
+		return err
+	}
+
+	status.End(true)
+	return nil
+}
+
 func createNodeContainers(
 	status *logutil.Status, cfg *config.Cluster, clusterName, clusterLabel string,
 ) error {
@@ -103,7 +124,7 @@ func createNodeContainers(
 		desiredNode := desiredNode // capture loop variable
 		fns = append(fns, func() error {
 			// create the node into a container (~= docker run -d)
-			_, err := desiredNode.Create(clusterLabel)
+			_, err := desiredNode.Create(cfg.Networking.BridgeName, clusterLabel)
 			return err
 		})
 	}
@@ -193,16 +214,16 @@ func nodesToCreate(cfg *config.Cluster, clusterName string) []nodeSpec {
 }
 
 // TODO(bentheelder): remove network in favor of []cri.PortMapping when that is in
-func (d *nodeSpec) Create(clusterLabel string) (node *nodes.Node, err error) {
+func (d *nodeSpec) Create(networkName, clusterLabel string) (node *nodes.Node, err error) {
 	// create the node into a container (docker run, but it is paused, see createNode)
 	// TODO(bentheelder): decouple from config objects further
 	switch d.Role {
 	case constants.ExternalLoadBalancerNodeRoleValue:
-		node, err = nodes.CreateExternalLoadBalancerNode(d.Name, d.Image, clusterLabel, d.APIServerAddress, d.APIServerPort)
+		node, err = nodes.CreateExternalLoadBalancerNode(d.Name, d.Image, networkName, clusterLabel, d.APIServerAddress, d.APIServerPort)
 	case constants.ControlPlaneNodeRoleValue:
-		node, err = nodes.CreateControlPlaneNode(d.Name, d.Image, clusterLabel, d.APIServerAddress, d.APIServerPort, d.ExtraMounts)
+		node, err = nodes.CreateControlPlaneNode(d.Name, d.Image, networkName, clusterLabel, d.APIServerAddress, d.APIServerPort, d.ExtraMounts)
 	case constants.WorkerNodeRoleValue:
-		node, err = nodes.CreateWorkerNode(d.Name, d.Image, clusterLabel, d.ExtraMounts)
+		node, err = nodes.CreateWorkerNode(d.Name, d.Image, networkName, clusterLabel, d.ExtraMounts)
 	default:
 		return nil, errors.Errorf("unknown node role: %s", d.Role)
 	}

pkg/cluster/internal/delete/delete.go

@@ -26,6 +26,7 @@ import (
 
 	"sigs.k8s.io/kind/pkg/cluster/internal/context"
 	"sigs.k8s.io/kind/pkg/cluster/nodes"
+	"sigs.k8s.io/kind/pkg/container/docker"
 )
 
 // Cluster deletes the cluster identified by ctx
@@ -46,5 +47,15 @@ func Cluster(c *context.Context) error {
 		fmt.Printf("$KUBECONFIG is still set to use %s even though that file has been deleted, remember to unset it\n", c.KubeConfigPath())
 	}
 
-	return nodes.Delete(n...)
+	// before delete node, we inspect it to get network name.
+	networkName, err := docker.Inspect(n[0].Name(), "{{range $name, $_:=.NetworkSettings.Networks}}{{$name}}{{end}}")
+	if err != nil {
+		return errors.Wrap(err, "error inspect nodes")
+	}
+
+	if err := nodes.Delete(n...); err != nil {
+		return errors.Wrap(err, "error deleting nodes")
+	}
+	// after all nodes have been deleted, we need delete the network
+	return docker.DeleteNetwork(networkName[0])
 }

pkg/cluster/internal/kubeadm/config.go

@@ -104,6 +104,7 @@ kubeletConfiguration:
       nodefs.available: "0%"
       nodefs.inodesFree: "0%"
       imagefs.available: "0%"
+    resolvConf: /kind/resolv.conf
 controllerManagerExtraArgs:
   enable-hostpath-provisioner: "true"
 nodeRegistration:
@@ -181,6 +182,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+resolvConf: /kind/resolv.conf
 ---
 # no-op entry that exists solely so it can be patched
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
@@ -245,6 +247,7 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+resolvConf: /kind/resolv.conf
 ---
 # no-op entry that exists solely so it can be patched
 apiVersion: kubeproxy.config.k8s.io/v1alpha1

pkg/cluster/nodes/create.go

@@ -49,7 +49,7 @@ func getPort() (int32, error) {
 
 // CreateControlPlaneNode creates a contol-plane node
 // and gets ready for exposing the the API server
-func CreateControlPlaneNode(name, image, clusterLabel, listenAddress string, port int32, mounts []cri.Mount) (node *Node, err error) {
+func CreateControlPlaneNode(name, image, networkName, clusterLabel, listenAddress string, port int32, mounts []cri.Mount) (node *Node, err error) {
 	// gets a random host port for the API server
 	if port == 0 {
 		p, err := getPort()
@@ -60,7 +60,7 @@ func CreateControlPlaneNode(name, image, clusterLabel, listenAddress string, por
 	}
 
 	node, err = createNode(
-		name, image, clusterLabel, constants.ControlPlaneNodeRoleValue, mounts,
+		name, image, networkName, clusterLabel, constants.ControlPlaneNodeRoleValue, mounts,
 		// publish selected port for the API server
 		"--expose", fmt.Sprintf("%d", port),
 		"-p", fmt.Sprintf("%s:%d:%d", listenAddress, port, kubeadm.APIServerPort),
@@ -74,12 +74,16 @@ func CreateControlPlaneNode(name, image, clusterLabel, listenAddress string, por
 		cache.ports = map[int32]int32{kubeadm.APIServerPort: port}
 	})
 
+	if err := addResolve(node); err != nil {
+		return node, err
+	}
+
 	return node, nil
 }
 
 // CreateExternalLoadBalancerNode creates an external loab balancer node
 // and gets ready for exposing the the API server and the load balancer admin console
-func CreateExternalLoadBalancerNode(name, image, clusterLabel, listenAddress string, port int32) (node *Node, err error) {
+func CreateExternalLoadBalancerNode(name, image, networkName, clusterLabel, listenAddress string, port int32) (node *Node, err error) {
 	// gets a random host port for control-plane load balancer
 	// gets a random host port for the API server
 	if port == 0 {
@@ -90,7 +94,7 @@ func CreateExternalLoadBalancerNode(name, image, clusterLabel, listenAddress str
 		port = p
 	}
 
-	node, err = createNode(name, image, clusterLabel, constants.ExternalLoadBalancerNodeRoleValue,
+	node, err = createNode(name, image, networkName, clusterLabel, constants.ExternalLoadBalancerNodeRoleValue,
 		nil,
 		// publish selected port for the control plane
 		"--expose", fmt.Sprintf("%d", port),
@@ -109,19 +113,24 @@ func CreateExternalLoadBalancerNode(name, image, clusterLabel, listenAddress str
 }
 
 // CreateWorkerNode creates a worker node
-func CreateWorkerNode(name, image, clusterLabel string, mounts []cri.Mount) (node *Node, err error) {
-	node, err = createNode(name, image, clusterLabel, constants.WorkerNodeRoleValue, mounts)
+func CreateWorkerNode(name, image, networkName, clusterLabel string, mounts []cri.Mount) (node *Node, err error) {
+	node, err = createNode(name, image, networkName, clusterLabel, constants.WorkerNodeRoleValue, mounts)
 	if err != nil {
 		return node, err
 	}
+
+	if err = addResolve(node); err != nil {
+		return node, err
+	}
+
 	return node, nil
 }
 
 // TODO(bentheelder): refactor this to not have extraArgs
 // createNode `docker run`s the node image, note that due to
 // images/node/entrypoint being the entrypoint, this container will
 // effectively be paused until we call actuallyStartNode(...)
-func createNode(name, image, clusterLabel, role string, mounts []cri.Mount, extraArgs ...string) (handle *Node, err error) {
+func createNode(name, image, networkName, clusterLabel, role string, mounts []cri.Mount, extraArgs ...string) (handle *Node, err error) {
 	runArgs := []string{
 		"-d", // run the container detached
 		"-t", // allocate a tty for entrypoint logs
@@ -142,6 +151,8 @@ func createNode(name, image, clusterLabel, role string, mounts []cri.Mount, extr
 		"--label", clusterLabel,
 		// label the node with the role ID
 		"--label", fmt.Sprintf("%s=%s", constants.NodeRoleKey, role),
+		// connect node to network
+		"--network", networkName,
 	}
 
 	// pass proxy environment variables to be used by node's docker deamon

pkg/cluster/nodes/util.go

@@ -18,6 +18,9 @@ package nodes
 
 import (
 	"fmt"
+	"io/ioutil"
+	"regexp"
+	"runtime"
 
 	"github.com/pkg/errors"
 	"sigs.k8s.io/kind/pkg/cluster/internal/loadbalancer"
@@ -44,3 +47,29 @@ func GetControlPlaneEndpoint(allNodes []Node) (string, error) {
 
 	return fmt.Sprintf("%s:%d", loadBalancerIP, loadbalancer.ControlPlanePort), nil
 }
+
+// to avoid the DNS crash we need copy host's /etc/resolv.conf to node
+// ref: https://github.com/kubernetes-sigs/kind/pull/484#issuecomment-489469044
+func addResolve(node *Node) error {
+	// TODO(Jintao Zhang): I can't accurately judge the behavior of windows, so I skipped it for now.
+	// https://github.com/kubernetes-sigs/kind/pull/484#discussion_r284064671
+	if runtime.GOOS == "windows" {
+		return nil
+	}
+
+	resolv, err := ioutil.ReadFile("/etc/resolv.conf")
+	if err != nil {
+		return errors.Wrap(err, "failed to read /etc/resolv.conf")
+	}
+
+	// filter the loopback addresses
+	re := regexp.MustCompile("(?m)[\r\n]+^.*((127.([0-9]{1,3}.){2}[0-9]{1,3})|(::1)).*$")
+	content := re.ReplaceAllString(string(resolv), "")
+
+	err = node.WriteFile("/kind/resolv.conf", content)
+	if err != nil {
+		return errors.Wrap(err, "failed to write /kind/resolv.conf to node")
+	}
+
+	return nil
+}

pkg/container/docker/network.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package docker
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kind/pkg/cluster/constants"
+	"sigs.k8s.io/kind/pkg/exec"
+)
+
+// CreateNetwork create a bridge network
+func CreateNetwork(networkName string) error {
+	cmd := exec.Command(
+		"docker", "network",
+		"create",
+		"--driver=bridge",
+		"--label="+fmt.Sprintf("%s=%s", constants.ClusterLabelKey, networkName),
+		networkName,
+	)
+	return cmd.Run()
+}
+
+// DeleteNetwork delete the special network
+func DeleteNetwork(networkName string) error {
+	cmd := exec.Command(
+		"docker", "network",
+		"rm",
+		networkName,
+	)
+	return cmd.Run()
+}
+
+// IsNetworkExist check if the network exist
+func IsNetworkExist(networkName string) bool {
+	cmd := exec.Command(
+		"docker", "network",
+		"inspect",
+		networkName,
+	)
+	if err := cmd.Run(); err != nil {
+		return false
+	}
+
+	return true
+}