Configurable DNS server in agent settings per IG

cloudfoundry-incubator · Nov 16, 2020 · f3f9af1 · f3f9af1
1 parent 9ed17d4
commit f3f9af1
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 20 deletions.
diff --git a/go.sum b/go.sum
@@ -18,6 +18,9 @@ code.cloudfoundry.org/quarks-secret v1.0.752 h1:DmqOuhVD+w3vDiWggEjmFcWRdCQGdDv4
 code.cloudfoundry.org/quarks-secret v1.0.752/go.mod h1:5XWVFJ4wVZCbElNBl4Sjb38r038WsxrrSt0DMIqhSQU=
 code.cloudfoundry.org/quarks-statefulset v0.0.1304-g9f060bca h1:B2M8EyaeIAA7dLaAGRVSnc0GGL0t07KoPWNhW44ymMY=
 code.cloudfoundry.org/quarks-statefulset v0.0.1304-g9f060bca/go.mod h1:X7lKMxIBLtTs/RYgV42Gdn1uBWvkCNrHP4OP9e5owPA=
+code.cloudfoundry.org/quarks-utils v0.0.2-0.20201027114038-8aab73d224e4/go.mod h1:K8KH67rdNk9+VPOA5QRgrujTFhbmtqxLOuPQ6APL6ks=
+code.cloudfoundry.org/quarks-utils v0.0.2-0.20201104164019-cb2fb89e3552 h1:5wX9bbxBi7ViUucYcnZTNnbwV/BZ9EDhsRXg/UtrleY=
+code.cloudfoundry.org/quarks-utils v0.0.2-0.20201104164019-cb2fb89e3552/go.mod h1:K8KH67rdNk9+VPOA5QRgrujTFhbmtqxLOuPQ6APL6ks=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=

diff --git a/pkg/bosh/bpmconverter/resources.go b/pkg/bosh/bpmconverter/resources.go
@@ -228,9 +228,16 @@ func (kc *BPMConverter) serviceToQuarksStatefulSet(
 
 	spec := &extSts.Spec.Template.Spec.Template.Spec
 
-	spec.DNSPolicy, spec.DNSConfig, err = boshdns.DNSSetting(manifest, serviceIP, namespace)
-	if err != nil {
-		return qstsv1a1.QuarksStatefulSet{}, err
+	if instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.DNS != "" {
+		spec.DNSPolicy, spec.DNSConfig = boshdns.CustomDNSSetting(
+			instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.DNS,
+			namespace,
+		)
+	} else {
+		spec.DNSPolicy, spec.DNSConfig, err = boshdns.DNSSetting(manifest, serviceIP, namespace)
+		if err != nil {
+			return qstsv1a1.QuarksStatefulSet{}, err
+		}
 	}
 
 	if len(instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.Tolerations) > 0 {
@@ -392,25 +399,34 @@ func (kc *BPMConverter) errandToQuarksJob(
 		},
 	}
 
-	qJob.Spec.Template.Spec.Template.Spec.DNSPolicy, qJob.Spec.Template.Spec.Template.Spec.DNSConfig, err = boshdns.DNSSetting(manifest, serviceIP, namespace)
-	if err != nil {
-		return qjv1a1.QuarksJob{}, err
+	spec := &qJob.Spec.Template.Spec.Template.Spec
+
+	if instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.DNS != "" {
+		spec.DNSPolicy, spec.DNSConfig = boshdns.CustomDNSSetting(
+			instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.DNS,
+			namespace,
+		)
+	} else {
+		spec.DNSPolicy, spec.DNSConfig, err = boshdns.DNSSetting(manifest, serviceIP, namespace)
+		if err != nil {
+			return qjv1a1.QuarksJob{}, err
+		}
 	}
 
 	if instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.Affinity != nil {
-		qJob.Spec.Template.Spec.Template.Spec.Affinity = instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.Affinity
+		spec.Affinity = instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.Affinity
 	}
 
 	if len(instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.Tolerations) > 0 {
-		qJob.Spec.Template.Spec.Template.Spec.Tolerations = instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.Tolerations
+		spec.Tolerations = instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.Tolerations
 	}
 
 	if instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.ServiceAccountName != "" {
-		qJob.Spec.Template.Spec.Template.Spec.ServiceAccountName = instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.ServiceAccountName
+		spec.ServiceAccountName = instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.ServiceAccountName
 	}
 
 	if instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.AutomountServiceAccountToken != nil {
-		qJob.Spec.Template.Spec.Template.Spec.AutomountServiceAccountToken = instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.AutomountServiceAccountToken
+		spec.AutomountServiceAccountToken = instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.AutomountServiceAccountToken
 	}
 
 	return qJob, nil

diff --git a/pkg/bosh/manifest/instance_group.go b/pkg/bosh/manifest/instance_group.go
@@ -267,6 +267,7 @@ type AgentSettings struct {
 	PreRenderOps                  *PreRenderOps                 `json:"preRenderOps,omitempty"`
 	InjectReplicasEnv             *bool                         `json:"injectReplicasEnv,omitempty"`
 	TerminationGracePeriodSeconds *int64                        `json:"terminationGracePeriodSeconds,omitempty" yaml:"terminationGracePeriodSeconds,omitempty"`
+	DNS                           string                        `json:"dns,omitempty"`
 }
 
 // Set overrides labels and annotations with operator-owned metadata.

diff --git a/pkg/kube/util/boshdns/interface.go b/pkg/kube/util/boshdns/interface.go
@@ -46,23 +46,29 @@ func Validate(m bdm.Manifest) error {
 	return err
 }
 
+// CustomDNSSetting sets the pod dns policy.
+func CustomDNSSetting(serviceIP, namespace string) (corev1.DNSPolicy, *corev1.PodDNSConfig) {
+	ndots := "5"
+	return corev1.DNSNone, &corev1.PodDNSConfig{
+		Nameservers: []string{serviceIP},
+		Searches: []string{
+			fmt.Sprintf("%s.svc.%s", namespace, clusterDomain),
+			fmt.Sprintf("svc.%s", clusterDomain),
+			clusterDomain,
+		},
+		Options: []corev1.PodDNSConfigOption{{Name: "ndots", Value: &ndots}},
+	}
+}
+
 // DNSSetting sets the pod dns policy.
 func DNSSetting(m bdm.Manifest, serviceIP, namespace string) (corev1.DNSPolicy, *corev1.PodDNSConfig, error) {
 	index := HasBoshDNSAddOn(m)
 	if index != -1 {
 		if serviceIP == "" {
 			return corev1.DNSNone, nil, errors.New("BoshDomainNameService: DNSSetting called before Apply")
 		}
-		ndots := "5"
-		return corev1.DNSNone, &corev1.PodDNSConfig{
-			Nameservers: []string{serviceIP},
-			Searches: []string{
-				fmt.Sprintf("%s.svc.%s", namespace, clusterDomain),
-				fmt.Sprintf("svc.%s", clusterDomain),
-				clusterDomain,
-			},
-			Options: []corev1.PodDNSConfigOption{{Name: "ndots", Value: &ndots}},
-		}, nil
+		p, c := CustomDNSSetting(serviceIP, namespace)
+		return p, c, nil
 	}
 
 	return corev1.DNSClusterFirst, nil, nil

diff --git a/pkg/kube/util/boshdns/interface_test.go b/pkg/kube/util/boshdns/interface_test.go
@@ -131,4 +131,14 @@ var _ = Describe("Interface", func() {
 			Expect(config).NotTo(BeNil())
 		})
 	})
+
+	Context("CustomDNSSetting", func() {
+		It("returns custom dns", func() {
+			policy, config := boshdns.CustomDNSSetting("1.2.3.5", "default")
+			Expect(policy).To(Equal(corev1.DNSNone))
+			Expect(config).NotTo(BeNil())
+			Expect(config.Nameservers).To(Equal([]string{"1.2.3.5"}))
+			Expect(config.Searches).To(ContainElements("default.svc.", "svc.", ""))
+		})
+	})
 })