Allow setting service topology aware routing mode

Author: domgoodwin

README.md

@@ -202,15 +202,22 @@ spec:
           egress.monzo.com/gateway: egress-gateway-name
 ```
 
-| Variable name                      | Default                                   | Description                                        |
-|------------------------------------|-------------------------------------------|----------------------------------------------------|
-| ENVOY_IMAGE                        | `envoyproxy/envoy-alpine:v1.16.5`         | Name of the Envoy Proxy image to use               |
-| TAINT_TOLERATION_KEY               | Empty, no tolerations applied             | Toleration key to apply to gateway pods            |
-| TAINT_TOLERATION_VALUE             | Empty, no tolerations applied             | Toleration value to apply to gateway pods          |
-| NODE_SELECTOR_KEY                  | Empty, no node selector added             | Node selector label key to apply to gateway pods   |
-| NODE_SELECTOR_VALUE                | Empty, no node selector added             | Node selector label value to apply to gateway pods |
-| POD_TOPOLOGY_ZONE_MAX_SKEW_KEY     | `topology.kubernetes.io/zone`             | Topology key for the zone constraint               |
-| POD_TOPOLOGY_ZONE_MAX_SKEW         | Empty, won't inject a zone constraint     | Value of maxSkew for the zone constraint           |
-| POD_TOPOLOGY_HOSTNAME_MAX_SKEW_KEY | `kubernetes.io/hostname`                  | Topology key for the hostname constraint           |
-| POD_TOPOLOGY_HOSTNAME_MAX_SKEW     | Empty, won't inject a hostname constraint | Value of maxSkew for the hostname constraint       |
-
+You can also setup services to have their endpoints aware of network topologies with [topology aware routing](https://kubernetes.io/docs/concepts/services-networking/topology-aware-routing/).
+
+If you set the environment variable `ENABLE_SERVICE_TOPOLOGY_MODE="true"` this will enable setting the annotations `service.kubernetes.io/topology-mode="auto"`.
+
+If you turn this on you can also set `spec.serviceTopologyMode` to any value if you want to disable this for particular egress'. For example
+if you set `spec.serviceTopologyMode: "None"` this will disable topology aware routing for your ExternalService.
+
+| Variable name                      | Default                                   | Description                                               |
+|------------------------------------|-------------------------------------------|-----------------------------------------------------------|
+| ENVOY_IMAGE                        | `envoyproxy/envoy-alpine:v1.16.5`         | Name of the Envoy Proxy image to use                      |
+| TAINT_TOLERATION_KEY               | Empty, no tolerations applied             | Toleration key to apply to gateway pods                   |
+| TAINT_TOLERATION_VALUE             | Empty, no tolerations applied             | Toleration value to apply to gateway pods                 |
+| NODE_SELECTOR_KEY                  | Empty, no node selector added             | Node selector label key to apply to gateway pods          |
+| NODE_SELECTOR_VALUE                | Empty, no node selector added             | Node selector label value to apply to gateway pods        |
+| POD_TOPOLOGY_ZONE_MAX_SKEW_KEY     | `topology.kubernetes.io/zone`             | Topology key for the zone constraint                      |
+| POD_TOPOLOGY_ZONE_MAX_SKEW         | Empty, won't inject a zone constraint     | Value of maxSkew for the zone constraint                  |
+| POD_TOPOLOGY_HOSTNAME_MAX_SKEW_KEY | `kubernetes.io/hostname`                  | Topology key for the hostname constraint                  |
+| POD_TOPOLOGY_HOSTNAME_MAX_SKEW     | Empty, won't inject a hostname constraint | Value of maxSkew for the hostname constraint              |
+| ENABLE_SERVICE_TOPOLOGY_MODE       | Empty, won't add the annotation           | Set to 'true' to add the topology mode service annotation |

api/v1/externalservice_types.go

@@ -62,6 +62,10 @@ type ExternalServiceSpec struct {
 	// If this circuit breaker overflows the upstream_cx_overflow counter for the cluster will increment.
 	// +optional
 	EnvoyClusterMaxConnections *uint32 `json:"envoyClusterMaxConnections,omitempty"`
+
+	// Provides a way to override the global default
+	// +optional
+	ServiceTopologyMode string `json:"serviceTopologyMode,omitempty"`
 }
 
 type ExternalServicePort struct {

controllers/externalservice_controller.go

@@ -18,9 +18,9 @@ package controllers
 import (
 	"bytes"
 	"context"
+	"os"
 
 	"github.com/go-logr/logr"
-	egressv1 "github.com/monzo/egress-operator/api/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -29,6 +29,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	egressv1 "github.com/monzo/egress-operator/api/v1"
 )
 
 const namespace = "egress-operator-system"
@@ -106,9 +108,19 @@ func labels(es *egressv1.ExternalService) map[string]string {
 }
 
 func annotations(es *egressv1.ExternalService) map[string]string {
-	return map[string]string{
+	annotations := map[string]string{
 		"egress.monzo.com/dns-name": es.Spec.DnsName,
 	}
+	// Allow setting the topology aware routing annotation
+	value, ok := os.LookupEnv("ENABLE_SERVICE_TOPOLOGY_MODE")
+	if ok && value == "true" {
+		if es.Spec.ServiceTopologyMode != "" {
+			annotations["service.kubernetes.io/topology-mode"] = es.Spec.ServiceTopologyMode
+		} else {
+			annotations["service.kubernetes.io/topology-mode"] = "Auto"
+		}
+	}
+	return annotations
 }
 
 func labelsToSelect(es *egressv1.ExternalService) map[string]string {