Skip to content

Commit ee0a5d2

Browse files
cottandcottand
authored
Merge pull request #43 from monzo/nico.envoy-optional-args
Add option to ExternalService to control envoy proxy arguments
2 parents 77b65bc + 6cf68f7 commit ee0a5d2

File tree

4 files changed

+138
-77
lines changed

4 files changed

+138
-77
lines changed

‎api/v1/externalservice_types.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,19 @@ type ExternalServiceSpec struct {
6363
// +optional
6464
EnvoyClusterMaxConnections *uint32 `json:"envoyClusterMaxConnections,omitempty"`
6565

66+
// Input to the --log-level command line option. See the help text for the available log levels and the default.
67+
EnvoyLogLevel string `json:"envoyLogLevel,omitempty"`
68+
69+
// Corresponds to Envoy's dns_refresh_rate config field for this cluster, in seconds
70+
// See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto
71+
// +optional
72+
EnvoyDnsRefreshRateS int64 `json:"envoyDnsRefreshRateS,omitempty"`
73+
74+
// Corresponds to Envoy's respect_dns_ttl config field for this cluster.
75+
// See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto
76+
// +optional
77+
EnvoyRespectDnsTTL bool `json:"envoyRespectDnsTTL,omitempty"`
78+
6679
// Provides a way to override the global default
6780
// +optional
6881
ServiceTopologyMode string `json:"serviceTopologyMode,omitempty"`

‎config/crd/bases/egress.monzo.com_externalservices.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,19 @@ spec:
4646
cluster will increment.
4747
format: int32
4848
type: integer
49+
envoyDnsRefreshRateS:
50+
description: "Corresponds to Envoy's dns_refresh_rate config field
51+
for this cluster, in seconds See\thttps://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto"
52+
format: int64
53+
type: integer
54+
envoyLogLevel:
55+
description: Input to the --log-level command line option. See the
56+
help text for the available log levels and the default.
57+
type: string
58+
envoyRespectDnsTTL:
59+
description: "Corresponds to Envoy's respect_dns_ttl config field
60+
for this cluster. See\thttps://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto"
61+
type: boolean
4962
hijackDns:
5063
description: 'If true, add a `egress.monzo.com/hijack-dns: true` label
5164
to produced Service objects CoreDNS can watch this label and decide
@@ -131,6 +144,9 @@ spec:
131144
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
132145
type: object
133146
type: object
147+
serviceTopologyMode:
148+
description: Provides a way to override the global default
149+
type: string
134150
targetCPUUtilizationPercentage:
135151
description: Target average CPU utilization (represented as a percentage
136152
of requested CPU) over all the pods. Defaults to 50

‎controllers/configmap.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package controllers
33
import (
44
"context"
55
"fmt"
6+
"google.golang.org/protobuf/types/known/durationpb"
67
"hash/fnv"
78
"strconv"
89

@@ -109,6 +110,10 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
109110
}
110111

111112
for _, port := range es.Spec.Ports {
113+
var dnsRefreshRate *duration.Duration
114+
if es.Spec.EnvoyDnsRefreshRateS != 0 {
115+
dnsRefreshRate = &durationpb.Duration{Seconds: es.Spec.EnvoyDnsRefreshRateS}
116+
}
112117
var clusters []*envoyv3.Cluster
113118
protocol := protocolToEnvoy(port.Protocol)
114119
name := fmt.Sprintf("%s_%s_%s", es.Name, envoycorev3.SocketAddress_Protocol_name[int32(protocol)], strconv.Itoa(int(port.Port)))
@@ -130,6 +135,9 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
130135
KeepaliveInterval: &wrapperspb.UInt32Value{Value: 5},
131136
},
132137
},
138+
139+
DnsRefreshRate: dnsRefreshRate,
140+
RespectDnsTtl: es.Spec.EnvoyRespectDnsTTL,
133141
LoadAssignment: &envoyendpoint.ClusterLoadAssignment{
134142
ClusterName: name,
135143
Endpoints: []*envoyendpoint.LocalityLbEndpoints{
@@ -298,6 +306,10 @@ func configmap(es *egressv1.ExternalService) (*corev1.ConfigMap, string, error)
298306

299307
func generateOverrideCluster(name string, spec egressv1.ExternalServiceSpec, port egressv1.ExternalServicePort, protocol envoycorev3.SocketAddress_Protocol) *envoyv3.Cluster {
300308
overrideClusterName := fmt.Sprintf("%v-override", name)
309+
var dnsRefreshRate *duration.Duration
310+
if spec.EnvoyDnsRefreshRateS != 0 {
311+
dnsRefreshRate = &durationpb.Duration{Seconds: spec.EnvoyDnsRefreshRateS}
312+
}
301313
var endpoints []*envoyendpoint.LocalityLbEndpoints
302314

303315
for _, ip := range spec.IpOverride {
@@ -356,6 +368,9 @@ func generateOverrideCluster(name string, spec egressv1.ExternalServiceSpec, por
356368
ClusterName: overrideClusterName,
357369
Endpoints: endpoints,
358370
},
371+
372+
DnsRefreshRate: dnsRefreshRate,
373+
RespectDnsTtl: spec.EnvoyRespectDnsTTL,
359374
}
360375
}
361376

‎controllers/deployment.go‎

Lines changed: 94 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,17 @@ import (
2020

2121
// +kubebuilder:rbac:namespace=egress-operator-system,groups=apps,resources=deployments,verbs=get;list;watch;create;patch
2222

23+
var validLogLevels = map[string]bool{
24+
"trace": true,
25+
"debug": true,
26+
"info": true,
27+
"warning": true,
28+
"warn": true,
29+
"error": true,
30+
"critical": true,
31+
"off": true,
32+
}
33+
2334
func (r *ExternalServiceReconciler) reconcileDeployment(ctx context.Context, req ctrl.Request, es *egressv1.ExternalService, configHash string) error {
2435
desired := deployment(es, configHash)
2536
if err := ctrl.SetControllerReference(es, desired, r.Scheme); err != nil {
@@ -157,93 +168,84 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym
157168
},
158169
}
159170
}
160-
161-
return &appsv1.Deployment{
162-
ObjectMeta: metav1.ObjectMeta{
163-
Name: es.Name,
164-
Namespace: namespace,
165-
Labels: labels(es),
166-
Annotations: annotations(es),
171+
deploymentSpec := appsv1.DeploymentSpec{
172+
ProgressDeadlineSeconds: proto.Int(600),
173+
RevisionHistoryLimit: proto.Int(10),
174+
Strategy: appsv1.DeploymentStrategy{
175+
Type: appsv1.RollingUpdateDeploymentStrategyType,
176+
RollingUpdate: &appsv1.RollingUpdateDeployment{
177+
MaxUnavailable: intstr.ValueOrDefault(nil, intstr.FromString("25%")),
178+
MaxSurge: intstr.ValueOrDefault(nil, intstr.FromString("25%")),
179+
},
167180
},
168-
Spec: appsv1.DeploymentSpec{
169-
ProgressDeadlineSeconds: proto.Int(600),
170-
RevisionHistoryLimit: proto.Int(10),
171-
Strategy: appsv1.DeploymentStrategy{
172-
Type: appsv1.RollingUpdateDeploymentStrategyType,
173-
RollingUpdate: &appsv1.RollingUpdateDeployment{
174-
MaxUnavailable: intstr.ValueOrDefault(nil, intstr.FromString("25%")),
175-
MaxSurge: intstr.ValueOrDefault(nil, intstr.FromString("25%")),
176-
},
181+
Selector: labelSelector,
182+
Template: corev1.PodTemplateSpec{
183+
ObjectMeta: metav1.ObjectMeta{
184+
Labels: labels(es),
185+
Annotations: a,
177186
},
178-
Selector: labelSelector,
179-
Template: corev1.PodTemplateSpec{
180-
ObjectMeta: metav1.ObjectMeta{
181-
Labels: labels(es),
182-
Annotations: a,
183-
},
184-
Spec: corev1.PodSpec{
185-
Tolerations: tolerations,
186-
NodeSelector: nodeSelector,
187-
TopologySpreadConstraints: podTopologySpread,
188-
Containers: []corev1.Container{
189-
{
190-
Name: "gateway",
191-
Image: img,
192-
ImagePullPolicy: corev1.PullIfNotPresent,
193-
Ports: deploymentPorts(es),
194-
VolumeMounts: []corev1.VolumeMount{
195-
{
196-
Name: "envoy-config",
197-
MountPath: "/etc/envoy",
198-
},
187+
Spec: corev1.PodSpec{
188+
Tolerations: tolerations,
189+
NodeSelector: nodeSelector,
190+
TopologySpreadConstraints: podTopologySpread,
191+
Containers: []corev1.Container{
192+
{
193+
Name: "gateway",
194+
Image: img,
195+
ImagePullPolicy: corev1.PullIfNotPresent,
196+
Ports: deploymentPorts(es),
197+
VolumeMounts: []corev1.VolumeMount{
198+
{
199+
Name: "envoy-config",
200+
MountPath: "/etc/envoy",
199201
},
200-
// Copying istio; don't try drain outbound listeners, but after going into terminating state,
201-
// wait 25 seconds for connections to naturally close before going ahead with stop.
202-
Lifecycle: &corev1.Lifecycle{
203-
PreStop: &corev1.LifecycleHandler{
204-
Exec: &corev1.ExecAction{
205-
Command: []string{"/bin/sleep", "25"},
206-
},
202+
},
203+
// Copying istio; don't try drain outbound listeners, but after going into terminating state,
204+
// wait 25 seconds for connections to naturally close before going ahead with stop.
205+
Lifecycle: &corev1.Lifecycle{
206+
PreStop: &corev1.LifecycleHandler{
207+
Exec: &corev1.ExecAction{
208+
Command: []string{"/bin/sleep", "25"},
207209
},
208210
},
209-
TerminationMessagePath: corev1.TerminationMessagePathDefault,
210-
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
211-
ReadinessProbe: &corev1.Probe{
212-
ProbeHandler: corev1.ProbeHandler{
213-
HTTPGet: &corev1.HTTPGetAction{
214-
Path: "/ready",
215-
Port: intstr.FromInt(int(adPort)),
216-
Scheme: corev1.URISchemeHTTP,
217-
},
211+
},
212+
TerminationMessagePath: corev1.TerminationMessagePathDefault,
213+
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
214+
ReadinessProbe: &corev1.Probe{
215+
ProbeHandler: corev1.ProbeHandler{
216+
HTTPGet: &corev1.HTTPGetAction{
217+
Path: "/ready",
218+
Port: intstr.FromInt(int(adPort)),
219+
Scheme: corev1.URISchemeHTTP,
218220
},
219-
FailureThreshold: 3,
220-
PeriodSeconds: 10,
221-
SuccessThreshold: 1,
222-
TimeoutSeconds: 1,
223221
},
224-
Resources: resources,
225-
Env: []corev1.EnvVar{
226-
{
227-
Name: "ENVOY_UID",
228-
Value: "0",
229-
},
222+
FailureThreshold: 3,
223+
PeriodSeconds: 10,
224+
SuccessThreshold: 1,
225+
TimeoutSeconds: 1,
226+
},
227+
Resources: resources,
228+
Env: []corev1.EnvVar{
229+
{
230+
Name: "ENVOY_UID",
231+
Value: "0",
230232
},
231233
},
232234
},
233-
RestartPolicy: corev1.RestartPolicyAlways,
234-
SchedulerName: corev1.DefaultSchedulerName,
235-
SecurityContext: &corev1.PodSecurityContext{},
236-
TerminationGracePeriodSeconds: proto.Int64(30),
237-
DNSPolicy: corev1.DNSDefault,
238-
Volumes: []corev1.Volume{
239-
{
240-
Name: "envoy-config",
241-
VolumeSource: corev1.VolumeSource{
242-
ConfigMap: &corev1.ConfigMapVolumeSource{
243-
DefaultMode: proto.Int(420),
244-
LocalObjectReference: corev1.LocalObjectReference{
245-
Name: es.Name,
246-
},
235+
},
236+
RestartPolicy: corev1.RestartPolicyAlways,
237+
SchedulerName: corev1.DefaultSchedulerName,
238+
SecurityContext: &corev1.PodSecurityContext{},
239+
TerminationGracePeriodSeconds: proto.Int64(30),
240+
DNSPolicy: corev1.DNSDefault,
241+
Volumes: []corev1.Volume{
242+
{
243+
Name: "envoy-config",
244+
VolumeSource: corev1.VolumeSource{
245+
ConfigMap: &corev1.ConfigMapVolumeSource{
246+
DefaultMode: proto.Int(420),
247+
LocalObjectReference: corev1.LocalObjectReference{
248+
Name: es.Name,
247249
},
248250
},
249251
},
@@ -252,4 +254,19 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym
252254
},
253255
},
254256
}
257+
258+
defaultArgs := []string{"-c", "/etc/envoy/envoy.yaml"}
259+
if validLogLevels[es.Spec.EnvoyLogLevel] {
260+
deploymentSpec.Template.Spec.Containers[0].Args = append(defaultArgs, "--log-level", es.Spec.EnvoyLogLevel)
261+
}
262+
263+
return &appsv1.Deployment{
264+
ObjectMeta: metav1.ObjectMeta{
265+
Name: es.Name,
266+
Namespace: namespace,
267+
Labels: labels(es),
268+
Annotations: annotations(es),
269+
},
270+
Spec: deploymentSpec,
271+
}
255272
}

0 commit comments

Comments
 (0)