feat: add support for cluster-level settings for non xRoute-generated…

… backend refs (envoyproxy#3954) * Add support for traffic management features for non xRoute-generated clusters. Signed-off-by: Lior Okman <[email protected]> * Make gen-check happy Signed-off-by: Lior Okman <[email protected]> * Update the CEL configuration and tests to make the required tests be installable. Unlimited arrays cause CEL validation costs to be higher than what the API server will accept. Signed-off-by: Lior Okman <[email protected]> * Added some tests Signed-off-by: Lior Okman <[email protected]> * DNS is also a feature that can be enabled for extProc and extAuth clusters, so treat it as such. Signed-off-by: Lior Okman <[email protected]> * Renamed BackendConfig to BackendSettings Signed-off-by: Lior Okman <[email protected]> * make gen-check happy Signed-off-by: Lior Okman <[email protected]> * Rebase to make gen-check happy again. Signed-off-by: Lior Okman <[email protected]> * Translate DNS settings correctly. Signed-off-by: Lior Okman <[email protected]> * Rebased to include upstream changes. Update the PR to include HTTP2 settings as well. Signed-off-by: Lior Okman <[email protected]> * Fix a typo. Signed-off-by: Lior Okman <[email protected]> --------- Signed-off-by: Lior Okman <[email protected]>
sanposhiho · Aug 6, 2024 · 866a6ce · 866a6ce
1 parent 3497600
commit 866a6ce
Show file tree

Hide file tree

Showing 39 changed files with 4,362 additions and 1,110 deletions.
diff --git a/api/v1alpha1/backendtrafficpolicy_types.go b/api/v1alpha1/backendtrafficpolicy_types.go
@@ -46,12 +46,61 @@ type BackendTrafficPolicy struct {
 // BackendTrafficPolicySpec defines the desired state of BackendTrafficPolicy.
 type BackendTrafficPolicySpec struct {
 	PolicyTargetReferences `json:",inline"`
+	ClusterSettings        `json:",inline"`
 
 	// RateLimit allows the user to limit the number of incoming requests
 	// to a predefined value based on attributes within the traffic flow.
 	// +optional
 	RateLimit *RateLimitSpec `json:"rateLimit,omitempty"`
 
+	// FaultInjection defines the fault injection policy to be applied. This configuration can be used to
+	// inject delays and abort requests to mimic failure scenarios such as service failures and overloads
+	// +optional
+	FaultInjection *FaultInjection `json:"faultInjection,omitempty"`
+
+	// Retry provides more advanced usage, allowing users to customize the number of retries, retry fallback strategy, and retry triggering conditions.
+	// If not set, retry will be disabled.
+	// +optional
+	Retry *Retry `json:"retry,omitempty"`
+
+	// UseClientProtocol configures Envoy to prefer sending requests to backends using
+	// the same HTTP protocol that the incoming request used. Defaults to false, which means
+	// that Envoy will use the protocol indicated by the attached BackendRef.
+	//
+	// +optional
+	UseClientProtocol *bool `json:"useClientProtocol,omitempty"`
+
+	// The compression config for the http streams.
+	//
+	// +optional
+	// +notImplementedHide
+	Compression []*Compression `json:"compression,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// BackendTrafficPolicyList contains a list of BackendTrafficPolicy resources.
+type BackendTrafficPolicyList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []BackendTrafficPolicy `json:"items"`
+}
+
+// BackendTrafficPolicyConnection allows users to configure connection-level settings of backend
+type BackendTrafficPolicyConnection struct {
+	// BufferLimit Soft limit on size of the cluster’s connections read and write buffers.
+	// If unspecified, an implementation defined default is applied (32768 bytes).
+	// For example, 20Mi, 1Gi, 256Ki etc.
+	// Note: that when the suffix is not provided, the value is interpreted as bytes.
+	//
+	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="BufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
+	// +optional
+	BufferLimit *resource.Quantity `json:"bufferLimit,omitempty"`
+}
+
+// ClusterSettings provides the various knobs that can be set to control how traffic to a given
+// backend will be configured.
+type ClusterSettings struct {
 	// LoadBalancer policy to apply when routing traffic from the gateway to
 	// the backend endpoints
 	// +optional
@@ -72,44 +121,22 @@ type BackendTrafficPolicySpec struct {
 	// +optional
 	HealthCheck *HealthCheck `json:"healthCheck,omitempty"`
 
-	// FaultInjection defines the fault injection policy to be applied. This configuration can be used to
-	// inject delays and abort requests to mimic failure scenarios such as service failures and overloads
-	// +optional
-	FaultInjection *FaultInjection `json:"faultInjection,omitempty"`
-
 	// Circuit Breaker settings for the upstream connections and requests.
 	// If not set, circuit breakers will be enabled with the default thresholds
 	//
 	// +optional
 	CircuitBreaker *CircuitBreaker `json:"circuitBreaker,omitempty"`
 
-	// Retry provides more advanced usage, allowing users to customize the number of retries, retry fallback strategy, and retry triggering conditions.
-	// If not set, retry will be disabled.
-	// +optional
-	Retry *Retry `json:"retry,omitempty"`
-
-	// UseClientProtocol configures Envoy to prefer sending requests to backends using
-	// the same HTTP protocol that the incoming request used. Defaults to false, which means
-	// that Envoy will use the protocol indicated by the attached BackendRef.
-	//
-	// +optional
-	UseClientProtocol *bool `json:"useClientProtocol,omitempty"`
-
 	// Timeout settings for the backend connections.
 	//
 	// +optional
 	Timeout *Timeout `json:"timeout,omitempty"`
 
-	// The compression config for the http streams.
-	//
-	// +optional
-	// +notImplementedHide
-	Compression []*Compression `json:"compression,omitempty"`
-
 	// Connection includes backend connection settings.
 	//
 	// +optional
 	Connection *BackendConnection `json:"connection,omitempty"`
+
 	// DNS includes dns resolution settings.
 	//
 	// +optional
@@ -121,27 +148,6 @@ type BackendTrafficPolicySpec struct {
 	HTTP2 *HTTP2Settings `json:"http2,omitempty"`
 }
 
-// +kubebuilder:object:root=true
-
-// BackendTrafficPolicyList contains a list of BackendTrafficPolicy resources.
-type BackendTrafficPolicyList struct {
-	metav1.TypeMeta `json:",inline"`
-	metav1.ListMeta `json:"metadata,omitempty"`
-	Items           []BackendTrafficPolicy `json:"items"`
-}
-
-// BackendTrafficPolicyConnection allows users to configure connection-level settings of backend
-type BackendTrafficPolicyConnection struct {
-	// BufferLimit Soft limit on size of the cluster’s connections read and write buffers.
-	// If unspecified, an implementation defined default is applied (32768 bytes).
-	// For example, 20Mi, 1Gi, 256Ki etc.
-	// Note: that when the suffix is not provided, the value is interpreted as bytes.
-	//
-	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="BufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
-	// +optional
-	BufferLimit *resource.Quantity `json:"bufferLimit,omitempty"`
-}
-
 func init() {
 	SchemeBuilder.Register(&BackendTrafficPolicy{}, &BackendTrafficPolicyList{})
 }
diff --git a/api/v1alpha1/connection_types.go b/api/v1alpha1/connection_types.go
@@ -22,15 +22,17 @@ type ClientConnection struct {
 	// Note that when the suffix is not provided, the value is interpreted as bytes.
 	// Default: 32768 bytes.
 	//
-	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="bufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
+	// +kubebuilder:validation:XIntOrString
+	// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
 	// +optional
 	BufferLimit *resource.Quantity `json:"bufferLimit,omitempty"`
 	// SocketBufferLimit provides configuration for the maximum buffer size in bytes for each incoming socket.
 	// SocketBufferLimit applies to socket streaming channel between TCP/IP stacks, it's in kernel space.
 	// For example, 20Mi, 1Gi, 256Ki etc.
 	// Note that when the suffix is not provided, the value is interpreted as bytes.
 	//
-	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="socketBufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
+	// +kubebuilder:validation:XIntOrString
+	// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
 	// +optional
 	// +notImplementedHide
 	SocketBufferLimit *resource.Quantity `json:"socketBufferLimit,omitempty"`
@@ -44,7 +46,8 @@ type BackendConnection struct {
 	// For example, 20Mi, 1Gi, 256Ki etc.
 	// Note: that when the suffix is not provided, the value is interpreted as bytes.
 	//
-	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="BufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
+	// +kubebuilder:validation:XIntOrString
+	// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
 	// +optional
 	BufferLimit *resource.Quantity `json:"bufferLimit,omitempty"`
 	// SocketBufferLimit provides configuration for the maximum buffer size in bytes for each socket
@@ -53,7 +56,8 @@ type BackendConnection struct {
 	// For example, 20Mi, 1Gi, 256Ki etc.
 	// Note that when the suffix is not provided, the value is interpreted as bytes.
 	//
-	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="socketBufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
+	// +kubebuilder:validation:XIntOrString
+	// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
 	// +optional
 	// +notImplementedHide
 	SocketBufferLimit *resource.Quantity `json:"socketBufferLimit,omitempty"`

diff --git a/api/v1alpha1/ext_auth_types.go b/api/v1alpha1/ext_auth_types.go
@@ -5,10 +5,6 @@
 
 package v1alpha1
 
-import (
-	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
-)
-
 // ExtAuth defines the configuration for External Authorization.
 //
 // +kubebuilder:validation:XValidation:rule="(has(self.grpc) || has(self.http))",message="one of grpc or http must be specified"
@@ -56,45 +52,23 @@ type ExtAuth struct {
 // The authorization request message is defined in
 // https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto
 // +kubebuilder:validation:XValidation:message="backendRef or backendRefs needs to be set",rule="has(self.backendRef) || self.backendRefs.size() > 0"
+// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core group.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.group == \"\") : true"
+// +kubebuilder:validation:XValidation:message="only support Service kind.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.kind == 'Service') : true"
+// +kubebuilder:validation:XValidation:message="only one backendRef can be specified.",rule="has(self.backendRefs) ? self.backendRefs.size() == 1 : true"
 type GRPCExtAuthService struct {
-	// BackendRef references a Kubernetes object that represents the
-	// backend server to which the authorization request will be sent.
-	// Only Service kind is supported for now.
-	//
-	// Deprecated: Use BackendRefs instead.
-	BackendRef *gwapiv1.BackendObjectReference `json:"backendRef,omitempty"`
-
-	// BackendRefs references a Kubernetes object that represents the
-	// backend server to which the authorization request will be sent.
 	// Only Service kind is supported for now.
-	//
-	// +optional
-	// +kubebuilder:validation:MaxItems=1
-	// +kubebuilder:validation:XValidation:message="only support Service kind.",rule="self.all(f, f.kind == 'Service')"
-	// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core group.",rule="self.all(f, f.group == '')"
-	BackendRefs []BackendRef `json:"backendRefs,omitempty"`
+	BackendCluster `json:",inline"`
 }
 
 // HTTPExtAuthService defines the HTTP External Authorization service
 //
 // +kubebuilder:validation:XValidation:message="backendRef or backendRefs needs to be set",rule="has(self.backendRef) || self.backendRefs.size() > 0"
+// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core group.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.group == \"\") : true"
+// +kubebuilder:validation:XValidation:message="only support Service kind.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.kind == 'Service') : true"
+// +kubebuilder:validation:XValidation:message="only one backendRef can be specified.",rule="has(self.backendRefs) ? self.backendRefs.size() == 1 : true"
 type HTTPExtAuthService struct {
-	// BackendRef references a Kubernetes object that represents the
-	// backend server to which the authorization request will be sent.
 	// Only Service kind is supported for now.
-	//
-	// Deprecated: Use BackendRefs instead.
-	BackendRef *gwapiv1.BackendObjectReference `json:"backendRef,omitempty"`
-
-	// BackendRefs references a Kubernetes object that represents the
-	// backend server to which the authorization request will be sent.
-	// Only Service kind is supported for now.
-	//
-	// +optional
-	// +kubebuilder:validation:MaxItems=1
-	// +kubebuilder:validation:XValidation:message="only support Service kind.",rule="self.all(f, f.kind == 'Service')"
-	// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core group.",rule="self.all(f, f.group == '')"
-	BackendRefs []BackendRef `json:"backendRefs,omitempty"`
+	BackendCluster `json:",inline"`
 
 	// Path is the path of the HTTP External Authorization service.
 	// If path is specified, the authorization request will be sent to that path,

diff --git a/api/v1alpha1/ext_proc_types.go b/api/v1alpha1/ext_proc_types.go
@@ -46,14 +46,12 @@ type ExtProcProcessingMode struct {
 }
 
 // ExtProc defines the configuration for External Processing filter.
+// +kubebuilder:validation:XValidation:message="BackendRefs must be used, backendRef is not supported.",rule="!has(self.backendRef)"
+// +kubebuilder:validation:XValidation:message="Exactly one backendRef can be specified in backendRefs.",rule="has(self.backendRefs) && self.backendRefs.size()==1"
+// +kubebuilder:validation:XValidation:message="BackendRefs only supports Service and Backend kind.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.kind == 'Service' || f.kind == 'Backend') : true"
+// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core and gateway.envoyproxy.io group.",rule="has(self.backendRefs) ? (self.backendRefs.all(f, f.group == \"\" || f.group == 'gateway.envoyproxy.io')) : true"
 type ExtProc struct {
-	// BackendRefs defines the configuration of the external processing service
-	//
-	// +kubebuilder:validation:MinItems=1
-	// +kubebuilder:validation:MaxItems=1
-	// +kubebuilder:validation:XValidation:message="BackendRefs only supports Service and Backend kind.",rule="self.all(f, f.kind == 'Service' || f.kind == 'Backend')"
-	// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core and gateway.envoyproxy.io group.",rule="self.all(f, f.group == '' || f.group == 'gateway.envoyproxy.io')"
-	BackendRefs []BackendRef `json:"backendRefs"`
+	BackendCluster `json:",inline"`
 
 	// MessageTimeout is the timeout for a response to be returned from the external processor
 	// Default: 200ms

diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go
@@ -475,6 +475,30 @@ type BackendRef struct {
 	gwapiv1.BackendObjectReference `json:",inline"`
 }
 
+// BackendCluster contains all the configuration required for configuring access
+// to a backend. This can include multiple endpoints, and settings that apply for
+// managing the connection to all these endpoints.
+type BackendCluster struct {
+	// BackendRef references a Kubernetes object that represents the
+	// backend server to which the authorization request will be sent.
+	//
+	// Deprecated: Use BackendRefs instead.
+	BackendRef *gwapiv1.BackendObjectReference `json:"backendRef,omitempty"`
+
+	// BackendRefs references a Kubernetes object that represents the
+	// backend server to which the authorization request will be sent.
+	//
+	// +kubebuilder:validation:MaxItems=16
+	// +optional
+	BackendRefs []BackendRef `json:"backendRefs,omitempty"`
+
+	// BackendSettings holds configuration for managing the connection
+	// to the backend.
+	//
+	// +optional
+	BackendSettings *ClusterSettings `json:"backendSettings,omitempty"`
+}
+
 // CIDR defines a CIDR Address range.
 // A CIDR can be an IPv4 address range such as "192.168.1.0/24" or an IPv6 address range such as "2001:0db8:11a3:09d7::/64".
 // +kubebuilder:validation:Pattern=`((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/([0-9]+))|((([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\/([0-9]+))`
@@ -492,14 +516,16 @@ type HTTP2Settings struct {
 	// InitialStreamWindowSize sets the initial window size for HTTP/2 streams.
 	// If not set, the default value is 64 KiB(64*1024).
 	//
-	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="initialStreamWindowSize must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
+	// +kubebuilder:validation:XIntOrString
+	// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
 	// +optional
 	InitialStreamWindowSize *resource.Quantity `json:"initialStreamWindowSize,omitempty"`
 
 	// InitialConnectionWindowSize sets the initial window size for HTTP/2 connections.
 	// If not set, the default value is 1 MiB.
 	//
-	// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="initialConnectionWindowSize must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
+	// +kubebuilder:validation:XIntOrString
+	// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
 	// +optional
 	InitialConnectionWindowSize *resource.Quantity `json:"initialConnectionWindowSize,omitempty"`