Skip to content

Commit

Permalink
feat: add support for cluster-level settings for non xRoute-generated…
Browse files Browse the repository at this point in the history
… backend refs (envoyproxy#3954)

* Add support for traffic management features for non xRoute-generated
clusters.

Signed-off-by: Lior Okman <[email protected]>

* Make gen-check happy

Signed-off-by: Lior Okman <[email protected]>

* Update the CEL configuration and tests to make the required tests be
installable. Unlimited arrays cause CEL validation costs to be higher
than what the API server will accept.

Signed-off-by: Lior Okman <[email protected]>

* Added some tests

Signed-off-by: Lior Okman <[email protected]>

* DNS is also a feature that can be enabled for extProc and extAuth
clusters, so treat it as such.

Signed-off-by: Lior Okman <[email protected]>

* Renamed BackendConfig to BackendSettings

Signed-off-by: Lior Okman <[email protected]>

* make gen-check happy

Signed-off-by: Lior Okman <[email protected]>

* Rebase to make gen-check happy again.

Signed-off-by: Lior Okman <[email protected]>

* Translate DNS settings correctly.

Signed-off-by: Lior Okman <[email protected]>

* Rebased to include upstream changes.

Update the PR to include HTTP2 settings as well.

Signed-off-by: Lior Okman <[email protected]>

* Fix a typo.

Signed-off-by: Lior Okman <[email protected]>

---------

Signed-off-by: Lior Okman <[email protected]>
  • Loading branch information
liorokman authored Aug 6, 2024
1 parent 3497600 commit 866a6ce
Show file tree
Hide file tree
Showing 39 changed files with 4,362 additions and 1,110 deletions.
94 changes: 50 additions & 44 deletions api/v1alpha1/backendtrafficpolicy_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,61 @@ type BackendTrafficPolicy struct {
// BackendTrafficPolicySpec defines the desired state of BackendTrafficPolicy.
type BackendTrafficPolicySpec struct {
PolicyTargetReferences `json:",inline"`
ClusterSettings `json:",inline"`

// RateLimit allows the user to limit the number of incoming requests
// to a predefined value based on attributes within the traffic flow.
// +optional
RateLimit *RateLimitSpec `json:"rateLimit,omitempty"`

// FaultInjection defines the fault injection policy to be applied. This configuration can be used to
// inject delays and abort requests to mimic failure scenarios such as service failures and overloads
// +optional
FaultInjection *FaultInjection `json:"faultInjection,omitempty"`

// Retry provides more advanced usage, allowing users to customize the number of retries, retry fallback strategy, and retry triggering conditions.
// If not set, retry will be disabled.
// +optional
Retry *Retry `json:"retry,omitempty"`

// UseClientProtocol configures Envoy to prefer sending requests to backends using
// the same HTTP protocol that the incoming request used. Defaults to false, which means
// that Envoy will use the protocol indicated by the attached BackendRef.
//
// +optional
UseClientProtocol *bool `json:"useClientProtocol,omitempty"`

// The compression config for the http streams.
//
// +optional
// +notImplementedHide
Compression []*Compression `json:"compression,omitempty"`
}

// +kubebuilder:object:root=true

// BackendTrafficPolicyList contains a list of BackendTrafficPolicy resources.
type BackendTrafficPolicyList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []BackendTrafficPolicy `json:"items"`
}

// BackendTrafficPolicyConnection allows users to configure connection-level settings of backend
type BackendTrafficPolicyConnection struct {
// BufferLimit Soft limit on size of the cluster’s connections read and write buffers.
// If unspecified, an implementation defined default is applied (32768 bytes).
// For example, 20Mi, 1Gi, 256Ki etc.
// Note: that when the suffix is not provided, the value is interpreted as bytes.
//
// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="BufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
// +optional
BufferLimit *resource.Quantity `json:"bufferLimit,omitempty"`
}

// ClusterSettings provides the various knobs that can be set to control how traffic to a given
// backend will be configured.
type ClusterSettings struct {
// LoadBalancer policy to apply when routing traffic from the gateway to
// the backend endpoints
// +optional
Expand All @@ -72,44 +121,22 @@ type BackendTrafficPolicySpec struct {
// +optional
HealthCheck *HealthCheck `json:"healthCheck,omitempty"`

// FaultInjection defines the fault injection policy to be applied. This configuration can be used to
// inject delays and abort requests to mimic failure scenarios such as service failures and overloads
// +optional
FaultInjection *FaultInjection `json:"faultInjection,omitempty"`

// Circuit Breaker settings for the upstream connections and requests.
// If not set, circuit breakers will be enabled with the default thresholds
//
// +optional
CircuitBreaker *CircuitBreaker `json:"circuitBreaker,omitempty"`

// Retry provides more advanced usage, allowing users to customize the number of retries, retry fallback strategy, and retry triggering conditions.
// If not set, retry will be disabled.
// +optional
Retry *Retry `json:"retry,omitempty"`

// UseClientProtocol configures Envoy to prefer sending requests to backends using
// the same HTTP protocol that the incoming request used. Defaults to false, which means
// that Envoy will use the protocol indicated by the attached BackendRef.
//
// +optional
UseClientProtocol *bool `json:"useClientProtocol,omitempty"`

// Timeout settings for the backend connections.
//
// +optional
Timeout *Timeout `json:"timeout,omitempty"`

// The compression config for the http streams.
//
// +optional
// +notImplementedHide
Compression []*Compression `json:"compression,omitempty"`

// Connection includes backend connection settings.
//
// +optional
Connection *BackendConnection `json:"connection,omitempty"`

// DNS includes dns resolution settings.
//
// +optional
Expand All @@ -121,27 +148,6 @@ type BackendTrafficPolicySpec struct {
HTTP2 *HTTP2Settings `json:"http2,omitempty"`
}

// +kubebuilder:object:root=true

// BackendTrafficPolicyList contains a list of BackendTrafficPolicy resources.
type BackendTrafficPolicyList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []BackendTrafficPolicy `json:"items"`
}

// BackendTrafficPolicyConnection allows users to configure connection-level settings of backend
type BackendTrafficPolicyConnection struct {
// BufferLimit Soft limit on size of the cluster’s connections read and write buffers.
// If unspecified, an implementation defined default is applied (32768 bytes).
// For example, 20Mi, 1Gi, 256Ki etc.
// Note: that when the suffix is not provided, the value is interpreted as bytes.
//
// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="BufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
// +optional
BufferLimit *resource.Quantity `json:"bufferLimit,omitempty"`
}

func init() {
SchemeBuilder.Register(&BackendTrafficPolicy{}, &BackendTrafficPolicyList{})
}
12 changes: 8 additions & 4 deletions api/v1alpha1/connection_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,17 @@ type ClientConnection struct {
// Note that when the suffix is not provided, the value is interpreted as bytes.
// Default: 32768 bytes.
//
// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="bufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
// +kubebuilder:validation:XIntOrString
// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
// +optional
BufferLimit *resource.Quantity `json:"bufferLimit,omitempty"`
// SocketBufferLimit provides configuration for the maximum buffer size in bytes for each incoming socket.
// SocketBufferLimit applies to socket streaming channel between TCP/IP stacks, it's in kernel space.
// For example, 20Mi, 1Gi, 256Ki etc.
// Note that when the suffix is not provided, the value is interpreted as bytes.
//
// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="socketBufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
// +kubebuilder:validation:XIntOrString
// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
// +optional
// +notImplementedHide
SocketBufferLimit *resource.Quantity `json:"socketBufferLimit,omitempty"`
Expand All @@ -44,7 +46,8 @@ type BackendConnection struct {
// For example, 20Mi, 1Gi, 256Ki etc.
// Note: that when the suffix is not provided, the value is interpreted as bytes.
//
// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="BufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
// +kubebuilder:validation:XIntOrString
// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
// +optional
BufferLimit *resource.Quantity `json:"bufferLimit,omitempty"`
// SocketBufferLimit provides configuration for the maximum buffer size in bytes for each socket
Expand All @@ -53,7 +56,8 @@ type BackendConnection struct {
// For example, 20Mi, 1Gi, 256Ki etc.
// Note that when the suffix is not provided, the value is interpreted as bytes.
//
// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="socketBufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
// +kubebuilder:validation:XIntOrString
// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
// +optional
// +notImplementedHide
SocketBufferLimit *resource.Quantity `json:"socketBufferLimit,omitempty"`
Expand Down
42 changes: 8 additions & 34 deletions api/v1alpha1/ext_auth_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,6 @@

package v1alpha1

import (
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
)

// ExtAuth defines the configuration for External Authorization.
//
// +kubebuilder:validation:XValidation:rule="(has(self.grpc) || has(self.http))",message="one of grpc or http must be specified"
Expand Down Expand Up @@ -56,45 +52,23 @@ type ExtAuth struct {
// The authorization request message is defined in
// https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto
// +kubebuilder:validation:XValidation:message="backendRef or backendRefs needs to be set",rule="has(self.backendRef) || self.backendRefs.size() > 0"
// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core group.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.group == \"\") : true"
// +kubebuilder:validation:XValidation:message="only support Service kind.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.kind == 'Service') : true"
// +kubebuilder:validation:XValidation:message="only one backendRef can be specified.",rule="has(self.backendRefs) ? self.backendRefs.size() == 1 : true"
type GRPCExtAuthService struct {
// BackendRef references a Kubernetes object that represents the
// backend server to which the authorization request will be sent.
// Only Service kind is supported for now.
//
// Deprecated: Use BackendRefs instead.
BackendRef *gwapiv1.BackendObjectReference `json:"backendRef,omitempty"`

// BackendRefs references a Kubernetes object that represents the
// backend server to which the authorization request will be sent.
// Only Service kind is supported for now.
//
// +optional
// +kubebuilder:validation:MaxItems=1
// +kubebuilder:validation:XValidation:message="only support Service kind.",rule="self.all(f, f.kind == 'Service')"
// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core group.",rule="self.all(f, f.group == '')"
BackendRefs []BackendRef `json:"backendRefs,omitempty"`
BackendCluster `json:",inline"`
}

// HTTPExtAuthService defines the HTTP External Authorization service
//
// +kubebuilder:validation:XValidation:message="backendRef or backendRefs needs to be set",rule="has(self.backendRef) || self.backendRefs.size() > 0"
// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core group.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.group == \"\") : true"
// +kubebuilder:validation:XValidation:message="only support Service kind.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.kind == 'Service') : true"
// +kubebuilder:validation:XValidation:message="only one backendRef can be specified.",rule="has(self.backendRefs) ? self.backendRefs.size() == 1 : true"
type HTTPExtAuthService struct {
// BackendRef references a Kubernetes object that represents the
// backend server to which the authorization request will be sent.
// Only Service kind is supported for now.
//
// Deprecated: Use BackendRefs instead.
BackendRef *gwapiv1.BackendObjectReference `json:"backendRef,omitempty"`

// BackendRefs references a Kubernetes object that represents the
// backend server to which the authorization request will be sent.
// Only Service kind is supported for now.
//
// +optional
// +kubebuilder:validation:MaxItems=1
// +kubebuilder:validation:XValidation:message="only support Service kind.",rule="self.all(f, f.kind == 'Service')"
// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core group.",rule="self.all(f, f.group == '')"
BackendRefs []BackendRef `json:"backendRefs,omitempty"`
BackendCluster `json:",inline"`

// Path is the path of the HTTP External Authorization service.
// If path is specified, the authorization request will be sent to that path,
Expand Down
12 changes: 5 additions & 7 deletions api/v1alpha1/ext_proc_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,12 @@ type ExtProcProcessingMode struct {
}

// ExtProc defines the configuration for External Processing filter.
// +kubebuilder:validation:XValidation:message="BackendRefs must be used, backendRef is not supported.",rule="!has(self.backendRef)"
// +kubebuilder:validation:XValidation:message="Exactly one backendRef can be specified in backendRefs.",rule="has(self.backendRefs) && self.backendRefs.size()==1"
// +kubebuilder:validation:XValidation:message="BackendRefs only supports Service and Backend kind.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.kind == 'Service' || f.kind == 'Backend') : true"
// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core and gateway.envoyproxy.io group.",rule="has(self.backendRefs) ? (self.backendRefs.all(f, f.group == \"\" || f.group == 'gateway.envoyproxy.io')) : true"
type ExtProc struct {
// BackendRefs defines the configuration of the external processing service
//
// +kubebuilder:validation:MinItems=1
// +kubebuilder:validation:MaxItems=1
// +kubebuilder:validation:XValidation:message="BackendRefs only supports Service and Backend kind.",rule="self.all(f, f.kind == 'Service' || f.kind == 'Backend')"
// +kubebuilder:validation:XValidation:message="BackendRefs only supports Core and gateway.envoyproxy.io group.",rule="self.all(f, f.group == '' || f.group == 'gateway.envoyproxy.io')"
BackendRefs []BackendRef `json:"backendRefs"`
BackendCluster `json:",inline"`

// MessageTimeout is the timeout for a response to be returned from the external processor
// Default: 200ms
Expand Down
30 changes: 28 additions & 2 deletions api/v1alpha1/shared_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,30 @@ type BackendRef struct {
gwapiv1.BackendObjectReference `json:",inline"`
}

// BackendCluster contains all the configuration required for configuring access
// to a backend. This can include multiple endpoints, and settings that apply for
// managing the connection to all these endpoints.
type BackendCluster struct {
// BackendRef references a Kubernetes object that represents the
// backend server to which the authorization request will be sent.
//
// Deprecated: Use BackendRefs instead.
BackendRef *gwapiv1.BackendObjectReference `json:"backendRef,omitempty"`

// BackendRefs references a Kubernetes object that represents the
// backend server to which the authorization request will be sent.
//
// +kubebuilder:validation:MaxItems=16
// +optional
BackendRefs []BackendRef `json:"backendRefs,omitempty"`

// BackendSettings holds configuration for managing the connection
// to the backend.
//
// +optional
BackendSettings *ClusterSettings `json:"backendSettings,omitempty"`
}

// CIDR defines a CIDR Address range.
// A CIDR can be an IPv4 address range such as "192.168.1.0/24" or an IPv6 address range such as "2001:0db8:11a3:09d7::/64".
// +kubebuilder:validation:Pattern=`((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/([0-9]+))|((([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\/([0-9]+))`
Expand All @@ -492,14 +516,16 @@ type HTTP2Settings struct {
// InitialStreamWindowSize sets the initial window size for HTTP/2 streams.
// If not set, the default value is 64 KiB(64*1024).
//
// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="initialStreamWindowSize must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
// +kubebuilder:validation:XIntOrString
// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
// +optional
InitialStreamWindowSize *resource.Quantity `json:"initialStreamWindowSize,omitempty"`

// InitialConnectionWindowSize sets the initial window size for HTTP/2 connections.
// If not set, the default value is 1 MiB.
//
// +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="initialConnectionWindowSize must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\""
// +kubebuilder:validation:XIntOrString
// +kubebuilder:validation:Pattern="^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$"
// +optional
InitialConnectionWindowSize *resource.Quantity `json:"initialConnectionWindowSize,omitempty"`

Expand Down
Loading

0 comments on commit 866a6ce

Please sign in to comment.