Skip to content

Commit

Permalink
feat: continue using and drain serving endpointslices during termination
Browse files Browse the repository at this point in the history 
Signed-off-by: Daniele Fognini <[email protected]>
  • Loading branch information
@fogninid @zirain
fogninid authored and zirain committed Jan 11, 2025
1 parent 8dbe6e0 commit 2e20569
Show file tree
Hide file tree
Showing 203 changed files with 928 additions and 455 deletions.
3 changes: 2 additions & 1 deletion internal/cmd/egctl/testdata/translate/out/quickstart.all.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@ xdsIR:
name: httproute/envoy-gateway-system/backend/rule/0
settings:
- endpoints:
- host: 7.7.7.7
- draining: false
host: 7.7.7.7
port: 3000
protocol: HTTP
weight: 1
Expand Down
2 changes: 1 addition & 1 deletion internal/gatewayapi/listener.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -541,7 +541,7 @@ func destinationSettingFromHostAndPort(host string, port uint32) []*ir.Destinati
{
Weight: ptr.To[uint32](1),
Protocol: ir.GRPC,
Endpoints: []*ir.DestinationEndpoint{ir.NewDestEndpoint(host, port)},
Endpoints: []*ir.DestinationEndpoint{ir.NewDestEndpoint(host, port, false)},
},
}
}
Expand Down
36 changes: 20 additions & 16 deletions internal/gatewayapi/route.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1280,9 +1280,7 @@ func (t *Translator) processDestination(backendRefContext BackendRefContext,
} else {
backendIps := resources.GetServiceImport(backendNamespace, string(backendRef.Name)).Spec.IPs
for _, ip := range backendIps {
ep := ir.NewDestEndpoint(
ip,
uint32(*backendRef.Port))
ep := ir.NewDestEndpoint(ip, uint32(*backendRef.Port), false)

Check warning on line 1283 in internal/gatewayapi/route.go

View check run for this annotation

Codecov / codecov/patch

internal/gatewayapi/route.go#L1283 

Added line #L1283 was not covered by tests
endpoints = append(endpoints, ep)
}
}
Expand Down Expand Up @@ -1412,9 +1410,7 @@ func (t *Translator) processServiceDestinationSetting(
endpoints, addrType = getIREndpointsFromEndpointSlices(endpointSlices, servicePort.Name, servicePort.Protocol)
} else {
// Fall back to Service ClusterIP routing
ep := ir.NewDestEndpoint(
service.Spec.ClusterIP,
uint32(*backendRef.Port))
ep := ir.NewDestEndpoint(service.Spec.ClusterIP, uint32(*backendRef.Port), false)
endpoints = append(endpoints, ep)
}

Expand Down Expand Up @@ -1625,17 +1621,25 @@ func getIREndpointsFromEndpointSlice(endpointSlice *discoveryv1.EndpointSlice, p
for _, endpoint := range endpointSlice.Endpoints {
for _, endpointPort := range endpointSlice.Ports {
// Check if the endpoint port matches the service port
// and if endpoint is Ready
if *endpointPort.Name == portName &&
*endpointPort.Protocol == portProtocol &&
// Unknown state (nil) should be interpreted as Ready, see https://pkg.go.dev/k8s.io/api/discovery/v1#EndpointConditions
(endpoint.Conditions.Ready == nil || *endpoint.Conditions.Ready) {
for _, address := range endpoint.Addresses {
ep := ir.NewDestEndpoint(
address,
uint32(*endpointPort.Port))
endpoints = append(endpoints, ep)
if *endpointPort.Name != portName || *endpointPort.Protocol != portProtocol {
continue
}
conditions := endpoint.Conditions
draining := false
// Unknown state (nil) should be interpreted as Ready, see https://pkg.go.dev/k8s.io/api/discovery/v1#EndpointConditions
switch {
case conditions.Serving == nil || conditions.Terminating == nil:
if conditions.Ready != nil && !*conditions.Ready {
continue
}
case !*conditions.Serving:
continue
default:
draining = *conditions.Terminating
}
for _, address := range endpoint.Addresses {
ep := ir.NewDestEndpoint(address, uint32(*endpointPort.Port), draining)
endpoints = append(endpoints, ep)
}
}
}
Expand Down
83 changes: 66 additions & 17 deletions internal/gatewayapi/route_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"fmt"
"testing"

"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -23,7 +24,7 @@ func TestGetIREndpointsFromEndpointSlices(t *testing.T) {
endpointSlices []*discoveryv1.EndpointSlice
portName string
portProtocol corev1.Protocol
expectedEndpoints int
expectedEndpoints []*ir.DestinationEndpoint
expectedAddrType ir.DestinationAddressType
}{
{
Expand Down Expand Up @@ -51,10 +52,14 @@ func TestGetIREndpointsFromEndpointSlices(t *testing.T) {
},
},
},
portName: "http",
portProtocol: corev1.ProtocolTCP,
expectedEndpoints: 3,
expectedAddrType: ir.IP,
portName: "http",
portProtocol: corev1.ProtocolTCP,
expectedEndpoints: []*ir.DestinationEndpoint{
{Host: "192.0.2.1", Port: 80, Draining: false},
{Host: "192.0.2.2", Port: 80, Draining: false},
{Host: "2001:db8::1", Port: 80, Draining: false},
},
expectedAddrType: ir.IP,
},
{
name: "Mixed IP and FQDN endpoints",
Expand All @@ -80,10 +85,13 @@ func TestGetIREndpointsFromEndpointSlices(t *testing.T) {
},
},
},
portName: "http",
portProtocol: corev1.ProtocolTCP,
expectedEndpoints: 2,
expectedAddrType: ir.MIXED,
portName: "http",
portProtocol: corev1.ProtocolTCP,
expectedEndpoints: []*ir.DestinationEndpoint{
{Host: "192.0.2.1", Port: 80, Draining: false},
{Host: "example.com", Port: 80, Draining: false},
},
expectedAddrType: ir.MIXED,
},
{
name: "Dual-stack IP endpoints",
Expand Down Expand Up @@ -111,10 +119,15 @@ func TestGetIREndpointsFromEndpointSlices(t *testing.T) {
},
},
},
portName: "http",
portProtocol: corev1.ProtocolTCP,
expectedEndpoints: 4,
expectedAddrType: ir.IP,
portName: "http",
portProtocol: corev1.ProtocolTCP,
expectedEndpoints: []*ir.DestinationEndpoint{
{Host: "192.0.2.1", Port: 80, Draining: false},
{Host: "192.0.2.2", Port: 80, Draining: false},
{Host: "2001:db8::1", Port: 80, Draining: false},
{Host: "2001:db8::2", Port: 80, Draining: false},
},
expectedAddrType: ir.IP,
},
{
name: "Dual-stack with FQDN",
Expand Down Expand Up @@ -150,10 +163,43 @@ func TestGetIREndpointsFromEndpointSlices(t *testing.T) {
},
},
},
portName: "http",
portProtocol: corev1.ProtocolTCP,
expectedEndpoints: 3,
expectedAddrType: ir.MIXED,
portName: "http",
portProtocol: corev1.ProtocolTCP,
expectedEndpoints: []*ir.DestinationEndpoint{
{Host: "192.0.2.1", Port: 80, Draining: false},
{Host: "2001:db8::1", Port: 80, Draining: false},
{Host: "example.com", Port: 80, Draining: false},
},
expectedAddrType: ir.MIXED,
},
{
name: "Keep serving and terminating as draining",
endpointSlices: []*discoveryv1.EndpointSlice{
{
ObjectMeta: metav1.ObjectMeta{Name: "slice1"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{
{Addresses: []string{"192.0.2.1"}, Conditions: discoveryv1.EndpointConditions{
Ready: ptr.To(false), Serving: ptr.To(true), Terminating: ptr.To(true),
}},
{Addresses: []string{"192.0.2.2"}, Conditions: discoveryv1.EndpointConditions{
Ready: ptr.To(false), Serving: ptr.To(false), Terminating: ptr.To(true),
}},
{Addresses: []string{"192.0.2.3"}, Conditions: discoveryv1.EndpointConditions{
Ready: ptr.To(false),
}},
},
Ports: []discoveryv1.EndpointPort{
{Name: ptr.To("http"), Port: ptr.To(int32(80)), Protocol: ptr.To(corev1.ProtocolTCP)},
},
},
},
portName: "http",
portProtocol: corev1.ProtocolTCP,
expectedEndpoints: []*ir.DestinationEndpoint{
{Host: "192.0.2.1", Port: 80, Draining: true},
},
expectedAddrType: ir.IP,
},
}

Expand All @@ -170,10 +216,13 @@ func TestGetIREndpointsFromEndpointSlices(t *testing.T) {
fmt.Printf(" Endpoint %d:\n", i+1)
fmt.Printf(" Address: %s\n", endpoint.Host)
fmt.Printf(" Port: %d\n", endpoint.Port)
fmt.Printf(" Draining: %t\n", endpoint.Draining)

}

fmt.Println()
require.Equal(t, tt.expectedEndpoints, endpoints)
require.Equal(t, tt.expectedAddrType, *addrType)
})
}
}
6 changes: 4 additions & 2 deletions internal/gatewayapi/testdata/backend-with-fallback.out.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,12 +158,14 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 1.1.1.1
- draining: false
host: 1.1.1.1
port: 3001
weight: 1
- addressType: IP
endpoints:
- host: 2.2.2.2
- draining: false
host: 2.2.2.2
port: 3001
priority: 1
weight: 1
Expand Down
3 changes: 2 additions & 1 deletion internal/gatewayapi/testdata/backendtlspolicy-across-ns.out.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 10.244.0.11
- draining: false
host: 10.244.0.11
port: 8080
protocol: HTTP
weight: 1
Expand Down
3 changes: 2 additions & 1 deletion internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.out.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 10.244.0.11
- draining: false
host: 10.244.0.11
port: 8080
protocol: HTTP
tls:
Expand Down
3 changes: 2 additions & 1 deletion internal/gatewayapi/testdata/backendtlspolicy-ca-only.out.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 10.244.0.11
- draining: false
host: 10.244.0.11
port: 8080
protocol: HTTP
tls:
Expand Down
12 changes: 8 additions & 4 deletions internal/gatewayapi/testdata/backendtlspolicy-default-ns-targetrefs.out.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 10.244.0.11
- draining: false
host: 10.244.0.11
port: 8080
protocol: HTTP
tls:
Expand All @@ -298,7 +299,8 @@ xdsIR:
weight: 1
- addressType: IP
endpoints:
- host: 2.2.2.2
- draining: false
host: 2.2.2.2
port: 3443
tls:
alpnProtocols: null
Expand Down Expand Up @@ -343,7 +345,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 10.244.0.11
- draining: false
host: 10.244.0.11
port: 8080
protocol: HTTP
tls:
Expand All @@ -355,7 +358,8 @@ xdsIR:
weight: 1
- addressType: IP
endpoints:
- host: 2.2.2.2
- draining: false
host: 2.2.2.2
port: 3443
tls:
alpnProtocols: null
Expand Down
9 changes: 6 additions & 3 deletions internal/gatewayapi/testdata/backendtlspolicy-default-ns.out.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 10.244.0.11
- draining: false
host: 10.244.0.11
port: 8080
protocol: HTTP
tls:
Expand All @@ -259,7 +260,8 @@ xdsIR:
weight: 1
- addressType: IP
endpoints:
- host: 2.2.2.2
- draining: false
host: 2.2.2.2
port: 3443
tls:
alpnProtocols: null
Expand All @@ -270,7 +272,8 @@ xdsIR:
weight: 1
- addressType: IP
endpoints:
- host: 3.3.3.3
- draining: false
host: 3.3.3.3
port: 3443
weight: 1
hostname: '*'
Expand Down
3 changes: 2 additions & 1 deletion internal/gatewayapi/testdata/backendtlspolicy-system-truststore.out.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 10.244.0.11
- draining: false
host: 10.244.0.11
port: 8080
protocol: HTTP
tls:
Expand Down
6 changes: 4 additions & 2 deletions internal/gatewayapi/testdata/backendtrafficpolicy-buffer-limit-out-of-range-error.out.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 7.7.7.7
- draining: false
host: 7.7.7.7
port: 8080
protocol: GRPC
weight: 1
Expand Down Expand Up @@ -312,7 +313,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 7.7.7.7
- draining: false
host: 7.7.7.7
port: 8080
protocol: HTTP
weight: 1
Expand Down
6 changes: 4 additions & 2 deletions internal/gatewayapi/testdata/backendtrafficpolicy-buffer-limit-with-invalid-value.out.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 7.7.7.7
- draining: false
host: 7.7.7.7
port: 8080
protocol: GRPC
weight: 1
Expand Down Expand Up @@ -312,7 +313,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 7.7.7.7
- draining: false
host: 7.7.7.7
port: 8080
protocol: HTTP
weight: 1
Expand Down
6 changes: 4 additions & 2 deletions internal/gatewayapi/testdata/backendtrafficpolicy-buffer-limit.out.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 7.7.7.7
- draining: false
host: 7.7.7.7
port: 8080
protocol: GRPC
weight: 1
Expand Down Expand Up @@ -312,7 +313,8 @@ xdsIR:
settings:
- addressType: IP
endpoints:
- host: 7.7.7.7
- draining: false
host: 7.7.7.7
port: 8080
protocol: HTTP
weight: 1
Expand Down
Loading

0 comments on commit 2e20569

Please sign in to comment.