From 7f5b913105ebbfb88e860f9aa30bf3e78d18d92a Mon Sep 17 00:00:00 2001 From: Lai Jiang Date: Wed, 18 Dec 2024 13:30:18 -0500 Subject: [PATCH] Expose EPP and WHOIS endpoints on reginal load balancers k8s does not have a way to expose a global load balancer with TCP endpoints, and setting up node port-based routing is a chore, even with Terraform (which is what we did with the standalone proxy). We will use Cloud DNS's geolocation routing policy to ensure that clients connect to the endpoint closest to them. --- jetty/build.gradle | 5 + jetty/deploy-nomulus-for-env.sh | 6 +- jetty/get-endpoints.py | 161 +++++++++++++++++++++++++ jetty/kubernetes/nomulus-frontend.yaml | 27 ++++- jetty/kubernetes/nomulus-pubapi.yaml | 27 ++++- 5 files changed, 214 insertions(+), 12 deletions(-) create mode 100755 jetty/get-endpoints.py diff --git a/jetty/build.gradle b/jetty/build.gradle index 0b93a173fb9..7946db043c3 100644 --- a/jetty/build.gradle +++ b/jetty/build.gradle @@ -116,4 +116,9 @@ tasks.register('deployNomulus', Exec) { commandLine './deploy-nomulus-for-env.sh', "${rootProject.environment}", "${rootProject.baseDomain}" } +tasks.register('getEndpoints', Exec) { + configure verifyDeploymentConfig + commandLine './get-endpoints.py', "${rootProject.gcpProject}" +} + project.build.dependsOn(tasks.named('buildNomulusImage')) diff --git a/jetty/deploy-nomulus-for-env.sh b/jetty/deploy-nomulus-for-env.sh index 1de2156802d..bf0e8c47e4d 100755 --- a/jetty/deploy-nomulus-for-env.sh +++ b/jetty/deploy-nomulus-for-env.sh @@ -37,13 +37,15 @@ do sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \ sed s/ENVIRONMENT/"${environment}"/g | \ sed s/PROXY_ENV/"${environment}"/g | \ - sed s/PROXY_NAME/"proxy"/g | \ + sed s/EPP/"epp"/g | \ + sed s/WHOIS/"whois"/g | \ kubectl apply -f - # canary sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \ sed s/ENVIRONMENT/"${environment}"/g | \ sed s/PROXY_ENV/"${environment}_canary"/g | \ - sed s/PROXY_NAME/"proxy-canary"/g | \ + sed s/EPP/"epp-canary"/g | \ + sed s/WHOIS/"whois-canary"/g | \ sed s/"${service}"/"${service}-canary"/g | \ kubectl apply -f - done diff --git a/jetty/get-endpoints.py b/jetty/get-endpoints.py new file mode 100755 index 00000000000..8f2fb1e07a0 --- /dev/null +++ b/jetty/get-endpoints.py @@ -0,0 +1,161 @@ +#! /bin/env python3 +# Copyright 2024 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +A script that outputs the IP endpoints of various load balancers, to be run +after Nomulus is deployed. +''' + +import ipaddress +import json +import subprocess +import sys +from dataclasses import dataclass +from ipaddress import IPv4Address +from ipaddress import IPv6Address +from operator import attrgetter +from operator import methodcaller + + +class PreserveContext: + def __enter__(self): + self._context = run_command('kubectl config current-context') + + def __exit__(self, type, value, traceback): + run_command('kubectl config use-context ' + self._context) + + +class UseCluster(PreserveContext): + def __init__(self, cluster: str, region: str, project: str): + self._cluster = cluster + self._region = region + self._project = project + + def __enter__(self): + super().__enter__() + cmd = f'gcloud container clusters get-credentials {self._cluster} --location {self._region} --project {self._project}' + run_command(cmd) + + +def run_command(cmd: str, print_output=False) -> str: + proc = subprocess.run(cmd, text=True, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if print_output: + print(proc.stdout) + return proc.stdout + + +def get_clusters(project: str) -> dict[str, str]: + cmd = f'gcloud container clusters list --project {project} --format=json' + content = json.loads(run_command(cmd)) + res = {} + for item in content: + name = item['name'] + region = item['location'] + if not name.startswith('nomulus-cluster'): + continue + res[name] = region + return res + + +def get_endpoints(resource: str, service: str, selector: list[str]) -> list[ + str]: + content = json.loads( + run_command(f'kubectl get {resource}/{service} -o json')) + ips = content + for key in selector[:-1]: + ips = ips[key] + return [x[selector[-1]] for x in ips] + + +def get_region_symbol(region: str) -> str: + if region.startswith('us'): + return 'amer' + if region.startswith('europe'): + return 'emea' + if region.startswith('asia'): + return 'apac' + return 'other' + + +@dataclass +class IP: + service: str + region: str + address: IPv4Address | IPv6Address + + def is_ipv6(self) -> bool: + return self.address.version == 6 + + def __str__(self) -> str: + return f'{self.service} {self.region}: {self.address}' + + +def terraform_str(item) -> str: + res = "" + if (isinstance(item, dict)): + res += '{\n' + for key, value in item.items(): + res += f'{key} = {terraform_str(value)}\n' + res += '}' + elif (isinstance(item, list)): + res += '[' + for i, value in enumerate(item): + if i != 0: + res += ', ' + res += terraform_str(value) + res += ']' + else: + res += f'"{item}"' + return res + + +if __name__ == '__main__': + if len(sys.argv) != 2: + raise ValueError('Usage: get-endpoints.py ') + project = sys.argv[1] + print(f'Project: {project}') + clusters = get_clusters(project) + ips = [] + res = {} + for cluster, region in clusters.items(): + with UseCluster(cluster, region, project): + for service in ['whois', 'whois-canary', 'epp', 'epp-canary']: + map_key = service.replace('-', '_') + for ip in get_endpoints('services', service, + ['status', 'loadBalancer', 'ingress', + 'ip']): + ip = ipaddress.ip_address(ip) + if isinstance(ip, IPv4Address): + map_key_with_iptype = map_key + '_ipv4' + else: + map_key_with_iptype = map_key + '_ipv6' + if map_key_with_iptype not in res: + res[map_key_with_iptype] = {} + res[map_key_with_iptype][get_region_symbol(region)] = [ip] + ips.append(IP(service, get_region_symbol(region), ip)) + if not region.startswith('us'): + continue + ip = get_endpoints('gateways.gateway.networking.k8s.io', 'nomulus', + ['status', 'addresses', 'value'])[0] + print(f'nomulus: {ip}') + res['https_ip'] = ipaddress.ip_address(ip) + ips.sort(key=attrgetter('region')) + ips.sort(key=methodcaller('is_ipv6')) + ips.sort(key=attrgetter('service')) + for ip in ips: + print(ip) + print("Terraform friendly output:") + print(terraform_str(res)) diff --git a/jetty/kubernetes/nomulus-frontend.yaml b/jetty/kubernetes/nomulus-frontend.yaml index 460ba7a3da6..aa950d78874 100644 --- a/jetty/kubernetes/nomulus-frontend.yaml +++ b/jetty/kubernetes/nomulus-frontend.yaml @@ -33,7 +33,7 @@ spec: fieldPath: metadata.namespace - name: CONTAINER_NAME value: frontend - - name: PROXY_NAME + - name: EPP image: gcr.io/GCP_PROJECT/proxy ports: - containerPort: 30002 @@ -52,7 +52,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CONTAINER_NAME - value: PROXY_NAME + value: EPP --- # Only need to define the service account once per cluster. apiVersion: v1 @@ -92,9 +92,26 @@ spec: - port: 80 targetPort: http name: http - - port: 700 - targetPort: epp - name: epp +--- +apiVersion: v1 +kind: Service +metadata: + name: EPP + annotations: + cloud.google.com/l4-rbs: enabled + networking.gke.io/weighted-load-balancing: pods-per-node +spec: + type: LoadBalancer + # Traffic is directly delivered to a node, preserving the original source IP. + externalTrafficPolicy: Local + ipFamilies: [IPv4, IPv6] + ipFamilyPolicy: RequireDualStack + selector: + service: frontend + ports: + - port: 700 + targetPort: epp + name: epp --- apiVersion: net.gke.io/v1 kind: ServiceExport diff --git a/jetty/kubernetes/nomulus-pubapi.yaml b/jetty/kubernetes/nomulus-pubapi.yaml index 42d3daa3bb9..579c7d4cb20 100644 --- a/jetty/kubernetes/nomulus-pubapi.yaml +++ b/jetty/kubernetes/nomulus-pubapi.yaml @@ -33,7 +33,7 @@ spec: fieldPath: metadata.namespace - name: CONTAINER_NAME value: pubapi - - name: PROXY_NAME + - name: WHOIS image: gcr.io/GCP_PROJECT/proxy ports: - containerPort: 30001 @@ -52,7 +52,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CONTAINER_NAME - value: PROXY_NAME + value: WHOIS --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -84,9 +84,26 @@ spec: - port: 80 targetPort: http name: http - - port: 43 - targetPort: whois - name: whois +--- +apiVersion: v1 +kind: Service +metadata: + name: WHOIS + annotations: + cloud.google.com/l4-rbs: enabled + networking.gke.io/weighted-load-balancing: pods-per-node +spec: + type: LoadBalancer + # Traffic is directly delivered to a node, preserving the original source IP. + externalTrafficPolicy: Local + ipFamilies: [IPv4, IPv6] + ipFamilyPolicy: RequireDualStack + selector: + service: pubapi + ports: + - port: 43 + targetPort: whois + name: whois --- apiVersion: net.gke.io/v1 kind: ServiceExport