Skip to content

Commit 1f69e9f

Browse files
committed
Expose EPP and WHOIS endpoints on reginal load balancers
k8s does not have a way to expose a global load balancer with TCP endpoints, and setting up node port-based routing is a chore, even with Terraform (which is what we did with the standalone proxy). We will use Cloud DNS's geolocation routing policy to ensure that clients connect to the endpoint closest to them.
1 parent d130e74 commit 1f69e9f

File tree

4 files changed

+215
-12
lines changed

4 files changed

+215
-12
lines changed

jetty/deploy-nomulus-for-env.sh

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,15 @@ do
3737
sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
3838
sed s/ENVIRONMENT/"${environment}"/g | \
3939
sed s/PROXY_ENV/"${environment}"/g | \
40-
sed s/PROXY_NAME/"proxy"/g | \
40+
sed s/EPP/"epp"/g | \
41+
sed s/WHOIS/"whois"/g | \
4142
kubectl apply -f -
4243
# canary
4344
sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
4445
sed s/ENVIRONMENT/"${environment}"/g | \
4546
sed s/PROXY_ENV/"${environment}_canary"/g | \
46-
sed s/PROXY_NAME/"proxy-canary"/g | \
47+
sed s/EPP/"epp-canary"/g | \
48+
sed s/WHOIS/"whois-canary"/g | \
4749
sed s/"${service}"/"${service}-canary"/g | \
4850
kubectl apply -f -
4951
done

jetty/get-endpoints.py

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
#! /bin/env python3
2+
# Copyright 2024 The Nomulus Authors. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
'''
17+
A script that outputs the IP endpoints of various load balancers, to be run
18+
after Nomulus is deployed.
19+
'''
20+
21+
import ipaddress
22+
import json
23+
import subprocess
24+
import sys
25+
from dataclasses import dataclass
26+
from ipaddress import IPv4Address
27+
from ipaddress import IPv6Address
28+
from operator import attrgetter
29+
from operator import methodcaller
30+
31+
32+
class PreserveContext:
33+
def __enter__(self):
34+
self._context = run_command('kubectl config current-context')
35+
36+
def __exit__(self, type, value, traceback):
37+
run_command('kubectl config use-context ' + self._context)
38+
39+
40+
class UseCluster(PreserveContext):
41+
def __init__(self, cluster: str, region: str, project: str):
42+
self._cluster = cluster
43+
self._region = region
44+
self._project = project
45+
46+
def __enter__(self):
47+
super().__enter__()
48+
cmd = f'gcloud container clusters get-credentials {self._cluster} --location {self._region} --project {self._project}'
49+
run_command(cmd)
50+
51+
52+
def run_command(cmd: str, print_output=False) -> str:
53+
proc = subprocess.run(cmd, text=True, shell=True, stdout=subprocess.PIPE,
54+
stderr=subprocess.STDOUT)
55+
if print_output:
56+
print(proc.stdout)
57+
return proc.stdout
58+
59+
60+
def get_clusters(project: str) -> dict[str, str]:
61+
cmd = f'gcloud container clusters list --project {project} --format=json'
62+
content = json.loads(run_command(cmd))
63+
res = {}
64+
for item in content:
65+
name = item['name']
66+
region = item['location']
67+
if not name.startswith('nomulus-cluster'):
68+
continue
69+
res[name] = region
70+
return res
71+
72+
73+
def get_endpoints(resource: str, service: str, selector: list[str]) -> list[
74+
str]:
75+
content = json.loads(
76+
run_command(f'kubectl get {resource}/{service} -o json'))
77+
name = content['metadata']['name']
78+
ips = content
79+
for key in selector[:-1]:
80+
ips = ips[key]
81+
return [x[selector[-1]] for x in ips]
82+
83+
84+
def get_region_symbol(region: str) -> str:
85+
if region.startswith('us'):
86+
return 'amer'
87+
if region.startswith('europe'):
88+
return 'emea'
89+
if region.startswith('asia'):
90+
return 'apac'
91+
return 'other'
92+
93+
94+
@dataclass
95+
class IP:
96+
service: str
97+
region: str
98+
address: IPv4Address | IPv6Address
99+
100+
def is_ipv6(self) -> bool:
101+
return self.address.version == 6
102+
103+
def __str__(self) -> str:
104+
return f'{self.service} {self.region}: {self.address}'
105+
106+
107+
def terraform_str(item) -> str:
108+
res = ""
109+
if (isinstance(item, dict)):
110+
res += '{\n'
111+
for key, value in item.items():
112+
res += f'{key} = {terraform_str(value)}\n'
113+
res += '}'
114+
elif (isinstance(item, list)):
115+
res += '['
116+
for i, value in enumerate(item):
117+
if i != 0:
118+
res += ', '
119+
res += terraform_str(value)
120+
res += ']'
121+
else:
122+
res += f'"{item}"'
123+
return res
124+
125+
126+
projects = {"domain-registry", "domain-registry-sandbox",
127+
"domain-registry-crash", "domain-registry-crash",
128+
"domain-registry-alpha"}
129+
130+
if __name__ == '__main__':
131+
if len(sys.argv) != 2:
132+
raise ValueError('Usage: get-endpoints.py <project>')
133+
project = sys.argv[1]
134+
if (project not in projects):
135+
raise ValueError(f'Invalid project: {project}')
136+
clusters = get_clusters(project)
137+
ips = []
138+
res = {}
139+
for cluster, region in clusters.items():
140+
with UseCluster(cluster, region, project):
141+
for service in ['whois', 'whois-canary', 'epp', 'epp-canary']:
142+
map_key = service.replace('-', '_')
143+
for ip in get_endpoints('services', service,
144+
['status', 'loadBalancer', 'ingress',
145+
'ip']):
146+
ip = ipaddress.ip_address(ip)
147+
if isinstance(ip, IPv4Address):
148+
map_key_with_iptype = map_key + '_ipv4'
149+
else:
150+
map_key_with_iptype = map_key + '_ipv6'
151+
if map_key_with_iptype not in res:
152+
res[map_key_with_iptype] = {}
153+
res[map_key_with_iptype][get_region_symbol(region)] = [ip]
154+
ips.append(IP(service, get_region_symbol(region), ip))
155+
if not region.startswith('us'):
156+
continue
157+
ip = get_endpoints('gateways.gateway.networking.k8s.io', 'nomulus',
158+
['status', 'addresses', 'value'])[0]
159+
print(f'nomulus: {ip}')
160+
res['https_ip'] = ipaddress.ip_address(ip)
161+
ips.sort(key=attrgetter('region'))
162+
ips.sort(key=methodcaller('is_ipv6'))
163+
ips.sort(key=attrgetter('service'))
164+
for ip in ips:
165+
print(ip)
166+
print("Terraform friendly output:")
167+
print(terraform_str(res))

jetty/kubernetes/nomulus-frontend.yaml

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ spec:
3333
fieldPath: metadata.namespace
3434
- name: CONTAINER_NAME
3535
value: frontend
36-
- name: PROXY_NAME
36+
- name: EPP
3737
image: gcr.io/GCP_PROJECT/proxy
3838
ports:
3939
- containerPort: 30002
@@ -52,7 +52,7 @@ spec:
5252
fieldRef:
5353
fieldPath: metadata.namespace
5454
- name: CONTAINER_NAME
55-
value: PROXY_NAME
55+
value: EPP
5656
---
5757
# Only need to define the service account once per cluster.
5858
apiVersion: v1
@@ -92,9 +92,26 @@ spec:
9292
- port: 80
9393
targetPort: http
9494
name: http
95-
- port: 700
96-
targetPort: epp
97-
name: epp
95+
---
96+
apiVersion: v1
97+
kind: Service
98+
metadata:
99+
name: EPP
100+
annotations:
101+
cloud.google.com/l4-rbs: enabled
102+
networking.gke.io/weighted-load-balancing: pods-per-node
103+
spec:
104+
type: LoadBalancer
105+
# Traffic is directly delivered to a node, preserving the original source IP.
106+
externalTrafficPolicy: Local
107+
ipFamilies: [IPv4, IPv6]
108+
ipFamilyPolicy: RequireDualStack
109+
selector:
110+
service: frontend
111+
ports:
112+
- port: 700
113+
targetPort: epp
114+
name: epp
98115
---
99116
apiVersion: net.gke.io/v1
100117
kind: ServiceExport

jetty/kubernetes/nomulus-pubapi.yaml

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ spec:
3333
fieldPath: metadata.namespace
3434
- name: CONTAINER_NAME
3535
value: pubapi
36-
- name: PROXY_NAME
36+
- name: WHOIS
3737
image: gcr.io/GCP_PROJECT/proxy
3838
ports:
3939
- containerPort: 30001
@@ -52,7 +52,7 @@ spec:
5252
fieldRef:
5353
fieldPath: metadata.namespace
5454
- name: CONTAINER_NAME
55-
value: PROXY_NAME
55+
value: WHOIS
5656
---
5757
apiVersion: autoscaling/v2
5858
kind: HorizontalPodAutoscaler
@@ -84,9 +84,26 @@ spec:
8484
- port: 80
8585
targetPort: http
8686
name: http
87-
- port: 43
88-
targetPort: whois
89-
name: whois
87+
---
88+
apiVersion: v1
89+
kind: Service
90+
metadata:
91+
name: WHOIS
92+
annotations:
93+
cloud.google.com/l4-rbs: enabled
94+
networking.gke.io/weighted-load-balancing: pods-per-node
95+
spec:
96+
type: LoadBalancer
97+
# Traffic is directly delivered to a node, preserving the original source IP.
98+
externalTrafficPolicy: Local
99+
ipFamilies: [IPv4, IPv6]
100+
ipFamilyPolicy: RequireDualStack
101+
selector:
102+
service: pubapi
103+
ports:
104+
- port: 43
105+
targetPort: whois
106+
name: whois
90107
---
91108
apiVersion: net.gke.io/v1
92109
kind: ServiceExport

0 commit comments

Comments
 (0)