Skip to content

Commit

Permalink
Make GKE the default in alpha and qa (#2624)
Browse files Browse the repository at this point in the history
  • Loading branch information
jianglai authored Dec 17, 2024
1 parent f649d96 commit da8df1f
Show file tree
Hide file tree
Showing 7 changed files with 66 additions and 14 deletions.
20 changes: 17 additions & 3 deletions core/src/main/java/google/registry/request/auth/AuthModule.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package google.registry.request.auth;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Suppliers.memoizeWithExpiration;
import static com.google.common.net.HttpHeaders.AUTHORIZATION;
import static google.registry.util.RegistryEnvironment.UNITTEST;

Expand All @@ -37,8 +38,11 @@
import google.registry.util.GoogleCredentialsBundle;
import google.registry.util.RegistryEnvironment;
import java.io.IOException;
import java.time.Duration;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import javax.inject.Named;
import javax.inject.Provider;
import javax.inject.Qualifier;
import javax.inject.Singleton;

Expand Down Expand Up @@ -87,13 +91,13 @@ ImmutableList<AuthenticationMechanism> provideApiAuthenticationMechanisms(
TokenVerifier provideIapTokenVerifier(
@Config("projectId") String projectId,
@Config("projectIdNumber") long projectIdNumber,
@Named("backendServiceIdMap") ImmutableMap<String, Long> backendServiceIdMap) {
@Named("backendServiceIdMap") Supplier<ImmutableMap<String, Long>> backendServiceIdMap) {
com.google.auth.oauth2.TokenVerifier.Builder tokenVerifierBuilder =
com.google.auth.oauth2.TokenVerifier.newBuilder().setIssuer(IAP_ISSUER_URL);
return (String service, String token) -> {
String audience;
if (RegistryEnvironment.isOnJetty()) {
Long backendServiceId = backendServiceIdMap.get(service);
Long backendServiceId = backendServiceIdMap.get().get(service);
checkNotNull(
backendServiceId,
"Backend service ID not found for service: %s, available IDs are %s",
Expand Down Expand Up @@ -156,7 +160,6 @@ static BackendServicesClient provideBackendServicesClients(
}

@Provides
@Singleton
@Named("backendServiceIdMap")
static ImmutableMap<String, Long> provideBackendServiceList(
Lazy<BackendServicesClient> client, @Config("projectId") String projectId) {
Expand All @@ -174,4 +177,15 @@ static ImmutableMap<String, Long> provideBackendServiceList(
}
return builder.build();
}

// Use an expiring cache so that the backend service ID map can be refreshed without restarting
// the server. The map is very unlikely to change, except for when services are just deployed
// for the first time, because some pods might receive traffic before all services are deployed.
@Provides
@Singleton
@Named("backendServiceIdMap")
static Supplier<ImmutableMap<String, Long>> provideBackendServiceIdMapSupplier(
@Named("backendServiceIdMap") Provider<ImmutableMap<String, Long>> backendServiceIdMap) {
return memoizeWithExpiration(backendServiceIdMap::get, Duration.ofMinutes(15));
}
}
14 changes: 14 additions & 0 deletions core/src/main/java/google/registry/tools/RegistryCli.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.beust.jcommander.ParametersDelegate;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.persistence.transaction.TransactionManagerFactory;
Expand All @@ -41,6 +42,9 @@
@Parameters(separators = " =", commandDescription = "Command-line interface to the registry")
final class RegistryCli implements CommandRunner {

private static final ImmutableSet<RegistryToolEnvironment> DEFAULT_GKE_ENVIRONMENTS =
ImmutableSet.of(RegistryToolEnvironment.ALPHA, RegistryToolEnvironment.QA);

// The environment parameter is parsed twice: once here, and once with {@link
// RegistryToolEnvironment#parseFromArgs} in the {@link RegistryTool#main} function.
//
Expand Down Expand Up @@ -73,6 +77,9 @@ final class RegistryCli implements CommandRunner {
@Parameter(names = "--gke", description = "Whether to use GKE runtime, instead of GAE")
private boolean useGke = false;

@Parameter(names = "--gae", description = "Whether to use GAE runtime, instead of GKE")
private boolean useGae = false;

@Parameter(names = "--canary", description = "Whether to connect to the canary instances")
private boolean useCanary = false;

Expand Down Expand Up @@ -149,6 +156,13 @@ public void run(String[] args) throws Exception {
}
throw e;
}

checkState(!useGke || !useGae, "Cannot specify both --gke and --gae");
// Special logic to set the default based on the environment if neither --gae nor --gke is set.
if (!useGke && !useGae) {
useGke = DEFAULT_GKE_ENVIRONMENTS.contains(environment);
}

String parsedCommand = jcommander.getParsedCommand();
// Show the list of all commands either if requested or if no subcommand name was specified
// (which does not throw a ParameterException parse error above).
Expand Down
30 changes: 29 additions & 1 deletion jetty/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,36 @@ tasks.register('run', JavaExec) {
dependsOn(tasks.named('stage'))
}

tasks.register('buildDeployer', Exec) {
workingDir("${rootDir}/release/builder/")
commandLine 'go', 'build', '-o', "${buildDir}/deployer", 'deployCloudSchedulerAndQueue.go'
}

// Once GKE is the only option, we can use the same task in the root project instaead.
tasks.register('deployCloudSchedulerAndQueue') {
dependsOn(tasks.named('deployCloudScheduler'), tasks.named('deployQueue'))
}

tasks.register('deployCloudScheduler', Exec) {
dependsOn(tasks.named('buildDeployer'))
workingDir("$buildDir")
commandLine './deployer',
"${rootDir}/core/src/main/java/google/registry/config/files/nomulus-config-${rootProject.environment}.yaml",
"${rootDir}/core/src/main/java/google/registry/env/${rootProject.environment}/default/WEB-INF/cloud-scheduler-tasks.xml",
rootProject.gcpProject, '--gke'
}

tasks.register('deployQueue', Exec) {
dependsOn(tasks.named('buildDeployer'))
workingDir("$buildDir")
commandLine './deployer',
"${rootDir}/core/src/main/java/google/registry/config/files/nomulus-config-${rootProject.environment}.yaml",
"${rootDir}/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml",
rootProject.gcpProject, '--gke'
}

tasks.register('deployNomulus', Exec) {
dependsOn('pushNomulusImage', ':proxy:pushProxyImage')
dependsOn('pushNomulusImage', 'deployCloudSchedulerAndQueue')
configure verifyDeploymentConfig
commandLine './deploy-nomulus-for-env.sh', "${rootProject.environment}", "${rootProject.baseDomain}"
}
Expand Down
8 changes: 4 additions & 4 deletions proxy/deploy-proxy-for-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,13 @@ do
gcloud container clusters get-credentials "${parts[0]}" \
--project "${project}" --zone "${parts[1]}"
sed s/GCP_PROJECT/${project}/g "./kubernetes/proxy-deployment-${environment}.yaml" | \
kubectl replace -f -
kubectl replace -f "./kubernetes/proxy-service.yaml" --force
kubectl apply -f -
kubectl apply -f "./kubernetes/proxy-service.yaml" --force
# Alpha does not have canary
if [[ ${environment} != "alpha" ]]; then
sed s/GCP_PROJECT/${project}/g "./kubernetes/proxy-deployment-${environment}-canary.yaml" | \
kubectl replace -f -
kubectl replace -f "./kubernetes/proxy-service-canary.yaml" --force
kubectl apply -f -
kubectl apply -f "./kubernetes/proxy-service-canary.yaml" --force
fi
# Kills all running pods, new pods created will be pulling the new image.
kubectl delete pods --all
Expand Down
1 change: 0 additions & 1 deletion proxy/src/main/java/google/registry/proxy/ProxyConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ enum Environment {

public String projectId;
public String oauthClientId;
public boolean canary;
public List<String> gcpScopes;
public int serverCertificateCacheSeconds;
public Gcs gcs;
Expand Down
4 changes: 2 additions & 2 deletions proxy/src/main/java/google/registry/proxy/ProxyModule.java
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,8 @@ static Supplier<String> provideOidcToken(
@Singleton
@Provides
@Named("canary")
static boolean provideIsCanary(ProxyConfig config) {
return config.canary;
boolean provideIsCanary(Environment env) {
return env.name().endsWith("_CANARY");
}

@Singleton
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@
# GCP project ID
projectId: your-gcp-project-id

# Whether to connect to the canary (instead of regular) service.
canary: false

# OAuth client ID set as the audience of the OIDC token. This value must be the
# same as the auth.oauthClientId value in Nomulus config file, which usually is
# the IAP client ID, to allow the request to access IAP protected endpoints.
Expand Down

0 comments on commit da8df1f

Please sign in to comment.