Update RemoteDefinitionsProvider to fallback to proper base url (#8494)

duffelhq · Aug 23, 2023 · 0ccd1f0 · 0ccd1f0
1 parent d856f70
commit 0ccd1f0
Show file tree

Hide file tree

Showing 8 changed files with 34 additions and 22 deletions.
diff --git a/airbyte-bootloader/src/main/resources/application.yml b/airbyte-bootloader/src/main/resources/application.yml
@@ -12,7 +12,7 @@ airbyte:
   connector-registry:
     seed-provider: ${CONNECTOR_REGISTRY_SEED_PROVIDER:local}
     remote:
-      base-url: ${CONNECTOR_REGISTRY_BASE_URL:`https://connectors.airbyte.com/`}
+      base-url: ${CONNECTOR_REGISTRY_BASE_URL:}
       timeout-ms: ${CONNECTOR_REGISTRY_TIMEOUT_MS:30000}
   deployment-mode: ${DEPLOYMENT_MODE:OSS}
   feature-flag:

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/constants/AirbyteCatalogConstants.java b/airbyte-commons/src/main/java/io/airbyte/commons/constants/AirbyteCatalogConstants.java
@@ -26,7 +26,7 @@ public final class AirbyteCatalogConstants {
       SEED_SUBDIRECTORY + LOCAL_CONNECTOR_CATALOG_PATH_FILE_NAME;
   public static final String LOCAL_SECRETS_MASKS_PATH = "/" + SEED_SUBDIRECTORY + LOCAL_SECRETS_MASKS_FILE_NAME;
 
-  public static final String REMOTE_REGISTRY_BASE_URL = "https://connectors.airbyte.com/";
+  public static final String REMOTE_REGISTRY_BASE_URL = "https://connectors.airbyte.com/files/";
 
   private AirbyteCatalogConstants() {
     // Private constructor to prevent instantiation

diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/RemoteDefinitionsProvider.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/RemoteDefinitionsProvider.java
@@ -7,6 +7,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.api.client.http.HttpStatusCodes;
 import com.google.common.annotations.VisibleForTesting;
+import io.airbyte.commons.constants.AirbyteCatalogConstants;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.version.AirbyteProtocolVersion;
 import io.airbyte.config.ActorType;
@@ -48,24 +49,31 @@ public class RemoteDefinitionsProvider implements DefinitionsProvider {
   private final DeploymentMode deploymentMode;
   private final Duration timeout;
 
-  public RemoteDefinitionsProvider(@Value("${airbyte.connector-registry.remote.base-url}") final String remoteRegistryBaseUrl,
-                                   final DeploymentMode deploymentMode,
-                                   @Value("${airbyte.connector-registry.remote.timeout-ms}") final long remoteCatalogTimeoutMs) {
-    LOGGER.info("Creating remote definitions provider for URL '{}' and registry '{}'...", remoteRegistryBaseUrl, deploymentMode);
-    if (remoteRegistryBaseUrl == null || remoteRegistryBaseUrl.isEmpty()) {
-      throw new IllegalArgumentException("Remote connector registry URL cannot be null or empty.");
-    }
-
+  private URI parsedRemoteRegistryBaseUrlOrDefault(final String remoteRegistryBaseUrl) {
     try {
-      this.remoteRegistryBaseUrl = new URI(remoteRegistryBaseUrl);
-      this.timeout = Duration.ofMillis(remoteCatalogTimeoutMs);
-      this.deploymentMode = deploymentMode;
+      if (remoteRegistryBaseUrl == null || remoteRegistryBaseUrl.isEmpty()) {
+        LOGGER.error("Remote connector registry base URL cannot be null or empty. Falling back to default");
+        return new URI(AirbyteCatalogConstants.REMOTE_REGISTRY_BASE_URL);
+      } else {
+        return new URI(remoteRegistryBaseUrl);
+      }
     } catch (final URISyntaxException e) {
       LOGGER.error("Invalid remote registry base URL: {}", remoteRegistryBaseUrl);
       throw new IllegalArgumentException("Remote connector registry base URL must be a valid URI.", e);
     }
   }
 
+  public RemoteDefinitionsProvider(@Value("${airbyte.connector-registry.remote.base-url}") final String remoteRegistryBaseUrl,
+                                   final DeploymentMode deploymentMode,
+                                   @Value("${airbyte.connector-registry.remote.timeout-ms}") final long remoteCatalogTimeoutMs) {
+    final URI remoteRegistryBaseUrlUri = parsedRemoteRegistryBaseUrlOrDefault(remoteRegistryBaseUrl);
+    LOGGER.info("Creating remote definitions provider for URL '{}' and registry '{}'...", remoteRegistryBaseUrlUri, deploymentMode);
+
+    this.remoteRegistryBaseUrl = remoteRegistryBaseUrlUri;
+    this.timeout = Duration.ofMillis(remoteCatalogTimeoutMs);
+    this.deploymentMode = deploymentMode;
+  }
+
   private Map<UUID, ConnectorRegistrySourceDefinition> getSourceDefinitionsMap() {
     final ConnectorRegistry registry = getRemoteConnectorRegistry();
     return registry.getSources().stream().collect(Collectors.toMap(
@@ -137,12 +145,12 @@ private String getRegistryName() {
 
   @VisibleForTesting
   URI getRegistryUrl() {
-    return remoteRegistryBaseUrl.resolve(String.format("files/registries/v0/%s_registry.json", getRegistryName()));
+    return remoteRegistryBaseUrl.resolve(String.format("registries/v0/%s_registry.json", getRegistryName()));
   }
 
   @VisibleForTesting
   URI getRegistryEntryUrl(final String connectorName, final String version) {
-    return remoteRegistryBaseUrl.resolve(String.format("files/metadata/%s/%s/%s.json", connectorName, version, getRegistryName()));
+    return remoteRegistryBaseUrl.resolve(String.format("metadata/%s/%s/%s.json", connectorName, version, getRegistryName()));
   }
 
   /**

diff --git a/...yte-config/specs/src/test/java/io/airbyte/config/specs/RemoteDefinitionsProviderTest.java b/...yte-config/specs/src/test/java/io/airbyte/config/specs/RemoteDefinitionsProviderTest.java
@@ -172,7 +172,7 @@ void testNonJson() {
   @ParameterizedTest
   @CsvSource({"OSS", "CLOUD"})
   void testGetRegistryUrl(final String deploymentMode) {
-    final String baseUrl = "https://connectors.airbyte.com/";
+    final String baseUrl = "https://connectors.airbyte.com/files/";
     final RemoteDefinitionsProvider definitionsProvider =
         new RemoteDefinitionsProvider(baseUrl, DeploymentMode.valueOf(deploymentMode), TimeUnit.SECONDS.toMillis(1));
     final URI registryUrl = definitionsProvider.getRegistryUrl();
@@ -183,7 +183,7 @@ void testGetRegistryUrl(final String deploymentMode) {
   @ParameterizedTest
   @CsvSource({"OSS", "CLOUD"})
   void testGetRegistryEntryUrl(final String deploymentMode) {
-    final String baseUrl = "https://connectors.airbyte.com/";
+    final String baseUrl = "https://connectors.airbyte.com/files/";
     final String connectorName = "airbyte/source-github";
     final String version = "1.0.0";
     final RemoteDefinitionsProvider definitionsProvider =

diff --git a/airbyte-cron/src/main/resources/application.yml b/airbyte-cron/src/main/resources/application.yml
@@ -11,7 +11,7 @@ airbyte:
   connector-registry:
     seed-provider: ${CONNECTOR_REGISTRY_SEED_PROVIDER:local}
     remote:
-      base-url: ${CONNECTOR_REGISTRY_BASE_URL:`https://connectors.airbyte.com/`}
+      base-url: ${CONNECTOR_REGISTRY_BASE_URL:}
       timeout-ms: ${CONNECTOR_REGISTRY_TIMEOUT_MS:30000}
   cron:
     update-definitions:

diff --git a/airbyte-server/src/main/resources/application.yml b/airbyte-server/src/main/resources/application.yml
@@ -68,7 +68,7 @@ airbyte:
     datadog-support-names: ${CONNECTOR_DATADOG_SUPPORT_NAMES:}
   connector-registry:
     remote:
-      base-url: ${CONNECTOR_REGISTRY_BASE_URL:`https://connectors.airbyte.com/`}
+      base-url: ${CONNECTOR_REGISTRY_BASE_URL:}
       timeout-ms: ${CONNECTOR_REGISTRY_TIMEOUT_MS:30000}
   deployment-mode: ${DEPLOYMENT_MODE:OSS}
   feature-flag:

diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml
@@ -58,7 +58,7 @@ airbyte:
     datadog-support-names: ${CONNECTOR_DATADOG_SUPPORT_NAMES:}
   connector-registry:
     remote:
-      base-url: ${CONNECTOR_REGISTRY_BASE_URL:`https://connectors.airbyte.com/`}
+      base-url: ${CONNECTOR_REGISTRY_BASE_URL:}
       timeout-ms: ${CONNECTOR_REGISTRY_TIMEOUT_MS:30000}
   container:
     orchestrator:

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -24,12 +24,13 @@ services:
     container_name: airbyte-bootloader
     environment:
       - AIRBYTE_VERSION=${VERSION}
+      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
       - CONNECTOR_REGISTRY_SEED_PROVIDER=${CONNECTOR_REGISTRY_SEED_PROVIDER:-}
       - DATABASE_PASSWORD=${DATABASE_PASSWORD}
       - DATABASE_URL=${DATABASE_URL}
       - DATABASE_USER=${DATABASE_USER}
-      - LOG_LEVEL=${LOG_LEVEL}
       - LOCAL_CONNECTOR_CATALOG_PATH=${LOCAL_CONNECTOR_CATALOG_PATH}
+      - LOG_LEVEL=${LOG_LEVEL}
     networks:
       - airbyte_internal
     depends_on:
@@ -71,6 +72,7 @@ services:
       - CONFIG_DATABASE_URL=${CONFIG_DATABASE_URL:-}
       - CONFIG_DATABASE_USER=${CONFIG_DATABASE_USER:-}
       - CONFIG_ROOT=${CONFIG_ROOT}
+      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
       - DATABASE_PASSWORD=${DATABASE_PASSWORD}
       - DATABASE_URL=${DATABASE_URL}
       - DATABASE_USER=${DATABASE_USER}
@@ -144,6 +146,7 @@ services:
       - CONFIG_DATABASE_URL=${CONFIG_DATABASE_URL:-}
       - CONFIG_DATABASE_USER=${CONFIG_DATABASE_USER:-}
       - CONFIG_ROOT=${CONFIG_ROOT}
+      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
       - DATABASE_PASSWORD=${DATABASE_PASSWORD}
       - DATABASE_URL=${DATABASE_URL}
       - DATABASE_USER=${DATABASE_USER}
@@ -226,13 +229,14 @@ services:
     environment:
       - AIRBYTE_VERSION=${VERSION}
       - CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION}
+      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
       - CONNECTOR_REGISTRY_SEED_PROVIDER=${CONNECTOR_REGISTRY_SEED_PROVIDER:-}
       - DATABASE_PASSWORD=${DATABASE_PASSWORD}
       - DATABASE_URL=${DATABASE_URL}
       - DATABASE_USER=${DATABASE_USER}
-      - DEPLOYMENT_MODE=${DEPLOYMENT_MODE}
       - DD_AGENT_HOST=${DD_AGENT_HOST}
       - DD_DOGSTATSD_PORT=${DD_DOGSTATSD_PORT}
+      - DEPLOYMENT_MODE=${DEPLOYMENT_MODE}
       - LOG_LEVEL=${LOG_LEVEL}
       - METRIC_CLIENT=${METRIC_CLIENT}
       - MICRONAUT_ENVIRONMENTS=${CRON_MICRONAUT_ENVIRONMENTS}