From 5c0c4045ed3c644dfd25c6e548e995ccc9b8629f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 21 Aug 2023 12:24:45 +0200 Subject: [PATCH] Connector builder server: Remove list endpoint (#8403) --- .../ConnectorBuilderController.java | 17 +---- .../handlers/ResolveManifestHandler.java | 2 +- .../handlers/StreamsHandler.java | 50 ------------- .../requester/AirbyteCdkRequester.java | 4 -- .../requester/AirbyteCdkRequesterImpl.java | 32 --------- .../src/main/openapi/openapi.yaml | 63 ---------------- .../src/main/resources/application.yml | 3 - ...ectorBuilderControllerIntegrationTest.java | 22 +----- .../ConnectorBuilderControllerTest.java | 24 +------ .../fixtures/RecordStreamsList.json | 15 ---- .../handlers/StreamsHandlerTest.java | 71 ------------------- .../AirbyteCdkRequesterImplTest.java | 35 --------- .../src/core/api/hooks/connectorBuilderApi.ts | 18 +---- 13 files changed, 5 insertions(+), 351 deletions(-) delete mode 100644 airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/handlers/StreamsHandler.java delete mode 100644 airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/fixtures/RecordStreamsList.json delete mode 100644 airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/handlers/StreamsHandlerTest.java diff --git a/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/controllers/ConnectorBuilderController.java b/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/controllers/ConnectorBuilderController.java index 9c155529df8..9700d4dbf77 100644 --- a/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/controllers/ConnectorBuilderController.java +++ b/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/controllers/ConnectorBuilderController.java @@ -12,12 +12,9 @@ import io.airbyte.connector_builder.api.model.generated.ResolveManifestRequestBody; import io.airbyte.connector_builder.api.model.generated.StreamRead; import io.airbyte.connector_builder.api.model.generated.StreamReadRequestBody; -import io.airbyte.connector_builder.api.model.generated.StreamsListRead; -import io.airbyte.connector_builder.api.model.generated.StreamsListRequestBody; import io.airbyte.connector_builder.handlers.HealthHandler; import io.airbyte.connector_builder.handlers.ResolveManifestHandler; import io.airbyte.connector_builder.handlers.StreamHandler; -import io.airbyte.connector_builder.handlers.StreamsHandler; import io.micronaut.context.annotation.Context; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; @@ -39,16 +36,13 @@ public class ConnectorBuilderController implements V1Api { private final HealthHandler healthHandler; private final StreamHandler streamHandler; private final ResolveManifestHandler resolveManifestHandler; - private final StreamsHandler streamsHandler; public ConnectorBuilderController(final HealthHandler healthHandler, final ResolveManifestHandler resolveManifestHandler, - final StreamHandler streamHandler, - final StreamsHandler streamsHandler) { + final StreamHandler streamHandler) { this.healthHandler = healthHandler; this.streamHandler = streamHandler; this.resolveManifestHandler = resolveManifestHandler; - this.streamsHandler = streamsHandler; } @Override @@ -60,15 +54,6 @@ public HealthCheckRead getHealthCheck() { return healthHandler.getHealthCheck(); } - @Override - @Post(uri = "/streams/list", - produces = MediaType.APPLICATION_JSON) - @Secured({AUTHENTICATED_USER}) - @ExecuteOn(TaskExecutors.IO) - public StreamsListRead listStreams(final StreamsListRequestBody streamsListRequestBody) { - return streamsHandler.listStreams(streamsListRequestBody); - } - @Override @Post(uri = "/stream/read", produces = MediaType.APPLICATION_JSON) diff --git a/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/handlers/ResolveManifestHandler.java b/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/handlers/ResolveManifestHandler.java index ece0a79eb33..301532a7d53 100644 --- a/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/handlers/ResolveManifestHandler.java +++ b/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/handlers/ResolveManifestHandler.java @@ -44,7 +44,7 @@ public ResolveManifest resolveManifest( resolveManifestRequestBody.getWorkspaceId(), resolveManifestRequestBody.getProjectId()); return this.requester.resolveManifest(resolveManifestRequestBody.getManifest()); } catch (final IOException exc) { - LOGGER.error("Error handling list_streams request.", exc); + LOGGER.error("Error handling resolve_manifest request.", exc); throw new ConnectorBuilderException("Error handling resolve_manifest request.", exc); } } diff --git a/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/handlers/StreamsHandler.java b/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/handlers/StreamsHandler.java deleted file mode 100644 index ef7b65f796b..00000000000 --- a/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/handlers/StreamsHandler.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.connector_builder.handlers; - -import io.airbyte.connector_builder.TracingHelper; -import io.airbyte.connector_builder.api.model.generated.StreamsListRead; -import io.airbyte.connector_builder.api.model.generated.StreamsListRequestBody; -import io.airbyte.connector_builder.exceptions.AirbyteCdkInvalidInputException; -import io.airbyte.connector_builder.exceptions.ConnectorBuilderException; -import io.airbyte.connector_builder.requester.AirbyteCdkRequester; -import jakarta.inject.Inject; -import jakarta.inject.Singleton; -import java.io.IOException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Handle /streams requests. - */ -@Singleton -public class StreamsHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(ResolveManifestHandler.class); - - private final AirbyteCdkRequester requester; - - @Inject - public StreamsHandler(final AirbyteCdkRequester requester) { - this.requester = requester; - } - - /** - * Handle list_streams. - */ - public StreamsListRead listStreams(final StreamsListRequestBody streamsListRequestBody) - throws AirbyteCdkInvalidInputException, ConnectorBuilderException { - try { - TracingHelper.addWorkspaceAndProjectIdsToTrace(streamsListRequestBody.getWorkspaceId(), streamsListRequestBody.getProjectId()); - LOGGER.info("Handling list_streams request for workspace '{}' with project ID = '{}'", - streamsListRequestBody.getWorkspaceId(), streamsListRequestBody.getProjectId()); - return this.requester.listStreams(streamsListRequestBody.getManifest(), streamsListRequestBody.getConfig()); - } catch (final IOException exc) { - LOGGER.error("Error handling list_streams request.", exc); - throw new ConnectorBuilderException("Error handling list_streams request.", exc); - } - } - -} diff --git a/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/requester/AirbyteCdkRequester.java b/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/requester/AirbyteCdkRequester.java index 0861be8f6b7..5c1229497ea 100644 --- a/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/requester/AirbyteCdkRequester.java +++ b/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/requester/AirbyteCdkRequester.java @@ -7,9 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.connector_builder.api.model.generated.ResolveManifest; import io.airbyte.connector_builder.api.model.generated.StreamRead; -import io.airbyte.connector_builder.api.model.generated.StreamsListRead; import io.airbyte.connector_builder.exceptions.AirbyteCdkInvalidInputException; -import io.airbyte.connector_builder.exceptions.CdkProcessException; import io.airbyte.connector_builder.exceptions.ConnectorBuilderException; import java.io.IOException; @@ -21,8 +19,6 @@ public interface AirbyteCdkRequester { ResolveManifest resolveManifest(final JsonNode manifest) throws IOException, AirbyteCdkInvalidInputException, ConnectorBuilderException; - StreamsListRead listStreams(final JsonNode manifest, JsonNode config) throws IOException, AirbyteCdkInvalidInputException, CdkProcessException; - StreamRead readStream(final JsonNode manifest, final JsonNode config, final String stream, final Integer recordLimit) throws IOException, AirbyteCdkInvalidInputException, ConnectorBuilderException; diff --git a/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/requester/AirbyteCdkRequesterImpl.java b/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/requester/AirbyteCdkRequesterImpl.java index a3f7fd6e257..c934e34b162 100644 --- a/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/requester/AirbyteCdkRequesterImpl.java +++ b/airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/requester/AirbyteCdkRequesterImpl.java @@ -16,8 +16,6 @@ import io.airbyte.connector_builder.api.model.generated.StreamRead; import io.airbyte.connector_builder.api.model.generated.StreamReadAuxiliaryRequestsInner; import io.airbyte.connector_builder.api.model.generated.StreamReadSlicesInner; -import io.airbyte.connector_builder.api.model.generated.StreamsListRead; -import io.airbyte.connector_builder.api.model.generated.StreamsListReadStreamsInner; import io.airbyte.connector_builder.command_runner.SynchronousCdkCommandRunner; import io.airbyte.connector_builder.exceptions.AirbyteCdkInvalidInputException; import io.airbyte.connector_builder.exceptions.CdkProcessException; @@ -25,7 +23,6 @@ import jakarta.inject.Singleton; import java.io.IOException; import java.util.List; -import java.util.stream.StreamSupport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,7 +37,6 @@ public class AirbyteCdkRequesterImpl implements AirbyteCdkRequester { private static final String manifestKey = "__injected_declarative_manifest"; private static final String recordLimitKey = "max_records"; private static final String resolveManifestCommand = "resolve_manifest"; - private static final String listStreamsCommand = "list_streams"; private static final String readStreamCommand = "test_read"; private static final String catalogTemplate = """ { @@ -107,34 +103,6 @@ public ResolveManifest resolveManifest(final JsonNode manifest) return new ResolveManifest().manifest(record.getData().get("manifest")); } - @Override - @Trace(operationName = TracingHelper.CONNECTOR_BUILDER_OPERATION_NAME) - public StreamsListRead listStreams(final JsonNode manifest, final JsonNode config) - throws IOException, AirbyteCdkInvalidInputException, CdkProcessException { - return new StreamsListRead().streams( - StreamSupport.stream(request(manifest, config, listStreamsCommand).getData().get("streams").spliterator(), false).map(this::adaptStream) - .toList()); - } - - private StreamsListReadStreamsInner adaptStream(final JsonNode stream) { - if (isNull(stream, "name")) { - throw new AirbyteCdkInvalidInputException(String.format( - "Unexpected fatal error: streams are expected to have field 'name' but could not find it in %s. Please open a GitHub issue with Airbyte", - stream)); - } - if (isNull(stream, "url")) { - throw new AirbyteCdkInvalidInputException(String.format( - "Unexpected fatal error: streams are expected to have field 'url' but could not find it in %s. Please open a GitHub issue with Airbyte", - stream)); - } - - return new StreamsListReadStreamsInner().name(stream.get("name").asText()).url(stream.get("url").asText()); - } - - private static boolean isNull(final JsonNode jsonNode, final String fieldName) { - return jsonNode.get(fieldName) == null || jsonNode.get(fieldName).isNull(); - } - /** * Launch a CDK process responsible for handling requests. */ diff --git a/airbyte-connector-builder-server/src/main/openapi/openapi.yaml b/airbyte-connector-builder-server/src/main/openapi/openapi.yaml index 95a239f47b8..a2936fb9098 100644 --- a/airbyte-connector-builder-server/src/main/openapi/openapi.yaml +++ b/airbyte-connector-builder-server/src/main/openapi/openapi.yaml @@ -37,27 +37,6 @@ paths: $ref: "#/components/responses/ExceptionResponse" "422": $ref: "#/components/responses/InvalidInputResponse" - /v1/streams/list: - post: - summary: List all streams present in the connector manifest, along with their specific request URLs - operationId: listStreams - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/StreamsListRequestBody" - required: true - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/StreamsListRead" - "400": - $ref: "#/components/responses/ExceptionResponse" - "422": - $ref: "#/components/responses/InvalidInputResponse" /v1/manifest/resolve: post: summary: Given a JSON manifest, returns a JSON manifest with all of the $refs and $parameters resolved and flattened @@ -251,48 +230,6 @@ components: # --- Commenting out for now since they do not work with our orval openapi client generator --- # AirbyteProtocol: # $ref: ../../../../airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml - StreamsListRequestBody: - type: object - required: - - config - - manifest - properties: - config: - $ref: "#/components/schemas/ConnectorConfig" - manifest: - $ref: "#/components/schemas/ConnectorManifest" - workspace_id: - type: string - description: ID of the workspace where the request is coming from - project_id: - type: string - description: ID of the project where the request is coming from - StreamsListRead: - type: object - required: - - streams - properties: - streams: - type: array - items: - type: object - description: The stream names present in the connector manifest - required: - - name - - url - properties: - name: - type: string - description: The name of the stream - url: - type: string - description: The URL to which read requests will be made for this stream - # --- Potential addition for a later phase --- - # slices: - # type: array - # description: list of slices that will be retrieved for this stream - # items: - # type: object ResolveManifestRequestBody: type: object required: diff --git a/airbyte-connector-builder-server/src/main/resources/application.yml b/airbyte-connector-builder-server/src/main/resources/application.yml index 5ab13037db7..a576a82a7a4 100644 --- a/airbyte-connector-builder-server/src/main/resources/application.yml +++ b/airbyte-connector-builder-server/src/main/resources/application.yml @@ -12,9 +12,6 @@ micronaut: access-logger: enabled: ${HTTP_ACCESS_LOG_ENABLED:true} endpoints: - v1/list/streams: - enabled: true - sensitive: true v1/manifest_template: enable: true sensitive: false diff --git a/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/controllers/ConnectorBuilderControllerIntegrationTest.java b/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/controllers/ConnectorBuilderControllerIntegrationTest.java index 72620b28d71..e6394c857c9 100644 --- a/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/controllers/ConnectorBuilderControllerIntegrationTest.java +++ b/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/controllers/ConnectorBuilderControllerIntegrationTest.java @@ -16,8 +16,6 @@ import io.airbyte.connector_builder.api.model.generated.ResolveManifestRequestBody; import io.airbyte.connector_builder.api.model.generated.StreamRead; import io.airbyte.connector_builder.api.model.generated.StreamReadRequestBody; -import io.airbyte.connector_builder.api.model.generated.StreamsListRead; -import io.airbyte.connector_builder.api.model.generated.StreamsListRequestBody; import io.airbyte.connector_builder.command_runner.MockSynchronousPythonCdkCommandRunner; import io.airbyte.connector_builder.command_runner.SynchronousCdkCommandRunner; import io.airbyte.connector_builder.exceptions.AirbyteCdkInvalidInputException; @@ -28,7 +26,6 @@ import io.airbyte.connector_builder.handlers.HealthHandler; import io.airbyte.connector_builder.handlers.ResolveManifestHandler; import io.airbyte.connector_builder.handlers.StreamHandler; -import io.airbyte.connector_builder.handlers.StreamsHandler; import io.airbyte.connector_builder.requester.AirbyteCdkRequesterImpl; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.internal.VersionedAirbyteStreamFactory; @@ -62,7 +59,6 @@ class ConnectorBuilderControllerIntegrationTest { private HealthHandler healthHandler; static String cdkException; static String streamRead; - static String streamsList; static String recordManifestResolve; static String traceManifestResolve; static JsonNode validManifest; @@ -81,7 +77,6 @@ public static void setUpClass() throws IOException { final String relativeDir = "src/test/java/io/airbyte/connector_builder/fixtures"; validManifest = new ObjectMapper().readTree(readContents(String.format("%s/ValidManifest.json", relativeDir))); streamRead = readContents(String.format("%s/RecordStreamRead.json", relativeDir)); - streamsList = readContents(String.format("%s/RecordStreamsList.json", relativeDir)); recordManifestResolve = readContents(String.format("%s/RecordManifestResolve.json", relativeDir)); traceManifestResolve = readContents(String.format("%s/TraceManifestResolve.json", relativeDir)); cdkException = readContents(String.format("%s/CdkException.txt", relativeDir)); @@ -101,8 +96,7 @@ ConnectorBuilderController createControllerWithSynchronousRunner( final SynchronousCdkCommandRunner commandRunner = new MockSynchronousPythonCdkCommandRunner( this.writer, this.streamFactory, shouldThrow, exitCode, inputStream, errorStream, outputStream); final AirbyteCdkRequesterImpl requester = new AirbyteCdkRequesterImpl(commandRunner); - return new ConnectorBuilderController(this.healthHandler, new ResolveManifestHandler(requester), new StreamHandler(requester), - new StreamsHandler(requester)); + return new ConnectorBuilderController(this.healthHandler, new ResolveManifestHandler(requester), new StreamHandler(requester)); } @Test @@ -122,20 +116,6 @@ void givenTraceMessageWhenStreamReadThenThrowException() { () -> controller.readStream(new StreamReadRequestBody().config(A_CONFIG).manifest(A_MANIFEST).stream(A_STREAM))); } - @Test - void testStreamsList() { - final ConnectorBuilderController controller = givenAirbyteCdkReturnMessage(streamsList); - final StreamsListRead streamsListRead = controller.listStreams(new StreamsListRequestBody().config(A_CONFIG).manifest(A_MANIFEST)); - assertTrue(streamsListRead.getStreams().size() > 0); - } - - @Test - void givenTraceMessageWhenStreamsListThenThrowException() { - final ConnectorBuilderController controller = givenAirbyteCdkReturnMessage(traceManifestResolve); - Assertions.assertThrows(AirbyteCdkInvalidInputException.class, - () -> controller.listStreams(new StreamsListRequestBody().config(A_CONFIG).manifest(A_MANIFEST))); - } - @Test void testResolveManifestSuccess() { final InputStream stream = new ByteArrayInputStream( diff --git a/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/controllers/ConnectorBuilderControllerTest.java b/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/controllers/ConnectorBuilderControllerTest.java index 7da3c5af2d5..2d12ae747e9 100644 --- a/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/controllers/ConnectorBuilderControllerTest.java +++ b/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/controllers/ConnectorBuilderControllerTest.java @@ -14,13 +14,10 @@ import io.airbyte.connector_builder.api.model.generated.ResolveManifestRequestBody; import io.airbyte.connector_builder.api.model.generated.StreamRead; import io.airbyte.connector_builder.api.model.generated.StreamReadRequestBody; -import io.airbyte.connector_builder.api.model.generated.StreamsListRead; -import io.airbyte.connector_builder.api.model.generated.StreamsListRequestBody; import io.airbyte.connector_builder.exceptions.AirbyteCdkInvalidInputException; import io.airbyte.connector_builder.handlers.HealthHandler; import io.airbyte.connector_builder.handlers.ResolveManifestHandler; import io.airbyte.connector_builder.handlers.StreamHandler; -import io.airbyte.connector_builder.handlers.StreamsHandler; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -30,11 +27,8 @@ class ConnectorBuilderControllerTest { private HealthHandler healthHandler; private StreamHandler streamHandler; private ResolveManifestHandler resolveManifestHandler; - private StreamsHandler streamsHandler; private StreamReadRequestBody streamReadRequestBody; private StreamRead streamReadResponse; - private StreamsListRequestBody streamsListRequestBody; - private StreamsListRead streamsListResponse; private ResolveManifestRequestBody resolveManifestRequestBody; private ResolveManifest resolveManifest; @@ -42,17 +36,14 @@ class ConnectorBuilderControllerTest { void setup() { this.healthHandler = mock(HealthHandler.class); this.resolveManifestHandler = mock(ResolveManifestHandler.class); - this.streamsHandler = mock(StreamsHandler.class); this.streamHandler = mock(StreamHandler.class); - this.streamsListRequestBody = mock(StreamsListRequestBody.class); - this.streamsListResponse = mock(StreamsListRead.class); this.streamReadRequestBody = mock(StreamReadRequestBody.class); this.streamReadResponse = mock(StreamRead.class); this.resolveManifestRequestBody = mock(ResolveManifestRequestBody.class); this.resolveManifest = mock(ResolveManifest.class); - this.controller = new ConnectorBuilderController(this.healthHandler, this.resolveManifestHandler, this.streamHandler, this.streamsHandler); + this.controller = new ConnectorBuilderController(this.healthHandler, this.resolveManifestHandler, this.streamHandler); } @Test @@ -68,19 +59,6 @@ void givenExceptionWhenReadStreamThenThrowSameException() { assertThrows(AirbyteCdkInvalidInputException.class, () -> this.controller.readStream(streamReadRequestBody)); } - @Test - void whenListStreamsThenReturnHandlerResponse() { - when(streamsHandler.listStreams(streamsListRequestBody)).thenReturn(streamsListResponse); - final StreamsListRead response = this.controller.listStreams(streamsListRequestBody); - assertEquals(streamsListResponse, response); - } - - @Test - void givenExceptionWhenListStreamsThenThrowSameException() { - when(streamsHandler.listStreams(any())).thenThrow(AirbyteCdkInvalidInputException.class); - assertThrows(AirbyteCdkInvalidInputException.class, () -> this.controller.listStreams(streamsListRequestBody)); - } - @Test void whenResolveManifestThenReturnHandlerResponse() { when(resolveManifestHandler.resolveManifest(resolveManifestRequestBody)).thenReturn(resolveManifest); diff --git a/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/fixtures/RecordStreamsList.json b/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/fixtures/RecordStreamsList.json deleted file mode 100644 index 359ea830ee6..00000000000 --- a/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/fixtures/RecordStreamsList.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "RECORD", - "record": { - "stream": "lists_stream", - "data": { - "streams": [ - { - "name": "a name", - "url": "http://a-url.com" - } - ] - }, - "emitted_at": 1679271449000 - } -} diff --git a/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/handlers/StreamsHandlerTest.java b/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/handlers/StreamsHandlerTest.java deleted file mode 100644 index 6dd64eda761..00000000000 --- a/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/handlers/StreamsHandlerTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.connector_builder.handlers; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.connector_builder.api.model.generated.StreamsListRead; -import io.airbyte.connector_builder.api.model.generated.StreamsListReadStreamsInner; -import io.airbyte.connector_builder.api.model.generated.StreamsListRequestBody; -import io.airbyte.connector_builder.exceptions.AirbyteCdkInvalidInputException; -import io.airbyte.connector_builder.exceptions.ConnectorBuilderException; -import io.airbyte.connector_builder.requester.AirbyteCdkRequester; -import java.io.IOException; -import java.util.List; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class StreamsHandlerTest { - - private static final StreamsListRead STREAM_LIST_READ = new StreamsListRead().streams( - List.of(new StreamsListReadStreamsInner().name("a name").url("a url"))); - private static final JsonNode A_CONFIG; - private static final JsonNode A_MANIFEST; - - static { - try { - A_CONFIG = new ObjectMapper().readTree("{\"config\": 1}"); - A_MANIFEST = new ObjectMapper().readTree("{\"manifest\": 1}"); - } catch (final JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - private AirbyteCdkRequester requester; - private StreamsHandler handler; - - @BeforeEach - void setUp() { - requester = mock(AirbyteCdkRequester.class); - handler = new StreamsHandler(requester); - } - - @Test - void whenListStreamsThenReturnRequesterResponse() throws Exception { - when(requester.listStreams(A_MANIFEST, A_CONFIG)).thenReturn(STREAM_LIST_READ); - final StreamsListRead response = handler.listStreams(new StreamsListRequestBody().manifest(A_MANIFEST).config(A_CONFIG)); - assertEquals(STREAM_LIST_READ, response); - } - - @Test - void givenIOExceptionWhenListStreamsThenRaiseConnectorBuilderException() throws Exception { - when(requester.listStreams(A_MANIFEST, A_CONFIG)).thenThrow(IOException.class); - assertThrows(ConnectorBuilderException.class, () -> handler.listStreams(new StreamsListRequestBody().manifest(A_MANIFEST).config(A_CONFIG))); - } - - @Test - void givenAirbyteCdkInvalidInputExceptionWhenListStreamsThenRaiseConnectorBuilderException() throws Exception { - when(requester.listStreams(A_MANIFEST, A_CONFIG)).thenThrow(AirbyteCdkInvalidInputException.class); - assertThrows(AirbyteCdkInvalidInputException.class, - () -> handler.listStreams(new StreamsListRequestBody().manifest(A_MANIFEST).config(A_CONFIG))); - } - -} diff --git a/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/requester/AirbyteCdkRequesterImplTest.java b/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/requester/AirbyteCdkRequesterImplTest.java index ab39878d8c7..cb26efa82f4 100644 --- a/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/requester/AirbyteCdkRequesterImplTest.java +++ b/airbyte-connector-builder-server/src/test/java/io/airbyte/connector_builder/requester/AirbyteCdkRequesterImplTest.java @@ -20,8 +20,6 @@ import io.airbyte.connector_builder.api.model.generated.StreamRead; import io.airbyte.connector_builder.api.model.generated.StreamReadAuxiliaryRequestsInner; import io.airbyte.connector_builder.api.model.generated.StreamReadSlicesInner; -import io.airbyte.connector_builder.api.model.generated.StreamsListRead; -import io.airbyte.connector_builder.api.model.generated.StreamsListReadStreamsInner; import io.airbyte.connector_builder.command_runner.SynchronousCdkCommandRunner; import io.airbyte.connector_builder.exceptions.AirbyteCdkInvalidInputException; import io.airbyte.protocol.models.AirbyteRecordMessage; @@ -32,13 +30,11 @@ class AirbyteCdkRequesterImplTest { - private static final String LIST_STREAMS_COMMAND = "list_streams"; private static final String READ_STREAM_COMMAND = "test_read"; private static final JsonNode A_CONFIG; private static final JsonNode A_MANIFEST; private static final String A_STREAM = "test"; private static final Integer A_LIMIT = 1; - private static final String EMPTY_CATALOG = ""; static { try { @@ -58,37 +54,6 @@ void setUp() { requester = new AirbyteCdkRequesterImpl(commandRunner); } - @Test - void whenListStreamsThenReturnAdaptedCommandRunnerResponse() throws Exception { - final ArgumentCaptor configCaptor = ArgumentCaptor.forClass(String.class); - when(commandRunner.runCommand(eq(LIST_STREAMS_COMMAND), configCaptor.capture(), eq(EMPTY_CATALOG))) - .thenReturn(new AirbyteRecordMessage().withData(new ObjectMapper() - .readTree("{\"streams\":[{\"name\":\"a name\", \"url\": \"a url\"}, {\"name\":\"another name\", \"url\": \"another url\"}]}"))); - - final StreamsListRead streamsListRead = requester.listStreams(A_MANIFEST, A_CONFIG); - - // assert returned object - assertEquals(2, streamsListRead.getStreams().size()); - assertEquals(new StreamsListReadStreamsInner().name("a name").url("a url"), streamsListRead.getStreams().get(0)); - assertEquals(new StreamsListReadStreamsInner().name("another name").url("another url"), streamsListRead.getStreams().get(1)); - - assertRunCommandArgs(configCaptor, LIST_STREAMS_COMMAND); - } - - @Test - void givenNameIsNullWhenListStreamsThenThrowException() throws Exception { - when(commandRunner.runCommand(eq(LIST_STREAMS_COMMAND), any(), any())) - .thenReturn(new AirbyteRecordMessage().withData(new ObjectMapper().readTree("{\"streams\":[{\"url\": \"missing name\"}]}"))); - assertThrows(AirbyteCdkInvalidInputException.class, () -> requester.listStreams(A_MANIFEST, A_CONFIG)); - } - - @Test - void givenUrlIsNullWhenListStreamsThenThrowException() throws Exception { - when(commandRunner.runCommand(eq(LIST_STREAMS_COMMAND), any(), any())) - .thenReturn(new AirbyteRecordMessage().withData(new ObjectMapper().readTree("{\"streams\":[{\"name\": \"missing url\", \"url\": null}]}"))); - assertThrows(AirbyteCdkInvalidInputException.class, () -> requester.listStreams(A_MANIFEST, A_CONFIG)); - } - ArgumentCaptor testReadStreamSuccess(final Integer limit) throws Exception { final ObjectMapper mapper = new ObjectMapper(); mapper.registerModule(new JavaTimeModule()); diff --git a/airbyte-webapp/src/core/api/hooks/connectorBuilderApi.ts b/airbyte-webapp/src/core/api/hooks/connectorBuilderApi.ts index 4df85e9af80..3bc6bf09e42 100644 --- a/airbyte-webapp/src/core/api/hooks/connectorBuilderApi.ts +++ b/airbyte-webapp/src/core/api/hooks/connectorBuilderApi.ts @@ -4,14 +4,13 @@ import { DEFAULT_JSON_MANIFEST_VALUES } from "components/connectorBuilder/types" import { useCurrentWorkspaceId } from "area/workspace/utils"; -import { listStreams, readStream, resolveManifest } from "../generated/ConnectorBuilderClient"; +import { readStream, resolveManifest } from "../generated/ConnectorBuilderClient"; import { ConnectorConfig, ConnectorManifest, ResolveManifestRequestBody, StreamRead, StreamReadRequestBody, - StreamsListRequestBody, } from "../types/ConnectorBuilderClient"; import { DeclarativeComponentSchema } from "../types/ConnectorManifest"; import { useRequestOptions } from "../useRequestOptions"; @@ -43,21 +42,6 @@ export const useBuilderReadStream = ( }); }; -export const useBuilderListStreams = (params: StreamsListRequestBody, enabled = true) => { - const requestOptions = useRequestOptions(); - - return useQuery( - connectorBuilderKeys.list(params.manifest, params.config), - () => listStreams(params, requestOptions), - { - keepPreviousData: true, - cacheTime: 0, - retry: false, - enabled, - } - ); -}; - export const useBuilderResolvedManifest = (params: ResolveManifestRequestBody, enabled = true) => { const requestOptions = useRequestOptions();