diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 85f125af74..9741a334fa 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -40,7 +40,7 @@ jobs: ) strategy: matrix: - tests: [bigquery, common, gcs, pubsub, spanner, gcsdelete, gcsmove, bigqueryexecute] + tests: [bigquery, common, gcs, pubsub, spanner, gcsdelete, gcsmove, bigqueryexecute, dataplex] fail-fast: false steps: # Pinned 1.0.0 version diff --git a/src/e2e-test/features/dataplex/sink/BigQueryToDataplex.feature b/src/e2e-test/features/dataplex/sink/BigQueryToDataplex.feature new file mode 100644 index 0000000000..44f843e4f2 --- /dev/null +++ b/src/e2e-test/features/dataplex/sink/BigQueryToDataplex.feature @@ -0,0 +1,38 @@ +@Dataplex_Sink +Feature: Dataplex sink - Verification of BQ source to dataplex sink with update metadata enabled + + @Dataplex_SINK_TEST @GCS_SINK_TEST @BQ_SOURCE_TEST @Dataplex_Sink_Required + Scenario:Validate successful records transfer from BigQuery to Dataplex with update metadata enabled + Given Open Datafusion Project to configure pipeline + When Source is BigQuery + When Sink is Dataplex + Then Connect source as "BigQuery" and sink as "Dataplex" to establish connection + Then Open BigQuery source properties + Then Enter BigQuery property reference name + Then Enter BigQuery property projectId "projectId" + Then Enter BigQuery property datasetProjectId "projectId" + Then Enter BigQuery property dataset "dataset" + Then Enter BigQuery source property table name + Then Override Service account details if set in environment variables + Then Enter BiqQuery property encryption key name "cmekBQ" if cmek is enabled + Then Validate output schema with expectedSchema "bqSourceSchema" + Then Validate "BigQuery" plugin properties + Then Close the BigQuery properties + Then Open Dataplex sink properties + Then Override Service account details if set in environment variables + Then Enter the Dataplex mandatory properties + Then Enter the Dataplex sink mandatory properties + Then Enable Metadata Update + Then Validate "Dataplex" plugin properties + Then Close the Dataplex properties + Then Preview and run the pipeline + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Save and Deploy Pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" diff --git a/src/e2e-test/features/dataplex/sink/DataplexSourceToDataplexSink.feature b/src/e2e-test/features/dataplex/sink/DataplexSourceToDataplexSink.feature new file mode 100644 index 0000000000..f26bd9d2de --- /dev/null +++ b/src/e2e-test/features/dataplex/sink/DataplexSourceToDataplexSink.feature @@ -0,0 +1,56 @@ +@Dataplex_Sink +Feature: Dataplex sink - Verification of BQ source to dataplex sink with update metadata enabled + + @Dataplex_SINK_TEST @GCS_SINK_TEST @BQ_SOURCE_TEST + Scenario:Validate successful records transfer from BigQuery to Dataplex with update metadata enabled + Given Open Datafusion Project to configure pipeline + When Source is BigQuery + When Sink is Dataplex + Then Connect source as "BigQuery" and sink as "Dataplex" to establish connection + Then Open BigQuery source properties + Then Enter BigQuery property reference name + Then Enter BigQuery property projectId "projectId" + Then Enter BigQuery property datasetProjectId "projectId" + Then Enter BigQuery property dataset "dataset" + Then Enter BigQuery source property table name + Then Override Service account details if set in environment variables + Then Enter BiqQuery property encryption key name "cmekBQ" if cmek is enabled + Then Validate output schema with expectedSchema "bqSourceSchema" + Then Validate "BigQuery" plugin properties + Then Close the BigQuery properties + Then Open Dataplex sink properties + Then Override Service account details if set in environment variables + Then Enter the Dataplex mandatory properties + Then Enter the Dataplex sink mandatory properties + Then Enable Metadata Update + Then Validate "Dataplex" plugin properties + Then Close the Dataplex properties + Then Save and Deploy Pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Given Open Datafusion Project to configure pipeline + When Source is Dataplex + When Sink is Dataplex + Then Connect source as "Dataplex" and sink as "Dataplex" to establish connection + Then Open Dataplex source properties + Then Enter the Dataplex mandatory properties + Then Enter the Dataplex source mandatory properties + Then Override Service account details if set in environment variables + Then Validate output schema with expectedSchema "dataplexSourceSchema" + Then Validate "Dataplex" plugin properties + Then Close the Dataplex properties + Then Open Dataplex sink properties + Then Override Service account details if set in environment variables + Then Enter the Dataplex mandatory properties + Then Enter the Dataplex sink mandatory properties + Then Enable Metadata Update + Then Remove "ts" column from output schema + Then Validate "Dataplex" plugin properties + Then Close the Dataplex properties + Then Save and Deploy Pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" diff --git a/src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/TestRunner.java new file mode 100644 index 0000000000..5f9bf49fdc --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/TestRunner.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.dataplex.runners.sinkrunner; + +import io.cucumber.junit.Cucumber; +import io.cucumber.junit.CucumberOptions; +import org.junit.runner.RunWith; + + +/** + * Test Runner to execute Dataplex sink cases. + */ +@RunWith(Cucumber.class) +@CucumberOptions( + features = {"src/e2e-test/features"}, + glue = {"io.cdap.plugin.dataplex.stepsdesign", "io.cdap.plugin.gcs.stepsdesign", + "io.cdap.plugin.bigquery.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"}, + tags = {"@Dataplex_Sink and not @ignore"}, + monochrome = true, + plugin = {"pretty", "html:target/cucumber-html-report/dataplex-sink", + "json:target/cucumber-reports/cucumber-dataplex-sink.json", + "junit:target/cucumber-reports/cucumber-dataplex-sink.xml"} +) +public class TestRunner { +} diff --git a/src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/TestRunnerRequired.java b/src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/TestRunnerRequired.java new file mode 100644 index 0000000000..13744e9c3b --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/TestRunnerRequired.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.dataplex.runners.sinkrunner; + +import io.cucumber.junit.Cucumber; +import io.cucumber.junit.CucumberOptions; +import org.junit.runner.RunWith; + +/** + * Test Runner to execute only required dataplex Sink plugin test cases. + */ +@RunWith(Cucumber.class) +@CucumberOptions( + features = {"src/e2e-test/features"}, + glue = {"io.cdap.plugin.dataplex.stepsdesign", "io.cdap.plugin.gcs.stepsdesign", + "io.cdap.plugin.bigquery.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"}, + tags = {"@Dataplex_Sink_Required and not @ignore"}, + monochrome = true, + plugin = {"pretty", "html:target/cucumber-html-report/dataplex-sink-required", + "json:target/cucumber-reports/cucumber-dataplex-sink-required.json", + "junit:target/cucumber-reports/cucumber-dataplex-sink-required.xml"} +) +public class TestRunnerRequired { +} diff --git a/src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/package-info.java b/src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/package-info.java new file mode 100644 index 0000000000..99d2bc510a --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * Package contains the runner for the Dataplex sink features. + */ +package io.cdap.plugin.dataplex.runners.sinkrunner; diff --git a/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexBase.java b/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexBase.java new file mode 100644 index 0000000000..530524a045 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexBase.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.dataplex.stepsdesign; + +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.plugin.utils.DataplexHelper; +import io.cdap.plugin.utils.E2EHelper; +import io.cucumber.java.en.Then; + +import java.io.IOException; + +/** + * Dataplex related common stepDesigns. + */ +public class DataplexBase implements E2EHelper { + @Then("Enter the Dataplex mandatory properties") + public void enterTheDataplexMandatoryProperties() throws IOException { + DataplexHelper.enterReferenceName(); + DataplexHelper.enterProjectId(); + DataplexHelper.enterDataplexProperty( + "location", PluginPropertyUtils.pluginProp("dataplexDefaultLocation")); + DataplexHelper.enterDataplexProperty("lake", PluginPropertyUtils.pluginProp("dataplexDefaultLake")); + DataplexHelper.enterDataplexProperty("zone", PluginPropertyUtils.pluginProp("dataplexDefaultZone")); + } +} diff --git a/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexSink.java b/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexSink.java new file mode 100644 index 0000000000..7f328aae4d --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexSink.java @@ -0,0 +1,67 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.dataplex.stepsdesign; + +import io.cdap.e2e.pages.actions.CdfPluginPropertiesActions; +import io.cdap.e2e.pages.actions.CdfStudioActions; +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.plugin.common.stepsdesign.TestSetupHooks; +import io.cdap.plugin.utils.DataplexHelper; +import io.cdap.plugin.utils.E2EHelper; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; + +import java.io.IOException; + + +/** + * Dataplex Sink related stepDesigns. + */ +public class DataplexSink implements E2EHelper { + + @When("Sink is Dataplex") + public void sinkIsDataplex() { + CdfStudioActions.clickSink(); + selectSinkPlugin("Dataplex"); + } + + @Then("Open Dataplex sink properties") + public void openDataplexSinkProperties() { + openSinkPluginProperties("Dataplex"); + } + + @Then("Close the Dataplex properties") + public void closeTheBigQueryProperties() { + CdfPluginPropertiesActions.clickCloseButton(); + } + + @Then("Enter the Dataplex sink mandatory properties") + public void enterTheDataplexSinkMandatoryProperties() throws IOException { + DataplexHelper.enterDataplexProperty("asset", PluginPropertyUtils.pluginProp("dataplexDefaultAsset")); + DataplexHelper.setAssetType("STORAGE_BUCKET"); + DataplexHelper.enterDataplexProperty("table", TestSetupHooks.gcsTargetBucketName); + } + + @Then("Enable Metadata Update") + public void enableMetadataUpdate() { + DataplexHelper.toggleMetadataUpdate(); + } + + @Then("Remove {string} column from output schema") + public void removeColumnFromOutputSchema(String fieldName) throws InterruptedException { + DataplexHelper.deleteSchemaField(fieldName); + } +} diff --git a/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexSource.java b/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexSource.java new file mode 100644 index 0000000000..9d03b8c0a9 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexSource.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.dataplex.stepsdesign; + +import io.cdap.plugin.common.stepsdesign.TestSetupHooks; +import io.cdap.plugin.utils.DataplexHelper; +import io.cdap.plugin.utils.E2EHelper; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; + +import java.io.IOException; + +/** + * Dataplex Source related stepDesigns. + */ +public class DataplexSource implements E2EHelper { + + @When("Source is Dataplex") + public void sourceIsDataplex() { + selectSourcePlugin("Dataplex"); + } + + @Then("Open Dataplex source properties") + public void openDataplexSourceProperties() { + openSourcePluginProperties("Dataplex"); + } + + @Then("Enter the Dataplex source mandatory properties") + public void enterTheDataplexSourceMandatoryProperties() throws IOException { + DataplexHelper.enterDataplexProperty( + "entity", TestSetupHooks.gcsTargetBucketName.replaceAll("[^a-zA-Z0-9_]", "_")); + } +} diff --git a/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/package-info.java b/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/package-info.java new file mode 100644 index 0000000000..bf857e9479 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * This package contains stepDesigns for Dataplex Plugin. + */ +package io.cdap.plugin.dataplex.stepsdesign; diff --git a/src/e2e-test/java/io/cdap/plugin/utils/DataplexHelper.java b/src/e2e-test/java/io/cdap/plugin/utils/DataplexHelper.java new file mode 100644 index 0000000000..a29e1d00f9 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/utils/DataplexHelper.java @@ -0,0 +1,82 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.utils; + + +import io.cdap.e2e.utils.ElementHelper; +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.e2e.utils.SeleniumDriver; +import org.openqa.selenium.By; +import org.openqa.selenium.WebElement; + +import java.io.IOException; +import java.util.UUID; + +/** + * Dataplex E2E tests helpers. + */ +public class DataplexHelper { + public static String getCssSelectorByDataTestId (String dataTestId) { + return "[data-testid=" + dataTestId + "]"; + } + + public static WebElement locateElementByCssSelector(String cssSelector) { + return SeleniumDriver.getDriver() + .findElement(By.cssSelector(cssSelector)); + } + + public static WebElement locateElementByXPath(String xpath) { + return SeleniumDriver.getDriver() + .findElement(By.xpath(xpath)); + } + + public static void enterReferenceName() { + ElementHelper.sendKeys(locateElementByXPath( + "//input[@data-testid='referenceName']" + ), "Dataplex_Ref_" + UUID.randomUUID()); + } + + public static void enterProjectId() throws IOException { + ElementHelper.replaceElementValue(locateElementByXPath( + "//input[@data-testid='project']" + ), PluginPropertyUtils.pluginProp("projectId")); + } + + public static void enterDataplexProperty (String prop, String value) { + ElementHelper.sendKeys(locateElementByXPath( + "//input[@data-testid='" + prop + "']" + ), value); + } + + public static void setAssetType (String assetType) { + String xpath = "//input[@name='assetType' and @value='" + assetType + "']"; + ElementHelper.selectRadioButton(locateElementByXPath(xpath)); + } + + public static void toggleMetadataUpdate() { + ElementHelper.clickOnElement(locateElementByCssSelector( + getCssSelectorByDataTestId("switch-updateDataplexMetadata") + )); + } + + public static void deleteSchemaField(String fieldName) { + String xpath = "//div[contains(@data-cy,'Output Schema')]//input[@value='" + fieldName + "']" + + "/ancestor::div[contains(@data-cy,'schema-row')]" + "//*[@data-cy='schema-field-remove-button']"; + ElementHelper.clickOnElement(locateElementByXPath(xpath)); + + } + +} diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 6a106920ed..bea4678694 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -252,3 +252,11 @@ bqExecuteDMLUpsert=MERGE `PROJECT_NAME.DATASET.TABLENAME` as T \ bqExecuteCountDMLUpsertInsert=SELECT COUNT(*) FROM `PROJECT_NAME.DATASET.TABLENAME` WHERE Id=101 AND Value=5000 AND UID='INSERTED RECORD' bqExecuteCountDMLUpsertUpdate=SELECT COUNT(*) FROM `PROJECT_NAME.DATASET.TABLENAME` WHERE Id=101 AND Value=5000 AND UID='UPDATED RECORD' ## BQEXECUTE-PLUGIN-PROPERTIES-END + +## DATAPLEX-PLUGIN-PROPERTIES-START +dataplexDefaultLocation=us-central1 +dataplexDefaultLake=e2e-test-lake +dataplexDefaultZone=e2e-test-zone +dataplexDefaultAsset=e2e-sb-asset +dataplexSourceSchema=[{"key":"Id","value":"long"},{"key":"Value","value":"long"},{"key":"UID","value":"string"},{"key":"ts","value":"string"}] +## DATAPLEX-PLUGIN-PROPERTIES-END \ No newline at end of file diff --git a/src/main/java/io/cdap/plugin/gcp/dataplex/common/config/DataplexBaseConfig.java b/src/main/java/io/cdap/plugin/gcp/dataplex/common/config/DataplexBaseConfig.java index 7c7375657f..3995fc040d 100644 --- a/src/main/java/io/cdap/plugin/gcp/dataplex/common/config/DataplexBaseConfig.java +++ b/src/main/java/io/cdap/plugin/gcp/dataplex/common/config/DataplexBaseConfig.java @@ -225,11 +225,13 @@ public String getServiceAccountEmail() throws IOException { if (connection.isServiceAccountJson() || connection.getServiceAccountFilePath() != null) { try (InputStream inputStream = GCPUtils.openServiceAccount(getServiceAccount(), isServiceAccountFilePath())) { ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream(inputStream); + LOG.info("Using Service Account: " + credentials.getClientEmail()); return credentials.getClientEmail(); } } ComputeEngineCredentials credentials = (ComputeEngineCredentials) ServiceAccountCredentials. getApplicationDefault(); + LOG.info("Using Service Account: " + credentials.getAccount()); // Adding this code for preview as dataplex doesn't allow user to act as Cloud Data Fusion API Service Agent and // create tasks in dataplex. So we are passing default Compute Engine service account in task creation. // This code will fetch the project number from service Agent and form compute service account email.