From 74e2152e47ef2018b70193a88b5285be7f304248 Mon Sep 17 00:00:00 2001 From: priyabhatnagar Date: Fri, 28 Jul 2023 21:04:49 +0530 Subject: [PATCH] BQMT e2e Tests Scenarios --- .github/workflows/e2e.yml | 38 ++- pom.xml | 7 + .../BiqQueryMultiTableSinkError.feature | 79 +++++++ ...ToBigQueryMultiTableSink_WithMacro.feature | 131 +++++++++++ ...abaseTableToBiqQueryMultiTableSink.feature | 104 +++++++++ .../runners/TestRunner.java | 37 +++ .../runners/TestRunnerRequired.java | 38 +++ .../runners/package-info.java | 4 + .../stepdesign/BQMultiTableValidation.java | 220 ++++++++++++++++++ .../stepdesign/BigQueryMultiTable.java | 42 ++++ .../stepdesign/MultiTableClient.java | 67 ++++++ .../stepdesign/package-info.java | 20 ++ .../common/stepsdesign/TestSetupHooks.java | 22 ++ .../resources/errorMessage.properties | 5 + .../pluginDataCyAttributes.properties | 2 + .../resources/pluginParameters.properties | 16 ++ 16 files changed, 828 insertions(+), 4 deletions(-) create mode 100644 src/e2e-test/features/bigquerymultitable/BiqQueryMultiTableSinkError.feature create mode 100644 src/e2e-test/features/bigquerymultitable/MultipleDatabaseTableToBigQueryMultiTableSink_WithMacro.feature create mode 100644 src/e2e-test/features/bigquerymultitable/MultipleDatabaseTableToBiqQueryMultiTableSink.feature create mode 100644 src/e2e-test/java/io/cdap/plugin/bigquerymultitable/runners/TestRunner.java create mode 100644 src/e2e-test/java/io/cdap/plugin/bigquerymultitable/runners/TestRunnerRequired.java create mode 100644 src/e2e-test/java/io/cdap/plugin/bigquerymultitable/runners/package-info.java create mode 100644 src/e2e-test/java/io/cdap/plugin/bigquerymultitable/stepdesign/BQMultiTableValidation.java create mode 100644 src/e2e-test/java/io/cdap/plugin/bigquerymultitable/stepdesign/BigQueryMultiTable.java create mode 100644 src/e2e-test/java/io/cdap/plugin/bigquerymultitable/stepdesign/MultiTableClient.java create mode 100644 src/e2e-test/java/io/cdap/plugin/bigquerymultitable/stepdesign/package-info.java diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1db5015306..38c0bc3177 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,4 +1,4 @@ -# Copyright © 2021 Cask Data, Inc. +# Copyright © 2021-2023 Cask Data, Inc. # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of # the License at @@ -12,6 +12,7 @@ # This workflow will build a Java project with Maven # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven # Note: Any changes to this workflow would be used only after merging into develop + name: Build e2e tests on: @@ -40,7 +41,8 @@ jobs: ) strategy: matrix: - tests: [bigquery, common, gcs, pubsub, spanner, gcscreate, gcsdelete, gcsmove, bigqueryexecute, gcscopy] + tests: [bigquery, common, gcs, pubsub, spanner, gcscreate, gcsdelete, gcsmove, bigqueryexecute, + bigquerymultitable, gcscopy] fail-fast: false steps: # Pinned 1.0.0 version @@ -68,24 +70,52 @@ jobs: key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven-${{ github.workflow }} + + - name: Get Secrets from GCP Secret Manager + id: secrets + uses: 'google-github-actions/get-secretmanager-secrets@v0' + with: + secrets: |- + MYSQL_HOST:cdapio-github-builds/MYSQL_HOST + MYSQL_USERNAME:cdapio-github-builds/MYSQL_USERNAME + MYSQL_PASSWORD:cdapio-github-builds/MYSQL_PASSWORD + MYSQL_PORT:cdapio-github-builds/MYSQL_PORT + BQMT_CONNECTION_STRING:cdapio-github-builds/BQMT_CONNECTION_STRING + - name: Run required e2e tests if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' && steps.filter.outputs.e2e-test == 'false' run: python3 e2e/src/main/scripts/run_e2e_test.py --testRunner **/${{ matrix.tests }}/**/TestRunnerRequired.java + env: + MYSQL_HOST: ${{ steps.secrets.outputs.MYSQL_HOST }} + MYSQL_USERNAME: ${{ steps.secrets.outputs.MYSQL_USERNAME }} + MYSQL_PASSWORD: ${{ steps.secrets.outputs.MYSQL_PASSWORD }} + MYSQL_PORT: ${{ steps.secrets.outputs.MYSQL_PORT }} + BQMT_CONNECTION_STRING: ${{ steps.secrets.outputs.BQMT_CONNECTION_STRING }} + - name: Run all e2e tests if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || steps.filter.outputs.e2e-test == 'true' run: python3 e2e/src/main/scripts/run_e2e_test.py --testRunner **/${{ matrix.tests }}/**/TestRunner.java - - name: Upload report + env: + MYSQL_HOST: ${{ steps.secrets.outputs.MYSQL_HOST }} + MYSQL_USERNAME: ${{ steps.secrets.outputs.MYSQL_USERNAME }} + MYSQL_PASSWORD: ${{ steps.secrets.outputs.MYSQL_PASSWORD }} + MYSQL_PORT: ${{ steps.secrets.outputs.MYSQL_PORT }} + BQMT_CONNECTION_STRING: ${{ steps.secrets.outputs.BQMT_CONNECTION_STRING }} + + - name: Upload report uses: actions/upload-artifact@v3 if: always() with: name: Cucumber report - ${{ matrix.tests }} path: ./plugin/target/cucumber-reports - - name: Upload debug files + + - name: Upload debug files uses: actions/upload-artifact@v3 if: always() with: name: Debug files - ${{ matrix.tests }} path: ./**/target/e2e-debug + - name: Upload files to GCS uses: google-github-actions/upload-cloud-storage@v0 if: always() diff --git a/pom.xml b/pom.xml index 22136d13b8..496918c625 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,5 @@ +