added flag to support sharded migrations via cli #4737
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2020 Google LLC | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| name: integration-tests-against-emulator | |
| jobs: | |
| integration-tests: | |
| runs-on: ubuntu-latest | |
| env: | |
| # set PostgreSQL related environment variables | |
| PGHOST: localhost | |
| PGPORT: 5432 | |
| PGUSER: postgres | |
| PGDATABASE: postgres | |
| PGPASSWORD: postgres | |
| # set MySQL related environment variables | |
| DB_HOST: localhost | |
| MYSQLHOST: localhost | |
| MYSQLPORT: 3306 | |
| MYSQLUSER: root | |
| MYSQLDATABASE: test_interleave_table_data | |
| MYSQLDB_FKACTION: test_foreign_key_action_data | |
| MYSQLPWD: root | |
| # set DynamoDB related environment variables | |
| AWS_ACCESS_KEY_ID: dummyId | |
| AWS_SECRET_ACCESS_KEY: dummyKey | |
| AWS_REGION: dummyRegion | |
| DYNAMODB_ENDPOINT_OVERRIDE: http://localhost:8000 | |
| # sql server envs | |
| # Password for connection as SA | |
| MSSQL_SA_PASSWORD: tCUE9c1&Ucp0 | |
| services: | |
| spanner_emulator: | |
| image: gcr.io/cloud-spanner-emulator/emulator:1.5.23 | |
| ports: | |
| - 9010:9010 | |
| - 9020:9020 | |
| postgres: | |
| image: postgres:9.6 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432:5432 | |
| # needed because the postgres container does not provide a healthcheck | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| mariadb: | |
| image: mariadb:10.11 | |
| env: | |
| MYSQL_ROOT_PASSWORD: root | |
| MYSQL_DATABASE: test | |
| ports: | |
| - 3306:3306 | |
| # needed because the mysql container does not provide a healthcheck | |
| options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 | |
| dynamodb_emulator: | |
| image: amazon/dynamodb-local:1.16.0 | |
| ports: | |
| - 8000:8000 | |
| options: --workdir /home/dynamodblocal --health-cmd "curl --fail http://127.0.0.1:8000/shell/ || exit 1" --health-interval 10s --health-timeout 5s --health-retries 5 | |
| sqlserver: | |
| image: mcr.microsoft.com/mssql/server:2022-latest | |
| env: | |
| SA_PASSWORD: ${{env.MSSQL_SA_PASSWORD}} | |
| MSSQL_PID: Express | |
| ACCEPT_EULA: Y | |
| ports: | |
| - 1433:1433 | |
| options: | |
| --health-cmd "/opt/mssql-tools18/bin/sqlcmd -U sa -P $SA_PASSWORD -Q 'select 1' -b -No -o /dev/null" | |
| --health-interval 10s --health-timeout 5s --health-retries 3 | |
| oracle: | |
| image: oracleinanutshell/oracle-xe-11g | |
| ports: | |
| - 1521:1521 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # init a PostgresSQL database from the test_data | |
| - name: Install PostgreSQL 12 client required for loading .sql files | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -yq install postgresql-client | |
| - run: psql --version | |
| - run: psql -f test_data/pg_dump.test.out | |
| # init a MySQL database from the test_data | |
| - run: mysql --version | |
| - run: mysql -v -P 3306 --protocol=tcp -u root -proot test < test_data/mysqldump.test.out | |
| - run: mysql -v -P 3306 --protocol=tcp -u root -proot < test_data/mysql_interleave_dump.test.out | |
| - run: mysql -v -P 3306 --protocol=tcp -u root -proot < test_data/mysql_foreignkeyaction_dump.test.out | |
| # init sql server with test_data | |
| - name: Install sqlcmd required for loading .sql files | |
| run: | | |
| curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - | |
| curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list | |
| sudo apt-get update | |
| sudo apt-get install mssql-tools unixodbc-dev | |
| echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bash_profile | |
| - run: sqlcmd -? | |
| - run: sqlcmd -U sa -P ${MSSQL_SA_PASSWORD} -i test_data/sqlserver.test.out | |
| # sqlplus set up init oracle db. | |
| - name: Install sqlplus required for loading .sql files | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libaio1 rpm2cpio cpio | |
| curl -O https://download.oracle.com/otn_software/linux/instantclient/2340000/oracle-instantclient-basic-23.4.0.24.05-1.el9.x86_64.rpm | |
| curl -O https://download.oracle.com/otn_software/linux/instantclient/2340000/oracle-instantclient-sqlplus-23.4.0.24.05-1.el9.x86_64.rpm | |
| rpm2cpio oracle-instantclient-basic-23.4.0.24.05-1.el9.x86_64.rpm | sudo cpio -idmv | |
| sudo cp -r usr/* /usr/ | |
| rpm2cpio oracle-instantclient-sqlplus-23.4.0.24.05-1.el9.x86_64.rpm | sudo cpio -idmv | |
| sudo cp -r usr/* /usr/ | |
| sudo sh -c 'echo /usr/lib/oracle/23/client64/lib/ > /etc/ld.so.conf.d/oracle.conf' | |
| sudo ldconfig | |
| - run: sqlplus SYS/[email protected]:1521/xe as sysdba @test_data/oracle.test.out | |
| # create a spanner instance | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| - uses: google-github-actions/setup-gcloud@v0 | |
| with: | |
| version: "410.0.0" | |
| - run: gcloud info | |
| - run: gcloud config configurations create emulator | |
| - run: gcloud config set auth/disable_credentials true | |
| - run: gcloud config set project emulator-test-project | |
| - run: gcloud config set api_endpoint_overrides/spanner http://localhost:9020/ | |
| - run: gcloud spanner instances create test-instance --config=emulator-config --description="Test Instance" --nodes=1 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v2 | |
| with: | |
| node-version: 16 | |
| - name: Install Dependencies | |
| run: | | |
| cd ui | |
| npm ci | |
| - name: Start Local Server | |
| run: | | |
| cd ui | |
| npm start & | |
| - name: Wait for Local Server to Start | |
| run: npx wait-on http://localhost:4200 -t 30000 | |
| - name: Cypress run | |
| run: | | |
| cd ui | |
| npx cypress run | |
| - uses: actions/setup-go@v2 | |
| with: | |
| go-version: "1.19" | |
| - run: go version | |
| - run: go test -v ./... | |
| env: | |
| SPANNER_EMULATOR_HOST: localhost:9010 | |
| SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_PROJECT_ID: emulator-test-project | |
| SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_INSTANCE_ID: test-instance |