Merge pull request #75 from d-ylee/migrationRequestTests

Migration Request Integration Tests Concept

.github/workflows/go.yml

@@ -7,7 +7,6 @@ on:
     branches: [ master ]
 
 jobs:
-
   build:
     runs-on: ubuntu-latest
     steps:
@@ -18,7 +17,15 @@ jobs:
       with:
         go-version: 1.18
 
-    - name: Build and test
+    - name: Build
+      env:
+        GOPATH: /home/runner/go
+      run: |
+        sed -i -e "s,_ \"github.com/mattn/go-oci8\",,g" web/server.go
+        sed -i -e "s,_ \"gopkg.in/rana/ora.v4\",,g" web/server.go
+        make
+
+    - name: Prepare code for tests and test DBS
       env:
         GOPATH: /home/runner/go
         PKG_CONFIG_PATH: /home/runner/go
@@ -32,4 +39,73 @@ jobs:
         sed -i -e "s,_ \"github.com/mattn/go-oci8\",,g" test/merge/main.go
         mkdir -p $GOPATH/src/github.com/vkuznet
         cp -r ../dbs2go $GOPATH/src/github.com/vkuznet
-        make test-github
+
+    - name: Test SQL
+      env:
+        GOPATH: /home/runner/go
+        PKG_CONFIG_PATH: /home/runner/go
+      run: |
+        make test-sql
+
+    - name: Test validator
+      env:
+        GOPATH: /home/runner/go
+        PKG_CONFIG_PATH: /home/runner/go
+      run: |
+        make test-validator
+
+    - name: Test bulk
+      env:
+        GOPATH: /home/runner/go
+        PKG_CONFIG_PATH: /home/runner/go
+      run: |
+        make test-bulk
+
+    - name: Test http
+      env:
+        GOPATH: /home/runner/go
+        PKG_CONFIG_PATH: /home/runner/go
+      run: |
+        make test-http
+
+    - name: Test utils
+      env:
+        GOPATH: /home/runner/go
+        PKG_CONFIG_PATH: /home/runner/go
+      run: |
+        make test-utils
+
+    - name: Test writer
+      env:
+        GOPATH: /home/runner/go
+        PKG_CONFIG_PATH: /home/runner/go
+      run: |
+        make test-writer
+
+    - name: Test integration
+      env:
+        GOPATH: /home/runner/go
+        PKG_CONFIG_PATH: /home/runner/go
+      run: |
+        make test-integration
+
+    - name: Test migration requests
+      env:
+        GOPATH: /home/runner/go
+        PKG_CONFIG_PATH: /home/runner/go
+      run: |
+        make test-migration-requests
+
+    - name: Test migration
+      env:
+        GOPATH: /home/runner/go
+        PKG_CONFIG_PATH: /home/runner/go
+      run: |
+        make test-migration
+
+    - name: Benchmark
+      env:
+        GOPATH: /home/runner/go
+        PKG_CONFIG_PATH: /home/runner/go
+      run: |
+        make bench

.gitignore

@@ -39,9 +39,13 @@ migrate.json
 migration.json
 test/data/dbsdata
 .env
-.vscode
+*.tmp
 test-pids
 
 # custom development files
 .devcontainer
-.vscode
+.vscode
+
+.globus
+instantclient*
+.DS_Store

Makefile

@@ -40,7 +40,7 @@ clean:
 
 test: test-dbs test-sql test-errors test-validator test-bulk test-http test-utils test-migrate test-writer test-integration test-lexicon bench
 
-test-github: test-dbs test-sql test-errors test-validator test-bulk test-http test-utils test-writer test-lexicon test-integration test-migration bench
+test-github: test-dbs test-sql test-errors test-validator test-bulk test-http test-utils test-writer test-lexicon test-integration test-migration-requests test-migration bench
 
 test-lexicon: test-lexicon-writer-pos test-lexicon-writer-neg test-lexicon-reader-pos test-lexicon-reader-neg
 
@@ -157,25 +157,19 @@ test-integration:
 	FILE_LUMI_LIST_LENGTH=30 \
 	go test -v -failfast -run Integration
 test-migration:
-	cd test && rm -f /tmp/dbs-one.db && \
-	sqlite3 /tmp/dbs-one.db < ../static/schema/sqlite-schema.sql && \
-	echo sqlite3 /tmp/dbs-one.db sqlite > ./dbfile_1 && \
-	rm -f /tmp/dbs-two.db && \
-	sqlite3 /tmp/dbs-two.db < ../static/schema/sqlite-schema.sql && \
-	echo sqlite3 /tmp/dbs-two.db sqlite > ./dbfile_2 && \
-	cd .. && \
+	LD_LIBRARY_PATH=${odir} DYLD_LIBRARY_PATH=${odir} \
 	./bin/start_test_migration && \
 	cd test && \
 	LD_LIBRARY_PATH=${odir} DYLD_LIBRARY_PATH=${odir} \
-	DBS_API_PARAMETERS_FILE=../static/parameters.json \
-	DBS_READER_LEXICON_FILE=../static/lexicon_reader.json \
-	DBS_WRITER_LEXICON_FILE=../static/lexicon_writer.json \
-	DBS_DB_FILE_1=./dbfile_1 \
-	DBS_DB_FILE_2=./dbfile_2 \
-	DBS_DB_PATH_1=/tmp/dbs-one.db \
-	DBS_DB_PATH_2=/tmp/dbs-two.db \
 	BULKBLOCKS_DATA_FILE=./data/migration/bulkblocks_data.json \
-	go test -v -failfast -run IntMigration
+	go test -v -failfast -timeout 10m -run IntMigration
+test-migration-requests:
+	LD_LIBRARY_PATH=${odir} DYLD_LIBRARY_PATH=${odir} \
+	./bin/start_test_migration && \
+	cd test && \
+	LD_LIBRARY_PATH=${odir} DYLD_LIBRARY_PATH=${odir} \
+	MIGRATION_REQUESTS_PATH=./data/migration/requests \
+	go test -v -failfast -timeout 10m -run MigrationRequests
 bench:
 	cd test && rm -f /tmp/dbs-test.db && \
 	sqlite3 /tmp/dbs-test.db < ../static/schema/sqlite-schema.sql && \

bin/db_snapshot

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Starts the DBSMigrate and DBSMigration servers for migration request tests
+# TODO: Creates a dbs database snapshot depending on migration requests files
+# X509_USER_CERT and X509_USER_KEY need to be defined in order to grab data from the production server
+
+# Stop currently running dbs2go processes
+pkill dbs2go
+
+# Reset the sqlite database for the snapshot
+cd test && rm -f /tmp/dbs-test.db && \
+sqlite3 /tmp/dbs-test.db < ../static/schema/sqlite-schema.sql && \
+
+cd ..
+
+# Start dbs-migrate
+X509_USER_CERT=$X509_USER_CERT X509_USER_KEY=$X509_USER_KEY \
+  ./dbs2go -config ./migrate.json &
+echo $! >> ./test-pids
+sleep 1
+
+# Start dbs-migration
+X509_USER_CERT=$X509_USER_CERT X509_USER_KEY=$X509_USER_KEY \
+  ./dbs2go -config ./migration.json &
+echo $! >> ./test-pids
+
+sleep 1
+
+# Read and process migration requests from directory
+REQUESTS=$PWD/test/data/migration/requests/*.json
+for f in $REQUESTS
+do
+    echo "Processing $f file..."
+    X509_USER_CERT=$X509_USER_CERT X509_USER_KEY=$X509_USER_KEY \
+        curl -H "Content-Type: application/json" \
+        http://localhost:9898/dbs2go/submit \
+        -d@$f
+done
+#curl --cacert "$CA_CERT" --cert $X509_USER_CERT --key $X509_USER_KEY --header 'Content-Type: application/json' --verbose https://cmsweb.cern.ch/dbs/prod/global/DBSReader/blockdump?block_name=/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM

bin/start_test_migration

@@ -1,45 +1,45 @@
 #!/bin/bash
 
-echo "building dbs2go"
-make build
-
-# create pid file
-rm ./test-pids
-touch ./test-pids
-
-pwd
-
-# Start dbs-one-reader
+# Delete original dbfiles and databases and reinitializing them with the schema
+rm -f /tmp/dbs-one.db
+sqlite3 /tmp/dbs-one.db < ./static/schema/sqlite-schema.sql
+echo sqlite3 /tmp/dbs-one.db sqlite > ./test/dbfile_1
+rm -f /tmp/dbs-two.db
+sqlite3 /tmp/dbs-two.db < ./static/schema/sqlite-schema.sql
+echo sqlite3 /tmp/dbs-two.db sqlite > ./test/dbfile_2
+
+# Starting DBS servers
+echo Starting dbs-one-reader
 ./dbs2go -config ./test/config/config_dbs_one_reader.json &
 echo $! >> ./test-pids
 
 sleep 1
 
-# Start dbs-one-writer
+echo Starting dbs-one-writer
 ./dbs2go -config ./test/config/config_dbs_one_writer.json &
 echo $! >> ./test-pids
 
 sleep 1
 
-# Start dbs-two-reader
+echo Starting dbs-two-reader
 ./dbs2go -config ./test/config/config_dbs_two_reader.json &
 echo $! >> ./test-pids
 
 sleep 1
 
-# Start dbs-two-writer
+echo Starting dbs-two-writer
 ./dbs2go -config ./test/config/config_dbs_two_writer.json &
 echo $! >> ./test-pids
 
 sleep 1
 
-# Start dbs-migrate
+echo Starting dbs-migrate
 ./dbs2go -config ./test/config/config_dbs_migrate.json &
 echo $! >> ./test-pids
 
 sleep 1
 
-# Start dbs-migration
+echo Starting dbs-migration
 ./dbs2go -config ./test/config/config_dbs_migration.json &
 echo $! >> ./test-pids
 

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/vkuznet/auth-proxy-server/logging v0.0.0-20220406163751-c36feb20c750
 	github.com/vkuznet/limiter v2.2.2+incompatible
 	github.com/vkuznet/x509proxy v0.0.0-20210801171832-e47b94db99b6
+	golang.org/x/exp v0.0.0-20220428152302-39d4317da171
 	gopkg.in/rana/ora.v4 v4.1.15
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -38,7 +39,6 @@ require (
 	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
-	golang.org/x/exp v0.0.0-20220428152302-39d4317da171 // indirect
 	golang.org/x/exp/errors v0.0.0-20220916125017-b168a2c6b86b // indirect
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect

test/config/config_dbs_migrate.json

@@ -7,6 +7,7 @@
     "lexicon_file": "./static/lexicon_writer.json",
     "api_parameters_file": "./static/parameters.json",
     "server_type": "DBSMigrate",
+    "concurrent_bulkblocks": true,
     "servercrt": "",
     "serverkey": "",
     "hkey": "",

test/config/config_dbs_migration.json

@@ -7,6 +7,7 @@
     "lexicon_file": "./static/lexicon_writer.json",
     "api_parameters_file": "./static/parameters.json",
     "server_type": "DBSMigration",
+    "concurrent_bulkblocks": true,
     "servercrt": "",
     "serverkey": "",
     "hkey": "",

test/data/migration/requests/README.md

@@ -0,0 +1,5 @@
+# Migration Requests
+* Place migration requests as JSON format in this directory.
+* Run `db_snapshot` to start the DBSMigrate and DBSMigration servers
+    * Currently commented are the scripts to create a snapshot of datasets into a local database.
+* Run `make test-migration-requests` afterwards to test migration requests in this directory.