Skip to content

Commit

Permalink
Merge pull request #75 from d-ylee/migrationRequestTests
Browse files Browse the repository at this point in the history
Migration Request Integration Tests Concept
  • Loading branch information
vkuznet authored Sep 23, 2022
2 parents b077e93 + 5c99359 commit 2aad8db
Show file tree
Hide file tree
Showing 14 changed files with 1,402 additions and 41 deletions.
82 changes: 79 additions & 3 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ on:
branches: [ master ]

jobs:

build:
runs-on: ubuntu-latest
steps:
Expand All @@ -18,7 +17,15 @@ jobs:
with:
go-version: 1.18

- name: Build and test
- name: Build
env:
GOPATH: /home/runner/go
run: |
sed -i -e "s,_ \"github.com/mattn/go-oci8\",,g" web/server.go
sed -i -e "s,_ \"gopkg.in/rana/ora.v4\",,g" web/server.go
make
- name: Prepare code for tests and test DBS
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
Expand All @@ -32,4 +39,73 @@ jobs:
sed -i -e "s,_ \"github.com/mattn/go-oci8\",,g" test/merge/main.go
mkdir -p $GOPATH/src/github.com/vkuznet
cp -r ../dbs2go $GOPATH/src/github.com/vkuznet
make test-github
- name: Test SQL
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
run: |
make test-sql
- name: Test validator
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
run: |
make test-validator
- name: Test bulk
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
run: |
make test-bulk
- name: Test http
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
run: |
make test-http
- name: Test utils
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
run: |
make test-utils
- name: Test writer
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
run: |
make test-writer
- name: Test integration
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
run: |
make test-integration
- name: Test migration requests
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
run: |
make test-migration-requests
- name: Test migration
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
run: |
make test-migration
- name: Benchmark
env:
GOPATH: /home/runner/go
PKG_CONFIG_PATH: /home/runner/go
run: |
make bench
8 changes: 6 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,13 @@ migrate.json
migration.json
test/data/dbsdata
.env
.vscode
*.tmp
test-pids

# custom development files
.devcontainer
.vscode
.vscode

.globus
instantclient*
.DS_Store
26 changes: 10 additions & 16 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ clean:

test: test-dbs test-sql test-errors test-validator test-bulk test-http test-utils test-migrate test-writer test-integration test-lexicon bench

test-github: test-dbs test-sql test-errors test-validator test-bulk test-http test-utils test-writer test-lexicon test-integration test-migration bench
test-github: test-dbs test-sql test-errors test-validator test-bulk test-http test-utils test-writer test-lexicon test-integration test-migration-requests test-migration bench

test-lexicon: test-lexicon-writer-pos test-lexicon-writer-neg test-lexicon-reader-pos test-lexicon-reader-neg

Expand Down Expand Up @@ -157,25 +157,19 @@ test-integration:
FILE_LUMI_LIST_LENGTH=30 \
go test -v -failfast -run Integration
test-migration:
cd test && rm -f /tmp/dbs-one.db && \
sqlite3 /tmp/dbs-one.db < ../static/schema/sqlite-schema.sql && \
echo sqlite3 /tmp/dbs-one.db sqlite > ./dbfile_1 && \
rm -f /tmp/dbs-two.db && \
sqlite3 /tmp/dbs-two.db < ../static/schema/sqlite-schema.sql && \
echo sqlite3 /tmp/dbs-two.db sqlite > ./dbfile_2 && \
cd .. && \
LD_LIBRARY_PATH=${odir} DYLD_LIBRARY_PATH=${odir} \
./bin/start_test_migration && \
cd test && \
LD_LIBRARY_PATH=${odir} DYLD_LIBRARY_PATH=${odir} \
DBS_API_PARAMETERS_FILE=../static/parameters.json \
DBS_READER_LEXICON_FILE=../static/lexicon_reader.json \
DBS_WRITER_LEXICON_FILE=../static/lexicon_writer.json \
DBS_DB_FILE_1=./dbfile_1 \
DBS_DB_FILE_2=./dbfile_2 \
DBS_DB_PATH_1=/tmp/dbs-one.db \
DBS_DB_PATH_2=/tmp/dbs-two.db \
BULKBLOCKS_DATA_FILE=./data/migration/bulkblocks_data.json \
go test -v -failfast -run IntMigration
go test -v -failfast -timeout 10m -run IntMigration
test-migration-requests:
LD_LIBRARY_PATH=${odir} DYLD_LIBRARY_PATH=${odir} \
./bin/start_test_migration && \
cd test && \
LD_LIBRARY_PATH=${odir} DYLD_LIBRARY_PATH=${odir} \
MIGRATION_REQUESTS_PATH=./data/migration/requests \
go test -v -failfast -timeout 10m -run MigrationRequests
bench:
cd test && rm -f /tmp/dbs-test.db && \
sqlite3 /tmp/dbs-test.db < ../static/schema/sqlite-schema.sql && \
Expand Down
39 changes: 39 additions & 0 deletions bin/db_snapshot
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#!/bin/bash

# Starts the DBSMigrate and DBSMigration servers for migration request tests
# TODO: Creates a dbs database snapshot depending on migration requests files
# X509_USER_CERT and X509_USER_KEY need to be defined in order to grab data from the production server

# Stop currently running dbs2go processes
pkill dbs2go

# Reset the sqlite database for the snapshot
cd test && rm -f /tmp/dbs-test.db && \
sqlite3 /tmp/dbs-test.db < ../static/schema/sqlite-schema.sql && \

cd ..

# Start dbs-migrate
X509_USER_CERT=$X509_USER_CERT X509_USER_KEY=$X509_USER_KEY \
./dbs2go -config ./migrate.json &
echo $! >> ./test-pids
sleep 1

# Start dbs-migration
X509_USER_CERT=$X509_USER_CERT X509_USER_KEY=$X509_USER_KEY \
./dbs2go -config ./migration.json &
echo $! >> ./test-pids

sleep 1

# Read and process migration requests from directory
REQUESTS=$PWD/test/data/migration/requests/*.json
for f in $REQUESTS
do
echo "Processing $f file..."
X509_USER_CERT=$X509_USER_CERT X509_USER_KEY=$X509_USER_KEY \
curl -H "Content-Type: application/json" \
http://localhost:9898/dbs2go/submit \
-d@$f
done
#curl --cacert "$CA_CERT" --cert $X509_USER_CERT --key $X509_USER_KEY --header 'Content-Type: application/json' --verbose https://cmsweb.cern.ch/dbs/prod/global/DBSReader/blockdump?block_name=/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM
30 changes: 15 additions & 15 deletions bin/start_test_migration
Original file line number Diff line number Diff line change
@@ -1,45 +1,45 @@
#!/bin/bash

echo "building dbs2go"
make build

# create pid file
rm ./test-pids
touch ./test-pids

pwd

# Start dbs-one-reader
# Delete original dbfiles and databases and reinitializing them with the schema
rm -f /tmp/dbs-one.db
sqlite3 /tmp/dbs-one.db < ./static/schema/sqlite-schema.sql
echo sqlite3 /tmp/dbs-one.db sqlite > ./test/dbfile_1
rm -f /tmp/dbs-two.db
sqlite3 /tmp/dbs-two.db < ./static/schema/sqlite-schema.sql
echo sqlite3 /tmp/dbs-two.db sqlite > ./test/dbfile_2

# Starting DBS servers
echo Starting dbs-one-reader
./dbs2go -config ./test/config/config_dbs_one_reader.json &
echo $! >> ./test-pids

sleep 1

# Start dbs-one-writer
echo Starting dbs-one-writer
./dbs2go -config ./test/config/config_dbs_one_writer.json &
echo $! >> ./test-pids

sleep 1

# Start dbs-two-reader
echo Starting dbs-two-reader
./dbs2go -config ./test/config/config_dbs_two_reader.json &
echo $! >> ./test-pids

sleep 1

# Start dbs-two-writer
echo Starting dbs-two-writer
./dbs2go -config ./test/config/config_dbs_two_writer.json &
echo $! >> ./test-pids

sleep 1

# Start dbs-migrate
echo Starting dbs-migrate
./dbs2go -config ./test/config/config_dbs_migrate.json &
echo $! >> ./test-pids

sleep 1

# Start dbs-migration
echo Starting dbs-migration
./dbs2go -config ./test/config/config_dbs_migration.json &
echo $! >> ./test-pids

Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
github.com/vkuznet/auth-proxy-server/logging v0.0.0-20220406163751-c36feb20c750
github.com/vkuznet/limiter v2.2.2+incompatible
github.com/vkuznet/x509proxy v0.0.0-20210801171832-e47b94db99b6
golang.org/x/exp v0.0.0-20220428152302-39d4317da171
gopkg.in/rana/ora.v4 v4.1.15
gopkg.in/yaml.v2 v2.4.0
)
Expand All @@ -38,7 +39,6 @@ require (
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
golang.org/x/exp v0.0.0-20220428152302-39d4317da171 // indirect
golang.org/x/exp/errors v0.0.0-20220916125017-b168a2c6b86b // indirect
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
Expand Down
1 change: 1 addition & 0 deletions test/config/config_dbs_migrate.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
"lexicon_file": "./static/lexicon_writer.json",
"api_parameters_file": "./static/parameters.json",
"server_type": "DBSMigrate",
"concurrent_bulkblocks": true,
"servercrt": "",
"serverkey": "",
"hkey": "",
Expand Down
1 change: 1 addition & 0 deletions test/config/config_dbs_migration.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
"lexicon_file": "./static/lexicon_writer.json",
"api_parameters_file": "./static/parameters.json",
"server_type": "DBSMigration",
"concurrent_bulkblocks": true,
"servercrt": "",
"serverkey": "",
"hkey": "",
Expand Down
5 changes: 5 additions & 0 deletions test/data/migration/requests/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Migration Requests
* Place migration requests as JSON format in this directory.
* Run `db_snapshot` to start the DBSMigrate and DBSMigration servers
* Currently commented are the scripts to create a snapshot of datasets into a local database.
* Run `make test-migration-requests` afterwards to test migration requests in this directory.
Loading

0 comments on commit 2aad8db

Please sign in to comment.