Merge pull request #85 from d-ylee/testRaceConditions

Test race conditions in bulkblocks API

Author: vkuznet

Makefile

@@ -178,3 +178,11 @@ bench:
 	DBS_API_PARAMETERS_FILE=../static/parameters.json \
 	DBS_LEXICON_FILE=../static/lexicon_writer.json \
 	go test -run Benchmark -bench=.
+test-race:
+	LD_LIBRARY_PATH=${odir} DYLD_LIBRARY_PATH=${odir} \
+	./bin/start_write_servers && \
+	cd test && \
+	LD_LIBRARY_PATH=${odir} DYLD_LIBRARY_PATH=${odir} \
+	MIGRATION_REQUESTS_PATH=./data/migration/requests \
+	INTEGRATION_DATA_FILE=./data/integration/integration_data.json \
+	go test -v -failfast -timeout 10m -run RaceConditions

bin/start_write_servers

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Delete original dbfiles and databases and reinitializing them with the schema
+rm -f /tmp/dbs-one.db
+sqlite3 /tmp/dbs-one.db < ./static/schema/sqlite-schema.sql
+echo sqlite3 /tmp/dbs-one.db sqlite > ./test/dbfile_1
+
+# Starting DBS servers
+echo Starting dbs-one-writer
+./dbs2go -config ./test/config/config_dbs_one_writer2.json &
+echo $! >> ./test-pids
+
+sleep 1

dbs/fileparents.go

@@ -17,7 +17,7 @@ func (a *API) FileParents() error {
 
 	if len(a.Params) == 0 {
 		msg := "logical_file_name, block_id or block_name is required for fileparents api"
-		return Error(InvalidParamErr, ParametersErrorCode, msg, "dbs.fileparents.FielParents")
+		return Error(InvalidParamErr, ParametersErrorCode, msg, "dbs.fileparents.FileParents")
 	}
 
 	tmpl := make(Record)
@@ -31,7 +31,7 @@ func (a *API) FileParents() error {
 
 	stm, err := LoadTemplateSQL("fileparent", tmpl)
 	if err != nil {
-		return Error(err, LoadErrorCode, "", "dbs.fileparents.FielParents")
+		return Error(err, LoadErrorCode, "", "dbs.fileparents.FileParents")
 	}
 
 	lfns := getValues(a.Params, "logical_file_name")
@@ -52,7 +52,7 @@ func (a *API) FileParents() error {
 	// use generic query API to fetch the results from DB
 	err = executeAll(a.Writer, a.Separator, stm, args...)
 	if err != nil {
-		return Error(err, QueryErrorCode, "", "dbs.fileparents.FielParents")
+		return Error(err, QueryErrorCode, "", "dbs.fileparents.FileParents")
 	}
 	return nil
 }

test/config/config_dbs_one_writer2.json

@@ -0,0 +1,17 @@
+{
+    "base": "/dbs-one-writer",
+    "port": 8990,
+    "staticdir": "./static",
+    "dbfile": "./test/dbfile_1",
+    "lexicon_file": "./static/lexicon_writer.json",
+    "api_parameters_file": "./static/parameters.json",
+    "server_type": "DBSWriter",
+    "servercrt": "",
+    "serverkey": "",
+    "hkey": "",
+    "updateDNs": 60,
+    "log_file": "/tmp/dbs2go_writer_1_2.log",
+    "verbose": 2,
+    "limiter_rate": "10000-s",
+    "concurrent_bulkblocks": true
+}

test/int_filearray.go

@@ -1553,7 +1553,7 @@ func getFileArrayTestTable(t *testing.T) []EndpointTestCase {
 					output:   []Response{errorResp4},
 					respCode: http.StatusBadRequest,
 				},
-				// TODO: Figure out logicl for DBSClientReader_t.test.034l
+				// TODO: Figure out logic for DBSClientReader_t.test.034l
 			},
 		},
 	}

test/int_fileparents.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/dmwm/dbs2go/dbs"
+	"github.com/dmwm/dbs2go/web"
+)
+
+func getFileParentsTestTable(t *testing.T) EndpointTestCase {
+	dbsError := dbs.DBSError{
+		Reason:   dbs.InvalidParamErr.Error(),
+		Message:  "logical_file_name, block_id or block_name is required for fileparents api",
+		Code:     dbs.ParametersErrorCode,
+		Function: "dbs.fileparents.FileParents",
+	}
+	hrec := createHTTPError("GET", "/dbs/primarydstypes?dataset=fnal")
+	errorResp := web.ServerError{
+		HTTPError: hrec,
+		DBSError:  &dbsError,
+		Exception: http.StatusBadRequest,
+		Type:      "HTTPError",
+		Message:   dbsError.Error(),
+	}
+	return EndpointTestCase{
+		description:     "Test fileparents",
+		defaultHandler:  web.FileParentsHandler,
+		defaultEndpoint: "/dbs/fileparents",
+		testCases: []testCase{
+			{ // DBSClientReader_t.test041
+				description: "Test fileparents with lfn",
+				method:      "GET",
+				serverType:  "DBSReader",
+				params: url.Values{
+					"logical_file_name": []string{TestData.ParentFiles[0]},
+				},
+				output:   []Response{},
+				respCode: http.StatusOK,
+			},
+			{ // DBSClientReader_t.test042
+				description: "Test fileparents with no params",
+				method:      "GET",
+				serverType:  "DBSReader",
+				params:      url.Values{},
+				output:      []Response{errorResp},
+				respCode:    http.StatusBadRequest,
+			},
+		},
+	}
+}

test/integration_cases.go

@@ -567,6 +567,7 @@ func LoadTestCases(t *testing.T, filepath string, bulkblockspath string, largeBu
 	bulkBlocksTest := getBulkBlocksTestTable(t)
 	filesReaderTestTable := getFilesLumiListRangeTestTable(t)
 	fileArrayTestTable := getFileArrayTestTable(t)
+	fileParentsTestTable := getFileParentsTestTable(t)
 	largeFileLumiInsertTestTable := getBulkBlocksLargeFileLumiInsertTestTable(t)
 	filesReaderAfterChunkTestTable := getFileLumiChunkTestTable(t)
 
@@ -592,8 +593,15 @@ func LoadTestCases(t *testing.T, filepath string, bulkblockspath string, largeBu
 	)
 	endpointTestCases = append(endpointTestCases, filesReaderTestTable...)
 	endpointTestCases = append(endpointTestCases, fileArrayTestTable...)
-	endpointTestCases = append(endpointTestCases, largeFileLumiInsertTestTable)
-	endpointTestCases = append(endpointTestCases, filesReaderAfterChunkTestTable)
+
+	endpointTestCases = append(
+		endpointTestCases,
+		fileParentsTestTable,
+		largeFileLumiInsertTestTable,
+		filesReaderAfterChunkTestTable,
+	)
+	// endpointTestCases = append(endpointTestCases, largeFileLumiInsertTestTable)
+	// endpointTestCases = append(endpointTestCases, filesReaderAfterChunkTestTable)
 
 	return endpointTestCases
 }

test/migration_test.go

@@ -677,7 +677,7 @@ func TestMigrationRequests(t *testing.T) {
 
 	bulkblocksPath := os.Getenv("MIGRATION_REQUESTS_PATH")
 	if bulkblocksPath == "" {
-		log.Fatal("MIGRATION_REQUESTS_PATH not defined")
+		t.Fatal("MIGRATION_REQUESTS_PATH not defined")
 	}
 
 	// load grandparent data