Skip to content
This repository has been archived by the owner on Aug 26, 2022. It is now read-only.

Speedup local tests #171

Closed
wants to merge 14 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 150 additions & 0 deletions cmd/db/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
package main

import (
"database/sql"
"flag"
"fmt"
"os"
"strings"

"github.com/go-kit/kit/log"
"github.com/kelseyhightower/envconfig"
"github.com/moov-io/customers/internal/database"
)

var flagLogFormat = flag.String("log.format", "", "Format for log lines (Options: json, plain")

type Config struct {
RootPassword string `split_words:"true" default:"secret"`
User string `default:"moov"`
Password string `default:"secret"`
Address string `default:"tcp(localhost:3306)"`
Database string `default:"paygate_test"`
}

func main() {
var config Config
err := envconfig.Process("mysql", &config)
if err != nil {
panic(err)
}

err = runCmd(os.Args[1], &config)
if err != nil {
panic(err)
}
}

func runCmd(cmd string, config *Config) error {
switch cmd {
case "setup":
err := dropDB(config)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We cannot drop the database. This needs to never be an option in production in fact we will not grant a service DROP.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For tests this may be useful. For example if you want to test some specific migrations, etc. We can add -force flag for drop and make it work only if database has _test.

if err != nil {
return err
}

err = createDB(config)
if err != nil {
return err
}

err = migrateDB(config)
if err != nil {
return err
}
case "create":
err := createDB(config)
if err != nil {
return err
}
case "drop":
err := dropDB(config)
if err != nil {
return err
}
case "migrate":
err := migrateDB(config)
if err != nil {
return err
}
}

return nil
}

func dropDB(config *Config) error {
dsn := fmt.Sprintf("root:%s@%s/", config.RootPassword, config.Address)
db, err := sql.Open("mysql", dsn)
if err != nil {
return err
}
defer db.Close()

_, err = db.Exec("DROP DATABASE IF EXISTS " + config.Database)
if err != nil {
return err
}

fmt.Printf("Database %s was deleted\n", config.Database)
return nil
}

func createDB(config *Config) error {
dsn := fmt.Sprintf("root:%s@%s/", config.RootPassword, config.Address)
db, err := sql.Open("mysql", dsn)
if err != nil {
return err
}
defer db.Close()

_, err = db.Exec("CREATE DATABASE " + config.Database)
if err != nil {
return err
}

_, err = db.Exec(fmt.Sprintf("CREATE USER IF NOT EXISTS '%s'@'%%' IDENTIFIED BY '%s'", config.User, config.Password))
if err != nil {
return err
}

_, err = db.Exec(fmt.Sprintf("GRANT ALL PRIVILEGES ON %s . * TO '%s'@'%%';", config.Database, config.User))
if err != nil {
return err
}

fmt.Printf("Database %s was created\n", config.Database)
return nil
}

// TODO: having migration inside database.New (inside Connect method) makes it
// ambiguous we should extract migtation method into separate public method
// that we can call from here.
func migrateDB(config *Config) error {
var logger log.Logger

// migrate database
if strings.ToLower(*flagLogFormat) == "json" {
logger = log.NewJSONLogger(os.Stderr)
} else {
logger = log.NewLogfmtLogger(os.Stderr)
}
logger = log.With(logger, "ts", log.DefaultTimestampUTC)
logger = log.With(logger, "caller", log.DefaultCaller)

dbConf := &database.MySQLConfig{
User: config.User,
Password: config.Password,
Address: config.Address,
Database: config.Database,
}

db, err := database.NewMySQL(logger, dbConf)
if err != nil {
return err
}
db.Close()

fmt.Printf("Database %s was migrated\n", config.Database)

return nil
}
12 changes: 12 additions & 0 deletions cmd/server/paygate/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ type deployment struct {
}

func (d *deployment) close(t *testing.T) {
if d.res == nil {
return
}

if err := d.res.Close(); err != nil {
t.Error(err)
}
Expand All @@ -41,6 +45,12 @@ func spawnPayGate(t *testing.T) *deployment {
if testing.Short() {
t.Skip("-short flag enabled")
}

if os.Getenv("PAYGATE_ENDPOINT") != "" {
client := NewClient(log.NewNopLogger(), os.Getenv("PAYGATE_ENDPOINT"), false)
return &deployment{client: client}
}

if !docker.Enabled() {
t.Skip("Docker not enabled")
}
Expand Down Expand Up @@ -72,12 +82,14 @@ func spawnPayGate(t *testing.T) *deployment {
}

client := NewClient(log.NewNopLogger(), fmt.Sprintf("http://localhost:%s", resource.GetPort("8080/tcp")), false)

err = pool.Retry(func() error {
return client.Ping()
})
if err != nil {
t.Fatal(err)
}

return &deployment{resource, client}
}

Expand Down
10 changes: 10 additions & 0 deletions cmd/server/watchman_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"math"
"os"
"strings"
"testing"

Expand Down Expand Up @@ -44,6 +45,9 @@ type watchmanDeployment struct {
}

func (d *watchmanDeployment) close(t *testing.T) {
if d.res == nil {
return
}
if err := d.res.Close(); err != nil {
t.Error(err)
}
Expand All @@ -55,6 +59,12 @@ func spawnWatchman(t *testing.T) *watchmanDeployment {
if testing.Short() {
t.Skip("-short flag enabled")
}

if os.Getenv("WATCHMAN_ENDPOINT") != "" {
client := newWatchmanClient(log.NewNopLogger(), os.Getenv("WATCHMAN_ENDPOINT"), false)
return &watchmanDeployment{client: client}
}

if !docker.Enabled() {
t.Skip("Docker not enabled")
}
Expand Down
28 changes: 28 additions & 0 deletions docker-compose.dev.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
version: '3'
services:
fed:
image: moov/fed:v0.5.1
ports:
- "8086:8086"
- "9096:9096"
environment:
FEDACH_DATA_PATH: './data/fed/FedACHdir.txt'
FEDWIRE_DATA_PATH: './data/fed/fpddir.txt'
watchman:
image: moov/watchman:v0.15.0
ports:
- "8084:8084"
- "9094:9094"
paygate:
image: moov/paygate:v0.8.0-dev
command: -config=/conf/config.yaml
volumes:
- "./testdata/paygate/config.yml:/conf/config.yaml"
ports:
- "8082:8082"
db:
image: mysql:8
environment:
MYSQL_ROOT_PASSWORD: secret
ports:
- "3306:3306"
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.14

require (
cloud.google.com/go v0.64.0 // indirect
github.com/DATA-DOG/go-txdb v0.1.3
github.com/PuerkitoBio/goquery v1.5.1 // indirect
github.com/antihax/optional v1.0.0
github.com/aws/aws-sdk-go v1.34.9
Expand All @@ -19,6 +20,7 @@ require (
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.4
github.com/hashicorp/vault/api v1.0.4
github.com/kelseyhightower/envconfig v1.4.0
github.com/lopezator/migrator v0.3.0
github.com/mattn/go-sqlite3 v2.0.3+incompatible
github.com/mitchellh/mapstructure v1.3.3
Expand All @@ -36,6 +38,7 @@ require (
github.com/plaid/plaid-go v0.0.0-20200805000941-b566963d2892
github.com/prometheus/client_golang v1.7.1
github.com/prometheus/common v0.13.0 // indirect
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.6.1
gocloud.dev v0.20.0
gocloud.dev/secrets/hashivault v0.20.0
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/ClickHouse/clickhouse-go v1.3.12/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/DATA-DOG/go-txdb v0.1.3 h1:R4v6OuOcy2O147e2zHxU0B4NDtF+INb5R9q/CV7AEMg=
github.com/DATA-DOG/go-txdb v0.1.3/go.mod h1:DhAhxMXZpUJVGnT+p9IbzJoRKvlArO2pkHjnGX7o0n0=
github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190418212003-6ac0b49e7197/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo=
github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
Expand Down Expand Up @@ -581,6 +583,8 @@ github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
Expand Down
50 changes: 49 additions & 1 deletion internal/database/mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ import (
"fmt"
"os"
"strings"
"sync"
"testing"
"time"

"github.com/DATA-DOG/go-txdb"
"github.com/moov-io/base/docker"

"github.com/go-kit/kit/log"
Expand Down Expand Up @@ -176,6 +178,17 @@ func (my *mysql) Connect() (*sql.DB, error) {
return db, nil
}

type MySQLConfig struct {
User string
Password string
Address string
Database string
}

func NewMySQL(logger log.Logger, config *MySQLConfig) (*sql.DB, error) {
return mysqlConnection(logger, config.User, config.Password, config.Address, config.Database).Connect()
}

func mysqlConnection(logger log.Logger, user, pass string, address string, database string) *mysql {
timeout := "30s"
if v := os.Getenv("MYSQL_TIMEOUT"); v != "" {
Expand All @@ -199,7 +212,10 @@ type TestMySQLDB struct {
}

func (r *TestMySQLDB) Close() error {
r.container.Close()
if r.container != nil {
r.container.Close()
}

return r.DB.Close()
}

Expand All @@ -211,6 +227,18 @@ func CreateTestMySQLDB(t *testing.T) *TestMySQLDB {
if testing.Short() {
t.Skip("-short flag enabled")
}

if os.Getenv("MYSQL_TEST") != "" {
params := "timeout=30s&charset=utf8mb4&parseTime=true&sql_mode=ALLOW_INVALID_DATES"
dsn := fmt.Sprintf("%s:%s@%s/%s?%s", "moov", "secret", "tcp(localhost:3306)", "paygate_test", params)
db, err := txDbConnect(dsn)
if err != nil {
t.Fatal(err)
}

return &TestMySQLDB{DB: db}
}

if !docker.Enabled() {
t.Skip("Docker not enabled")
}
Expand Down Expand Up @@ -268,3 +296,23 @@ func MySQLUniqueViolation(err error) bool {
}
return match
}

var initTestDbOnce sync.Once

func txDbConnect(dsn string) (*sql.DB, error) {
initTestDbOnce.Do(func() {
txdb.Register("txdb", "mysql", dsn)
})

db, err := sql.Open("txdb", "identifier")
if err != nil {
return nil, err
}

// Check out DB is up and working
if err := db.Ping(); err != nil {
return nil, err
}

return db, nil
}
6 changes: 6 additions & 0 deletions internal/database/mysql_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ package database

import (
"errors"
"fmt"
"testing"

"github.com/go-kit/kit/log"
"github.com/stretchr/testify/require"
)

func TestMySQL__basic(t *testing.T) {
Expand All @@ -31,6 +33,10 @@ func TestMySQL__basic(t *testing.T) {
if conn != nil || err == nil {
t.Fatalf("conn=%#v expected error", conn)
}

fmt.Println("Hello")
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove this :)

_, err = txDbConnect("user:pass@tcp(localhost:1234)/db?timeout=1s")
require.Error(t, err)
}

func TestMySQLUniqueViolation(t *testing.T) {
Expand Down
Loading