Merge pull request #1188 from flanksource/test-commons

refactor: move common test logic to separate package

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/elastic/go-elasticsearch/v8 v8.1.0
 	github.com/fergusstrange/embedded-postgres v1.21.0
 	github.com/flanksource/commons v1.10.2
-	github.com/flanksource/duty v1.0.140
+	github.com/flanksource/duty v1.0.141
 	github.com/flanksource/gomplate/v3 v3.20.9
 	github.com/flanksource/is-healthy v0.0.0-20230713150444-ad2a5ef4bb37
 	github.com/flanksource/kommons v0.31.3
@@ -33,6 +33,7 @@ require (
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/go-sql-driver/mysql v1.7.0
 	github.com/gobwas/glob v0.2.3
+	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/henvic/httpretty v0.1.2
 	github.com/hirochachacha/go-smb2 v1.0.10
@@ -153,7 +154,6 @@ require (
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/cel-go v0.17.1 // indirect
 	github.com/google/gnostic v0.6.9 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
 	github.com/google/s2a-go v0.1.4 // indirect

go.sum

@@ -880,8 +880,8 @@ github.com/fergusstrange/embedded-postgres v1.21.0 h1:Sups0nR31+OB4iOZ0ZU4IwUDsB
 github.com/fergusstrange/embedded-postgres v1.21.0/go.mod h1:wL562t1V+iuFwq0UcgMi2e9rp8CROY9wxWZEfP8Y874=
 github.com/flanksource/commons v1.10.2 h1:Nw9foNBAt6QVylbgfaDojRGgRUAyQAHOfBv9qk9G714=
 github.com/flanksource/commons v1.10.2/go.mod h1:zYEhi6E2+diQ+loVcROUHo/Bgv+Tn61W2NYmrb5MgVI=
-github.com/flanksource/duty v1.0.140 h1:0+iqW+aaSF4CaYtUUx4lsKk5z/57A1hg3yUf4fWacG4=
-github.com/flanksource/duty v1.0.140/go.mod h1:RJ/kcZ7dbL8/52tem757szVIA3IomS8bOAZIK0xb4rk=
+github.com/flanksource/duty v1.0.141 h1:mUAcRkiacuEYzQyPLWan0qrCKzJGwqrHrG85uW+mVIg=
+github.com/flanksource/duty v1.0.141/go.mod h1:RJ/kcZ7dbL8/52tem757szVIA3IomS8bOAZIK0xb4rk=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.20.9 h1:I3H/l1FUDepe6IuG8Nj51QNX9ocdU2EGL4GWz31sZdk=
 github.com/flanksource/gomplate/v3 v3.20.9/go.mod h1:1N1aptaAo0XUaGsyU5CWiwn9GMRpbIKX1AdsypfmZYo=
@@ -1248,14 +1248,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
-github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
-github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
-github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
 github.com/henvic/httpretty v0.1.2 h1:EQo556sO0xeXAjP10eB+BZARMuvkdGqtfeS4Ntjvkiw=
 github.com/henvic/httpretty v0.1.2/go.mod h1:ViEsly7wgdugYtymX54pYp6Vv2wqZmNHayJ6q8tlKCc=
-github.com/hetznercloud/hcloud-go v1.33.1/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME=
-github.com/hetznercloud/hcloud-go v1.39.0/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA=
 github.com/hirochachacha/go-smb2 v1.0.10 h1:fiSNyMOOlWzfdTVk6VtvxfDGqhjNDI2iYZjd/jdtmhk=
 github.com/hirochachacha/go-smb2 v1.0.10/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=

hack/generate-schemas/go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/flanksource/duty v1.0.140 // indirect
+	github.com/flanksource/duty v1.0.141 // indirect
 	github.com/flanksource/is-healthy v0.0.0-20230713150444-ad2a5ef4bb37 // indirect
 	github.com/flanksource/mapstructure v1.6.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect

hack/generate-schemas/go.sum

@@ -705,8 +705,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fergusstrange/embedded-postgres v1.21.0 h1:Sups0nR31+OB4iOZ0ZU4IwUDsB+dVGmcqj4S2ko0qTI=
 github.com/flanksource/commons v1.10.2 h1:Nw9foNBAt6QVylbgfaDojRGgRUAyQAHOfBv9qk9G714=
 github.com/flanksource/commons v1.10.2/go.mod h1:zYEhi6E2+diQ+loVcROUHo/Bgv+Tn61W2NYmrb5MgVI=
-github.com/flanksource/duty v1.0.140 h1:0+iqW+aaSF4CaYtUUx4lsKk5z/57A1hg3yUf4fWacG4=
-github.com/flanksource/duty v1.0.140/go.mod h1:RJ/kcZ7dbL8/52tem757szVIA3IomS8bOAZIK0xb4rk=
+github.com/flanksource/duty v1.0.141 h1:mUAcRkiacuEYzQyPLWan0qrCKzJGwqrHrG85uW+mVIg=
+github.com/flanksource/duty v1.0.141/go.mod h1:RJ/kcZ7dbL8/52tem757szVIA3IomS8bOAZIK0xb4rk=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.20.9 h1:I3H/l1FUDepe6IuG8Nj51QNX9ocdU2EGL4GWz31sZdk=
 github.com/flanksource/gomplate/v3 v3.20.9/go.mod h1:1N1aptaAo0XUaGsyU5CWiwn9GMRpbIKX1AdsypfmZYo=

pkg/jobs/canary/suite_test.go

@@ -3,9 +3,7 @@ package canary
 import (
 	"context"
 	"fmt"
-	"io"
 	"net/http"
-	"os"
 	"strconv"
 	"testing"
 	"time"
@@ -15,11 +13,10 @@ import (
 	"github.com/flanksource/canary-checker/pkg/db"
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty"
-	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/flanksource/duty/testutils"
 	"github.com/labstack/echo/v4"
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"gorm.io/gorm"
 )
 
 var (
@@ -51,15 +48,19 @@ func DelayedResponseHandler(c echo.Context) error {
 }
 
 var _ = ginkgo.BeforeSuite(func() {
+	var err error
+
 	port := 9881
-	config := GetPGConfig("test", port)
+	config, dbString := testutils.GetEmbeddedPGConfig("test_canary_job", port)
 	postgresServer = embeddedPG.NewDatabase(config)
-	if err := postgresServer.Start(); err != nil {
+	if err = postgresServer.Start(); err != nil {
 		ginkgo.Fail(err.Error())
 	}
 	logger.Infof("Started postgres on port: %d", port)
 
-	db.Gorm, db.Pool = setupDB(fmt.Sprintf("postgres://postgres:postgres@localhost:%d/test?sslmode=disable", port))
+	if db.Gorm, db.Pool, err = duty.SetupDB(dbString, nil); err != nil {
+		ginkgo.Fail(err.Error())
+	}
 	cache.PostgresCache = cache.NewPostgresCache(db.Pool)
 
 	testEchoServer = echo.New()
@@ -89,53 +90,3 @@ var _ = ginkgo.AfterSuite(func() {
 		ginkgo.Fail(err.Error())
 	}
 })
-
-func setupDB(connectionString string) (*gorm.DB, *pgxpool.Pool) {
-	pgxpool, err := duty.NewPgxPool(connectionString)
-	if err != nil {
-		ginkgo.Fail(err.Error())
-	}
-
-	conn, err := pgxpool.Acquire(context.Background())
-	if err != nil {
-		ginkgo.Fail(err.Error())
-	}
-	defer conn.Release()
-
-	gormDB, err := duty.NewGorm(connectionString, duty.DefaultGormConfig())
-	if err != nil {
-		ginkgo.Fail(err.Error())
-	}
-
-	if err = duty.Migrate(connectionString, nil); err != nil {
-		ginkgo.Fail(err.Error())
-	}
-
-	return gormDB, pgxpool
-}
-
-func GetPGConfig(database string, port int) embeddedPG.Config {
-	// We are firing up multiple instances of the embedded postgres server at once when running tests in parallel.
-	//
-	// By default fergusstrange/embedded-postgres directly extracts the Postgres binary to a set location
-	// (/home/runner/.embedded-postgres-go/extracted/bin/initdb) and starts it.
-	// If two instances try to do this at the same time, they conflict, and throw the error
-	// "unable to extract postgres archive: open /home/runner/.embedded-postgres-go/extracted/bin/initdb: text file busy."
-	//
-	// This is a way to have separate instances of the running postgres servers.
-
-	var runTimePath string
-	homeDir, err := os.UserHomeDir()
-	if err != nil {
-		logger.Errorf("error getting user home dir: %v", err)
-		runTimePath = fmt.Sprintf("/tmp/.embedded-postgres-go/extracted-%d", port)
-	} else {
-		runTimePath = fmt.Sprintf("%s/.embedded-postgres-go/extracted-%d", homeDir, port)
-	}
-
-	return embeddedPG.DefaultConfig().
-		Database(database).
-		Port(uint32(port)).
-		RuntimePath(runTimePath).
-		Logger(io.Discard)
-}

pkg/topology/checks/suite_test.go

@@ -1,20 +1,15 @@
 package checks
 
 import (
-	"context"
-	"fmt"
-	"io"
-	"os"
 	"testing"
 
 	embeddedPG "github.com/fergusstrange/embedded-postgres"
 	"github.com/flanksource/canary-checker/pkg/db"
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty"
-	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/flanksource/duty/testutils"
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"gorm.io/gorm"
 )
 
 var (
@@ -27,15 +22,19 @@ func TestComponentCheckRun(t *testing.T) {
 }
 
 var _ = ginkgo.BeforeSuite(func() {
+	var err error
 	port := 9841
-	config := GetPGConfig("test_component_check", port)
+	config, dbString := testutils.GetEmbeddedPGConfig("test_component_check", port)
 	postgresServer = embeddedPG.NewDatabase(config)
 	if err := postgresServer.Start(); err != nil {
 		ginkgo.Fail(err.Error())
 	}
 	logger.Infof("Started postgres on port: %d", port)
 
-	db.Gorm, db.Pool = setupDB(fmt.Sprintf("postgres://postgres:postgres@localhost:%d/test_component_check?sslmode=disable", port))
+	if db.Gorm, db.Pool, err = duty.SetupDB(dbString, nil); err != nil {
+		ginkgo.Fail(err.Error())
+	}
+
 })
 
 var _ = ginkgo.AfterSuite(func() {
@@ -44,53 +43,3 @@ var _ = ginkgo.AfterSuite(func() {
 		ginkgo.Fail(err.Error())
 	}
 })
-
-func setupDB(connectionString string) (*gorm.DB, *pgxpool.Pool) {
-	pgxpool, err := duty.NewPgxPool(connectionString)
-	if err != nil {
-		ginkgo.Fail(err.Error())
-	}
-
-	conn, err := pgxpool.Acquire(context.Background())
-	if err != nil {
-		ginkgo.Fail(err.Error())
-	}
-	defer conn.Release()
-
-	gormDB, err := duty.NewGorm(connectionString, duty.DefaultGormConfig())
-	if err != nil {
-		ginkgo.Fail(err.Error())
-	}
-
-	if err = duty.Migrate(connectionString, nil); err != nil {
-		ginkgo.Fail(err.Error())
-	}
-
-	return gormDB, pgxpool
-}
-
-func GetPGConfig(database string, port int) embeddedPG.Config {
-	// We are firing up multiple instances of the embedded postgres server at once when running tests in parallel.
-	//
-	// By default fergusstrange/embedded-postgres directly extracts the Postgres binary to a set location
-	// (/home/runner/.embedded-postgres-go/extracted/bin/initdb) and starts it.
-	// If two instances try to do this at the same time, they conflict, and throw the error
-	// "unable to extract postgres archive: open /home/runner/.embedded-postgres-go/extracted/bin/initdb: text file busy."
-	//
-	// This is a way to have separate instances of the running postgres servers.
-
-	var runTimePath string
-	homeDir, err := os.UserHomeDir()
-	if err != nil {
-		logger.Errorf("error getting user home dir: %v", err)
-		runTimePath = fmt.Sprintf("/tmp/.embedded-postgres-go/extracted-%d", port)
-	} else {
-		runTimePath = fmt.Sprintf("%s/.embedded-postgres-go/extracted-%d", homeDir, port)
-	}
-
-	return embeddedPG.DefaultConfig().
-		Database(database).
-		Port(uint32(port)).
-		RuntimePath(runTimePath).
-		Logger(io.Discard)
-}

pkg/topology/suite_test.go

@@ -1,20 +1,15 @@
 package topology
 
 import (
-	"context"
-	"fmt"
-	"io"
-	"os"
 	"testing"
 
 	embeddedPG "github.com/fergusstrange/embedded-postgres"
 	"github.com/flanksource/canary-checker/pkg/db"
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty"
-	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/flanksource/duty/testutils"
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"gorm.io/gorm"
 )
 
 var (
@@ -27,15 +22,19 @@ func TestTopologySync(t *testing.T) {
 }
 
 var _ = ginkgo.BeforeSuite(func() {
+	var err error
+
 	port := 9842
-	config := GetPGConfig("test_topology", port)
+	config, dbString := testutils.GetEmbeddedPGConfig("test_topology", port)
 	postgresServer = embeddedPG.NewDatabase(config)
-	if err := postgresServer.Start(); err != nil {
+	if err = postgresServer.Start(); err != nil {
 		ginkgo.Fail(err.Error())
 	}
 	logger.Infof("Started postgres on port: %d", port)
 
-	db.Gorm, db.Pool = setupDB(fmt.Sprintf("postgres://postgres:postgres@localhost:%d/test_topology?sslmode=disable", port))
+	if db.Gorm, db.Pool, err = duty.SetupDB(dbString, nil); err != nil {
+		ginkgo.Fail(err.Error())
+	}
 })
 
 var _ = ginkgo.AfterSuite(func() {
@@ -44,53 +43,3 @@ var _ = ginkgo.AfterSuite(func() {
 		ginkgo.Fail(err.Error())
 	}
 })
-
-func setupDB(connectionString string) (*gorm.DB, *pgxpool.Pool) {
-	pgxpool, err := duty.NewPgxPool(connectionString)
-	if err != nil {
-		ginkgo.Fail(err.Error())
-	}
-
-	conn, err := pgxpool.Acquire(context.Background())
-	if err != nil {
-		ginkgo.Fail(err.Error())
-	}
-	defer conn.Release()
-
-	gormDB, err := duty.NewGorm(connectionString, duty.DefaultGormConfig())
-	if err != nil {
-		ginkgo.Fail(err.Error())
-	}
-
-	if err = duty.Migrate(connectionString, nil); err != nil {
-		ginkgo.Fail(err.Error())
-	}
-
-	return gormDB, pgxpool
-}
-
-func GetPGConfig(database string, port int) embeddedPG.Config {
-	// We are firing up multiple instances of the embedded postgres server at once when running tests in parallel.
-	//
-	// By default fergusstrange/embedded-postgres directly extracts the Postgres binary to a set location
-	// (/home/runner/.embedded-postgres-go/extracted/bin/initdb) and starts it.
-	// If two instances try to do this at the same time, they conflict, and throw the error
-	// "unable to extract postgres archive: open /home/runner/.embedded-postgres-go/extracted/bin/initdb: text file busy."
-	//
-	// This is a way to have separate instances of the running postgres servers.
-
-	var runTimePath string
-	homeDir, err := os.UserHomeDir()
-	if err != nil {
-		logger.Errorf("error getting user home dir: %v", err)
-		runTimePath = fmt.Sprintf("/tmp/.embedded-postgres-go/extracted-%d", port)
-	} else {
-		runTimePath = fmt.Sprintf("%s/.embedded-postgres-go/extracted-%d", homeDir, port)
-	}
-
-	return embeddedPG.DefaultConfig().
-		Database(database).
-		Port(uint32(port)).
-		RuntimePath(runTimePath).
-		Logger(io.Discard)
-}