diff --git a/backend/internal/neosyncdb/integration_test.go b/backend/internal/neosyncdb/integration_test.go index 8d6aea0fda..1efbef5f62 100644 --- a/backend/internal/neosyncdb/integration_test.go +++ b/backend/internal/neosyncdb/integration_test.go @@ -2,25 +2,21 @@ package neosyncdb import ( "context" - "fmt" "io" "log/slog" - "os" "testing" "time" "github.com/jackc/pgx/v5/pgtype" - "github.com/jackc/pgx/v5/pgxpool" db_queries "github.com/nucleuscloud/neosync/backend/gen/go/db" mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" nucleuserrors "github.com/nucleuscloud/neosync/backend/internal/errors" pg_models "github.com/nucleuscloud/neosync/backend/sql/postgresql/models" neomigrate "github.com/nucleuscloud/neosync/internal/migrate" + "github.com/nucleuscloud/neosync/internal/testutil" + tcpostgres "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/postgres" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/testcontainers/testcontainers-go" - testpg "github.com/testcontainers/testcontainers-go/modules/postgres" - "github.com/testcontainers/testcontainers-go/wait" "golang.org/x/sync/errgroup" ) @@ -31,12 +27,9 @@ var ( type IntegrationTestSuite struct { suite.Suite - pgpool *pgxpool.Pool - ctx context.Context - pgcontainer *testpg.PostgresContainer - connstr string + pgcontainer *tcpostgres.PostgresTestContainer migrationsDir string db *NeosyncDb @@ -45,38 +38,21 @@ type IntegrationTestSuite struct { func (s *IntegrationTestSuite) SetupSuite() { s.ctx = context.Background() - pgcontainer, err := testpg.Run( - s.ctx, - "postgres:15", - testcontainers.WithWaitStrategy( - wait.ForLog("database system is ready to accept connections"). - WithOccurrence(2).WithStartupTimeout(5*time.Second), - ), - ) + pgcontainer, err := tcpostgres.NewPostgresTestContainer(s.ctx) if err != nil { panic(err) } s.pgcontainer = pgcontainer - connstr, err := pgcontainer.ConnectionString(s.ctx, "sslmode=disable") - if err != nil { - panic(err) - } - s.connstr = connstr - pool, err := pgxpool.New(s.ctx, connstr) - if err != nil { - panic(err) - } - s.pgpool = pool s.migrationsDir = "../../sql/postgresql/schema" - s.db = New(pool, db_queries.New()) + s.db = New(s.pgcontainer.DB, db_queries.New()) } // Runs before each test func (s *IntegrationTestSuite) SetupTest() { discardLogger := slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})) - err := neomigrate.Up(s.ctx, s.connstr, s.migrationsDir, discardLogger) + err := neomigrate.Up(s.ctx, s.pgcontainer.URL, s.migrationsDir, discardLogger) if err != nil { panic(err) } @@ -86,22 +62,19 @@ func (s *IntegrationTestSuite) TearDownTest() { // Dropping here because 1) more efficient and 2) we have a bad down migration // _jobs-connection-id-null.down that breaks due to having a null connection_id column. // we should do something about that at some point. Running this single drop is easier though - _, err := s.pgpool.Exec(s.ctx, "DROP SCHEMA IF EXISTS neosync_api CASCADE") + _, err := s.pgcontainer.DB.Exec(s.ctx, "DROP SCHEMA IF EXISTS neosync_api CASCADE") if err != nil { panic(err) } - _, err = s.pgpool.Exec(s.ctx, "DROP TABLE IF EXISTS public.schema_migrations") + _, err = s.pgcontainer.DB.Exec(s.ctx, "DROP TABLE IF EXISTS public.schema_migrations") if err != nil { panic(err) } } func (s *IntegrationTestSuite) TearDownSuite() { - if s.pgpool != nil { - s.pgpool.Close() - } if s.pgcontainer != nil { - err := s.pgcontainer.Terminate(s.ctx) + err := s.pgcontainer.TearDown(s.ctx) if err != nil { panic(err) } @@ -109,10 +82,8 @@ func (s *IntegrationTestSuite) TearDownSuite() { } func TestIntegrationTestSuite(t *testing.T) { - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } suite.Run(t, new(IntegrationTestSuite)) diff --git a/backend/pkg/integration-test/integration-test-util.go b/backend/pkg/integration-test/integration-test-util.go index 3523116728..01e50da9f1 100644 --- a/backend/pkg/integration-test/integration-test-util.go +++ b/backend/pkg/integration-test/integration-test-util.go @@ -48,6 +48,34 @@ func CreatePostgresConnection( return resp.Msg.GetConnection() } +func CreateMysqlConnection( + ctx context.Context, + t *testing.T, + connclient mgmtv1alpha1connect.ConnectionServiceClient, + accountId string, + name string, + mysqlurl string, +) *mgmtv1alpha1.Connection { + resp, err := connclient.CreateConnection( + ctx, + connect.NewRequest(&mgmtv1alpha1.CreateConnectionRequest{ + AccountId: accountId, + Name: name, + ConnectionConfig: &mgmtv1alpha1.ConnectionConfig{ + Config: &mgmtv1alpha1.ConnectionConfig_MysqlConfig{ + MysqlConfig: &mgmtv1alpha1.MysqlConnectionConfig{ + ConnectionConfig: &mgmtv1alpha1.MysqlConnectionConfig_Url{ + Url: mysqlurl, + }, + }, + }, + }, + }), + ) + RequireNoErrResp(t, resp, err) + return resp.Msg.GetConnection() +} + func SetUser(ctx context.Context, t *testing.T, client mgmtv1alpha1connect.UserAccountServiceClient) string { resp, err := client.SetUser(ctx, connect.NewRequest(&mgmtv1alpha1.SetUserRequest{})) RequireNoErrResp(t, resp, err) diff --git a/backend/pkg/sqlmanager/mssql/integration_test.go b/backend/pkg/sqlmanager/mssql/integration_test.go index d1ade01e95..443ef17a1d 100644 --- a/backend/pkg/sqlmanager/mssql/integration_test.go +++ b/backend/pkg/sqlmanager/mssql/integration_test.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "log/slog" "net/url" "os" "path/filepath" @@ -17,6 +16,7 @@ import ( mssql_queries "github.com/nucleuscloud/neosync/backend/pkg/mssql-querier" sqlmanager_shared "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager/shared" + "github.com/nucleuscloud/neosync/internal/testutil" "github.com/stretchr/testify/suite" testmssql "github.com/testcontainers/testcontainers-go/modules/mssql" ) @@ -298,10 +298,8 @@ func (s *IntegrationTestSuite) TearDownSuite() { } func TestIntegrationTestSuite(t *testing.T) { - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } suite.Run(t, new(IntegrationTestSuite)) diff --git a/backend/pkg/sqlmanager/mssql_sql-manager_integration_test.go b/backend/pkg/sqlmanager/mssql_sql-manager_integration_test.go index 52f15ce1b1..8134090173 100644 --- a/backend/pkg/sqlmanager/mssql_sql-manager_integration_test.go +++ b/backend/pkg/sqlmanager/mssql_sql-manager_integration_test.go @@ -5,7 +5,6 @@ import ( "fmt" slog "log/slog" "net/url" - "os" "sync" "testing" @@ -13,6 +12,7 @@ import ( mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" mssql_queries "github.com/nucleuscloud/neosync/backend/pkg/mssql-querier" "github.com/nucleuscloud/neosync/backend/pkg/sqlconnect" + "github.com/nucleuscloud/neosync/internal/testutil" "github.com/stretchr/testify/suite" testmssql "github.com/testcontainers/testcontainers-go/modules/mssql" @@ -90,10 +90,8 @@ func (s *MssqlIntegrationTestSuite) TearDownSuite() { } func TestMssqlIntegrationTestSuite(t *testing.T) { - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } suite.Run(t, new(MssqlIntegrationTestSuite)) diff --git a/backend/pkg/sqlmanager/mysql/integration_test.go b/backend/pkg/sqlmanager/mysql/integration_test.go index 867ef3b375..3b76415d8c 100644 --- a/backend/pkg/sqlmanager/mysql/integration_test.go +++ b/backend/pkg/sqlmanager/mysql/integration_test.go @@ -3,13 +3,12 @@ package sqlmanager_mysql import ( "context" "fmt" - "log/slog" - "os" "testing" _ "github.com/go-sql-driver/mysql" mysql_queries "github.com/nucleuscloud/neosync/backend/gen/go/db/dbschemas/mysql" + "github.com/nucleuscloud/neosync/internal/testutil" tcmysql "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/mysql" "github.com/stretchr/testify/suite" ) @@ -75,10 +74,8 @@ func (s *IntegrationTestSuite) TearDownSuite() { } func TestIntegrationTestSuite(t *testing.T) { - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } suite.Run(t, new(IntegrationTestSuite)) diff --git a/backend/pkg/sqlmanager/mysql_sql-manager_integration_test.go b/backend/pkg/sqlmanager/mysql_sql-manager_integration_test.go index 3f8dda54e7..538d6ac6af 100644 --- a/backend/pkg/sqlmanager/mysql_sql-manager_integration_test.go +++ b/backend/pkg/sqlmanager/mysql_sql-manager_integration_test.go @@ -2,9 +2,7 @@ package sqlmanager import ( context "context" - "fmt" slog "log/slog" - "os" "sync" "testing" @@ -15,6 +13,7 @@ import ( "github.com/stretchr/testify/suite" _ "github.com/go-sql-driver/mysql" + "github.com/nucleuscloud/neosync/internal/testutil" tcmysql "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/mysql" ) @@ -77,10 +76,8 @@ func (s *MysqlIntegrationTestSuite) TearDownSuite() { } func TestMysqlIntegrationTestSuite(t *testing.T) { - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } suite.Run(t, new(MysqlIntegrationTestSuite)) diff --git a/backend/pkg/sqlmanager/postgres/integration_test.go b/backend/pkg/sqlmanager/postgres/integration_test.go index e09f0c8215..62c51ef381 100644 --- a/backend/pkg/sqlmanager/postgres/integration_test.go +++ b/backend/pkg/sqlmanager/postgres/integration_test.go @@ -4,12 +4,11 @@ import ( "context" "database/sql" "fmt" - "log/slog" - "os" "testing" pg_queries "github.com/nucleuscloud/neosync/backend/gen/go/db/dbschemas/postgresql" sqlmanager_shared "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager/shared" + "github.com/nucleuscloud/neosync/internal/testutil" tcpostgres "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/postgres" "github.com/stretchr/testify/suite" ) @@ -79,10 +78,8 @@ func (s *IntegrationTestSuite) TearDownSuite() { } func TestIntegrationTestSuite(t *testing.T) { - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } suite.Run(t, new(IntegrationTestSuite)) diff --git a/backend/pkg/sqlmanager/postgres_sql-manager_integration_test.go b/backend/pkg/sqlmanager/postgres_sql-manager_integration_test.go index 2aee422bbd..57e653042e 100644 --- a/backend/pkg/sqlmanager/postgres_sql-manager_integration_test.go +++ b/backend/pkg/sqlmanager/postgres_sql-manager_integration_test.go @@ -2,12 +2,9 @@ package sqlmanager import ( context "context" - "fmt" slog "log/slog" - "os" "sync" "testing" - "time" "github.com/google/uuid" pg_queries "github.com/nucleuscloud/neosync/backend/gen/go/db/dbschemas/postgresql" @@ -17,15 +14,14 @@ import ( "github.com/stretchr/testify/suite" _ "github.com/jackc/pgx/v5/stdlib" - "github.com/testcontainers/testcontainers-go" - testpg "github.com/testcontainers/testcontainers-go/modules/postgres" - "github.com/testcontainers/testcontainers-go/wait" + "github.com/nucleuscloud/neosync/internal/testutil" + tcpostgres "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/postgres" ) type PostgresIntegrationTestSuite struct { suite.Suite - pgcontainer *testpg.PostgresContainer + pgcontainer *tcpostgres.PostgresTestContainer ctx context.Context @@ -40,27 +36,15 @@ type PostgresIntegrationTestSuite struct { func (s *PostgresIntegrationTestSuite) SetupSuite() { s.ctx = context.Background() - pgcontainer, err := testpg.Run( - s.ctx, - "postgres:15", - testcontainers.WithWaitStrategy( - wait.ForLog("database system is ready to accept connections"). - WithOccurrence(2).WithStartupTimeout(5*time.Second), - ), - ) + pgcontainer, err := tcpostgres.NewPostgresTestContainer(s.ctx) if err != nil { panic(err) } s.pgcontainer = pgcontainer - connstr, err := pgcontainer.ConnectionString(s.ctx, "sslmode=disable") - if err != nil { - panic(err) - } - s.pgcfg = &mgmtv1alpha1.PostgresConnectionConfig{ ConnectionConfig: &mgmtv1alpha1.PostgresConnectionConfig_Url{ - Url: connstr, + Url: pgcontainer.URL, }, } s.mgmtconn = &mgmtv1alpha1.Connection{ @@ -85,7 +69,7 @@ func (s *PostgresIntegrationTestSuite) TearDownTest() { func (s *PostgresIntegrationTestSuite) TearDownSuite() { if s.pgcontainer != nil { - err := s.pgcontainer.Terminate(s.ctx) + err := s.pgcontainer.TearDown(s.ctx) if err != nil { panic(err) } @@ -93,10 +77,8 @@ func (s *PostgresIntegrationTestSuite) TearDownSuite() { } func TestPostgresIntegrationTestSuite(t *testing.T) { - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } suite.Run(t, new(PostgresIntegrationTestSuite)) diff --git a/backend/services/mgmt/v1alpha1/integration_tests/integration_test.go b/backend/services/mgmt/v1alpha1/integration_tests/integration_test.go index dc4c160563..8557646ace 100644 --- a/backend/services/mgmt/v1alpha1/integration_tests/integration_test.go +++ b/backend/services/mgmt/v1alpha1/integration_tests/integration_test.go @@ -2,12 +2,10 @@ package integrationtests_test import ( "context" - "fmt" - "log/slog" - "os" "testing" tcneosyncapi "github.com/nucleuscloud/neosync/backend/pkg/integration-test" + "github.com/nucleuscloud/neosync/internal/testutil" "github.com/stretchr/testify/suite" ) @@ -50,10 +48,8 @@ func (s *IntegrationTestSuite) TearDownSuite() { } func TestIntegrationTestSuite(t *testing.T) { - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } suite.Run(t, new(IntegrationTestSuite)) diff --git a/cli/internal/cmds/neosync/sync/sync_integration_test.go b/cli/internal/cmds/neosync/sync/sync_integration_test.go index 1fda80b520..a95b0b4f7c 100644 --- a/cli/internal/cmds/neosync/sync/sync_integration_test.go +++ b/cli/internal/cmds/neosync/sync/sync_integration_test.go @@ -7,14 +7,14 @@ import ( tcneosyncapi "github.com/nucleuscloud/neosync/backend/pkg/integration-test" "github.com/nucleuscloud/neosync/cli/internal/output" "github.com/nucleuscloud/neosync/internal/testutil" + tcmysql "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/mysql" tcpostgres "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/postgres" "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" ) const neosyncDbMigrationsPath = "../../../../../backend/sql/postgresql/schema" -func Test_Sync_Postgres(t *testing.T) { +func Test_Sync(t *testing.T) { t.Parallel() ok := testutil.ShouldRunIntegrationTest() if !ok { @@ -22,86 +22,128 @@ func Test_Sync_Postgres(t *testing.T) { } ctx := context.Background() - var neosyncApi *tcneosyncapi.NeosyncApiTestClient - var postgres *tcpostgres.PostgresTestSyncContainer - - errgrp := errgroup.Group{} - errgrp.Go(func() error { - p, err := tcpostgres.NewPostgresTestSyncContainer(ctx, []tcpostgres.Option{}, []tcpostgres.Option{}) - if err != nil { - return err - } - postgres = p - return nil - }) - - errgrp.Go(func() error { - api, err := tcneosyncapi.NewNeosyncApiTestClient(ctx, t, tcneosyncapi.WithMigrationsDirectory(neosyncDbMigrationsPath)) - if err != nil { - return err - } - neosyncApi = api - return nil - }) - - err := errgrp.Wait() - if err != nil { - panic(err) - } - - testdataFolder := "../../../../../internal/testutil/testdata/postgres/humanresources" - err = postgres.Source.RunSqlFiles(ctx, &testdataFolder, []string{"create-tables.sql"}) - if err != nil { - panic(err) - } - err = postgres.Target.RunSqlFiles(ctx, &testdataFolder, []string{"create-schema.sql"}) + neosyncApi, err := tcneosyncapi.NewNeosyncApiTestClient(ctx, t, tcneosyncapi.WithMigrationsDirectory(neosyncDbMigrationsPath)) if err != nil { panic(err) } connclient := neosyncApi.UnauthdClients.Connections conndataclient := neosyncApi.UnauthdClients.ConnectionData - sqlmanagerclient := tcneosyncapi.NewTestSqlManagerClient() discardLogger := testutil.GetTestCharmSlogger() - accountId := tcneosyncapi.CreatePersonalAccount(ctx, t, neosyncApi.UnauthdClients.Users) - sourceConn := tcneosyncapi.CreatePostgresConnection(ctx, t, neosyncApi.UnauthdClients.Connections, accountId, "source", postgres.Source.URL) - t.Run("sync_postgres", func(t *testing.T) { - outputType := output.PlainOutput - cmdConfig := &cmdConfig{ - Source: &sourceConfig{ - ConnectionId: sourceConn.Id, - }, - Destination: &sqlDestinationConfig{ - ConnectionUrl: postgres.Target.URL, - Driver: postgresDriver, - InitSchema: true, - TruncateBeforeInsert: true, - TruncateCascade: true, - }, - OutputType: &outputType, - AccountId: &accountId, + outputType := output.PlainOutput + + t.Run("postgres", func(t *testing.T) { + t.Parallel() + postgres, err := tcpostgres.NewPostgresTestSyncContainer(ctx, []tcpostgres.Option{}, []tcpostgres.Option{}) + if err != nil { + panic(err) + } + + testdataFolder := "../../../../../internal/testutil/testdata/postgres/humanresources" + err = postgres.Source.RunSqlFiles(ctx, &testdataFolder, []string{"create-tables.sql"}) + if err != nil { + panic(err) } - sync := &clisync{ - connectiondataclient: conndataclient, - connectionclient: connclient, - sqlmanagerclient: sqlmanagerclient, - ctx: ctx, - logger: discardLogger, - cmd: cmdConfig, + err = postgres.Target.RunSqlFiles(ctx, &testdataFolder, []string{"create-schema.sql"}) + if err != nil { + panic(err) } - err := sync.configureAndRunSync() - require.NoError(t, err) + sourceConn := tcneosyncapi.CreatePostgresConnection(ctx, t, neosyncApi.UnauthdClients.Connections, accountId, "postgres-source", postgres.Source.URL) + + t.Run("sync", func(t *testing.T) { + cmdConfig := &cmdConfig{ + Source: &sourceConfig{ + ConnectionId: sourceConn.Id, + }, + Destination: &sqlDestinationConfig{ + ConnectionUrl: postgres.Target.URL, + Driver: postgresDriver, + InitSchema: true, + TruncateBeforeInsert: true, + TruncateCascade: true, + }, + OutputType: &outputType, + AccountId: &accountId, + } + sync := &clisync{ + connectiondataclient: conndataclient, + connectionclient: connclient, + sqlmanagerclient: sqlmanagerclient, + ctx: ctx, + logger: discardLogger, + cmd: cmdConfig, + } + err := sync.configureAndRunSync() + require.NoError(t, err) + }) + + t.Cleanup(func() { + err := postgres.TearDown(ctx) + if err != nil { + panic(err) + } + }) }) - err = postgres.TearDown(ctx) - if err != nil { - panic(err) - } - err = neosyncApi.TearDown(ctx) - if err != nil { - panic(err) - } + t.Run("mysql", func(t *testing.T) { + t.Parallel() + mysql, err := tcmysql.NewMysqlTestSyncContainer(ctx, []tcmysql.Option{}, []tcmysql.Option{}) + if err != nil { + panic(err) + } + + testdataFolder := "../../../../../internal/testutil/testdata/mysql/humanresources" + err = mysql.Source.RunSqlFiles(ctx, &testdataFolder, []string{"create-tables.sql"}) + if err != nil { + panic(err) + } + err = mysql.Target.RunSqlFiles(ctx, &testdataFolder, []string{"create-schema.sql"}) + if err != nil { + panic(err) + } + sourceConn := tcneosyncapi.CreateMysqlConnection(ctx, t, neosyncApi.UnauthdClients.Connections, accountId, "mysql-source", mysql.Source.URL) + + t.Run("sync", func(t *testing.T) { + cmdConfig := &cmdConfig{ + Source: &sourceConfig{ + ConnectionId: sourceConn.Id, + }, + Destination: &sqlDestinationConfig{ + ConnectionUrl: mysql.Target.URL, + Driver: mysqlDriver, + InitSchema: true, + TruncateBeforeInsert: true, + }, + OutputType: &outputType, + AccountId: &accountId, + } + sync := &clisync{ + connectiondataclient: conndataclient, + connectionclient: connclient, + sqlmanagerclient: sqlmanagerclient, + ctx: ctx, + logger: discardLogger, + cmd: cmdConfig, + } + err := sync.configureAndRunSync() + require.NoError(t, err) + }) + + t.Cleanup(func() { + err := mysql.TearDown(ctx) + if err != nil { + panic(err) + } + }) + }) + + t.Cleanup(func() { + err = neosyncApi.TearDown(ctx) + if err != nil { + panic(err) + } + }) } diff --git a/internal/sshtunnel/dialer_integration_test.go b/internal/sshtunnel/dialer_integration_test.go index 5a947c5de9..5910c7e07d 100644 --- a/internal/sshtunnel/dialer_integration_test.go +++ b/internal/sshtunnel/dialer_integration_test.go @@ -4,9 +4,6 @@ import ( "context" "database/sql" "database/sql/driver" - "fmt" - "log/slog" - "os" "testing" "time" @@ -15,24 +12,19 @@ import ( "github.com/nucleuscloud/neosync/internal/sshtunnel/connectors/mssqltunconnector" "github.com/nucleuscloud/neosync/internal/sshtunnel/connectors/mysqltunconnector" "github.com/nucleuscloud/neosync/internal/sshtunnel/connectors/postgrestunconnector" + "github.com/nucleuscloud/neosync/internal/testutil" + tcmysql "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/mysql" + tcpostgres "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/postgres" "github.com/stretchr/testify/require" "golang.org/x/crypto/ssh" - "github.com/testcontainers/testcontainers-go" testmssql "github.com/testcontainers/testcontainers-go/modules/mssql" - testmysql "github.com/testcontainers/testcontainers-go/modules/mysql" - "github.com/testcontainers/testcontainers-go/modules/postgres" - testpg "github.com/testcontainers/testcontainers-go/modules/postgres" - - "github.com/testcontainers/testcontainers-go/wait" ) func Test_NewLazySSHDialer(t *testing.T) { t.Parallel() - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } @@ -60,20 +52,13 @@ func Test_NewLazySSHDialer(t *testing.T) { t.Run("postgres", func(t *testing.T) { t.Parallel() - container, err := testpg.Run( - ctx, - "postgres:15", - postgres.WithDatabase("postgres"), - testcontainers.WithWaitStrategy( - wait.ForLog("database system is ready to accept connections"). - WithOccurrence(2).WithStartupTimeout(20*time.Second), - ), - ) - require.NoError(t, err) - connstr, err := container.ConnectionString(ctx, "sslmode=disable") + container, err := tcpostgres.NewPostgresTestContainer(ctx) + if err != nil { + panic(err) + } require.NoError(t, err) - connector, cleanup, err := postgrestunconnector.New(dialer, connstr) + connector, cleanup, err := postgrestunconnector.New(dialer, container.URL) require.NoError(t, err) defer cleanup() @@ -83,21 +68,12 @@ func Test_NewLazySSHDialer(t *testing.T) { t.Run("mysql", func(t *testing.T) { t.Parallel() - container, err := testmysql.Run(ctx, - "mysql:8.0.36", - testmysql.WithDatabase("mydb"), - testmysql.WithUsername("root"), - testmysql.WithPassword("password"), - testcontainers.WithWaitStrategy( - wait.ForLog("port: 3306 MySQL Community Server"). - WithOccurrence(1).WithStartupTimeout(20*time.Second), - ), - ) - require.NoError(t, err) - connstr, err := container.ConnectionString(ctx) - require.NoError(t, err) + container, err := tcmysql.NewMysqlTestContainer(ctx) + if err != nil { + panic(err) + } - connector, cleanup, err := mysqltunconnector.New(dialer, connstr) + connector, cleanup, err := mysqltunconnector.New(dialer, container.URL) require.NoError(t, err) defer cleanup() diff --git a/internal/testutil/testdata/mysql/humanresources/create-schema.sql b/internal/testutil/testdata/mysql/humanresources/create-schema.sql new file mode 100644 index 0000000000..8f0d1c740e --- /dev/null +++ b/internal/testutil/testdata/mysql/humanresources/create-schema.sql @@ -0,0 +1 @@ +CREATE DATABASE IF NOT EXISTS humanresources; diff --git a/internal/testutil/testdata/mysql/humanresources/create-tables.sql b/internal/testutil/testdata/mysql/humanresources/create-tables.sql new file mode 100644 index 0000000000..39a32b25bb --- /dev/null +++ b/internal/testutil/testdata/mysql/humanresources/create-tables.sql @@ -0,0 +1,225 @@ +CREATE DATABASE IF NOT EXISTS humanresources; +USE humanresources; +CREATE TABLE regions ( + region_id INT (11) AUTO_INCREMENT PRIMARY KEY, + region_name VARCHAR (25) DEFAULT NULL +); + +CREATE TABLE countries ( + country_id CHAR (2) PRIMARY KEY, + country_name VARCHAR (40) DEFAULT NULL, + region_id INT (11) NOT NULL, + FOREIGN KEY (region_id) REFERENCES regions (region_id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE TABLE locations ( + location_id INT (11) AUTO_INCREMENT PRIMARY KEY, + street_address VARCHAR (40) DEFAULT NULL, + postal_code VARCHAR (12) DEFAULT NULL, + city VARCHAR (30) NOT NULL, + state_province VARCHAR (25) DEFAULT NULL, + country_id CHAR (2) NOT NULL, + FOREIGN KEY (country_id) REFERENCES countries (country_id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE TABLE jobs ( + job_id INT (11) AUTO_INCREMENT PRIMARY KEY, + job_title VARCHAR (35) NOT NULL, + min_salary DECIMAL (8, 2) DEFAULT NULL, + max_salary DECIMAL (8, 2) DEFAULT NULL +); + +CREATE TABLE departments ( + department_id INT (11) AUTO_INCREMENT PRIMARY KEY, + department_name VARCHAR (30) NOT NULL, + location_id INT (11) DEFAULT NULL, + FOREIGN KEY (location_id) REFERENCES locations (location_id) ON DELETE CASCADE ON UPDATE CASCADE +); + +-- DATE broken in CLI SYNC +-- CREATE TABLE employees ( +-- employee_id INT (11) AUTO_INCREMENT PRIMARY KEY, +-- first_name VARCHAR (20) DEFAULT NULL, +-- last_name VARCHAR (25) NOT NULL, +-- email VARCHAR (100) NOT NULL, +-- phone_number VARCHAR (20) DEFAULT NULL, +-- hire_date DATE NOT NULL, +-- job_id INT (11) NOT NULL, +-- salary DECIMAL (8, 2) NOT NULL, +-- manager_id INT (11) DEFAULT NULL, +-- department_id INT (11) DEFAULT NULL, +-- FOREIGN KEY (job_id) REFERENCES jobs (job_id) ON DELETE CASCADE ON UPDATE CASCADE, +-- FOREIGN KEY (department_id) REFERENCES departments (department_id) ON DELETE CASCADE ON UPDATE CASCADE, +-- FOREIGN KEY (manager_id) REFERENCES employees (employee_id) +-- ); + +-- CREATE TABLE dependents ( +-- dependent_id INT (11) AUTO_INCREMENT PRIMARY KEY, +-- first_name VARCHAR (50) NOT NULL, +-- last_name VARCHAR (50) NOT NULL, +-- relationship VARCHAR (25) NOT NULL, +-- employee_id INT (11) NOT NULL, +-- FOREIGN KEY (employee_id) REFERENCES employees (employee_id) ON DELETE CASCADE ON UPDATE CASCADE +-- ); + + +/*Data for the table regions */ + +INSERT INTO regions(region_id,region_name) VALUES (1,'Europe'); +INSERT INTO regions(region_id,region_name) VALUES (2,'Americas'); +INSERT INTO regions(region_id,region_name) VALUES (3,'Asia'); +INSERT INTO regions(region_id,region_name) VALUES (4,'Middle East and Africa'); + + +/*Data for the table countries */ +INSERT INTO countries(country_id,country_name,region_id) VALUES ('AR','Argentina',2); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('AU','Australia',3); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('BE','Belgium',1); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('BR','Brazil',2); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('CA','Canada',2); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('CH','Switzerland',1); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('CN','China',3); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('DE','Germany',1); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('DK','Denmark',1); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('EG','Egypt',4); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('FR','France',1); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('HK','HongKong',3); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('IL','Israel',4); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('IN','India',3); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('IT','Italy',1); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('JP','Japan',3); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('KW','Kuwait',4); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('MX','Mexico',2); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('NG','Nigeria',4); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('NL','Netherlands',1); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('SG','Singapore',3); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('UK','United Kingdom',1); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('US','United States of America',2); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('ZM','Zambia',4); +INSERT INTO countries(country_id,country_name,region_id) VALUES ('ZW','Zimbabwe',4); + +/*Data for the table locations */ +INSERT INTO locations(location_id,street_address,postal_code,city,state_province,country_id) VALUES (1400,'2014 Jabberwocky Rd','26192','Southlake','Texas','US'); +INSERT INTO locations(location_id,street_address,postal_code,city,state_province,country_id) VALUES (1500,'2011 Interiors Blvd','99236','South San Francisco','California','US'); +INSERT INTO locations(location_id,street_address,postal_code,city,state_province,country_id) VALUES (1700,'2004 Charade Rd','98199','Seattle','Washington','US'); +INSERT INTO locations(location_id,street_address,postal_code,city,state_province,country_id) VALUES (1800,'147 Spadina Ave','M5V 2L7','Toronto','Ontario','CA'); +INSERT INTO locations(location_id,street_address,postal_code,city,state_province,country_id) VALUES (2400,'8204 Arthur St',NULL,'London',NULL,'UK'); +INSERT INTO locations(location_id,street_address,postal_code,city,state_province,country_id) VALUES (2500,'Magdalen Centre, The Oxford Science Park','OX9 9ZB','Oxford','Oxford','UK'); +INSERT INTO locations(location_id,street_address,postal_code,city,state_province,country_id) VALUES (2700,'Schwanthalerstr. 7031','80925','Munich','Bavaria','DE'); + + +/*Data for the table jobs */ + +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (1,'Public Accountant',4200.00,9000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (2,'Accounting Manager',8200.00,16000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (3,'Administration Assistant',3000.00,6000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (4,'President',20000.00,40000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (5,'Administration Vice President',15000.00,30000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (6,'Accountant',4200.00,9000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (7,'Finance Manager',8200.00,16000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (8,'Human Resources Representative',4000.00,9000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (9,'Programmer',4000.00,10000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (10,'Marketing Manager',9000.00,15000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (11,'Marketing Representative',4000.00,9000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (12,'Public Relations Representative',4500.00,10500.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (13,'Purchasing Clerk',2500.00,5500.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (14,'Purchasing Manager',8000.00,15000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (15,'Sales Manager',10000.00,20000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (16,'Sales Representative',6000.00,12000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (17,'Shipping Clerk',2500.00,5500.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (18,'Stock Clerk',2000.00,5000.00); +INSERT INTO jobs(job_id,job_title,min_salary,max_salary) VALUES (19,'Stock Manager',5500.00,8500.00); + + +/*Data for the table departments */ + +INSERT INTO departments(department_id,department_name,location_id) VALUES (1,'Administration',1700); +INSERT INTO departments(department_id,department_name,location_id) VALUES (2,'Marketing',1800); +INSERT INTO departments(department_id,department_name,location_id) VALUES (3,'Purchasing',1700); +INSERT INTO departments(department_id,department_name,location_id) VALUES (4,'Human Resources',2400); +INSERT INTO departments(department_id,department_name,location_id) VALUES (5,'Shipping',1500); +INSERT INTO departments(department_id,department_name,location_id) VALUES (6,'IT',1400); +INSERT INTO departments(department_id,department_name,location_id) VALUES (7,'Public Relations',2700); +INSERT INTO departments(department_id,department_name,location_id) VALUES (8,'Sales',2500); +INSERT INTO departments(department_id,department_name,location_id) VALUES (9,'Executive',1700); +INSERT INTO departments(department_id,department_name,location_id) VALUES (10,'Finance',1700); +INSERT INTO departments(department_id,department_name,location_id) VALUES (11,'Accounting',1700); + + + +-- /*Data for the table employees */ + +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (100,'Steven','King','steven.king@sqltutorial.org','515.123.4567','1987-06-17',4,24000.00,NULL,9); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (101,'Neena','Kochhar','neena.kochhar@sqltutorial.org','515.123.4568','1989-09-21',5,17000.00,100,9); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (102,'Lex','De Haan','lex.de haan@sqltutorial.org','515.123.4569','1993-01-13',5,17000.00,100,9); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (103,'Alexander','Hunold','alexander.hunold@sqltutorial.org','590.423.4567','1990-01-03',9,9000.00,102,6); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (104,'Bruce','Ernst','bruce.ernst@sqltutorial.org','590.423.4568','1991-05-21',9,6000.00,103,6); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (105,'David','Austin','david.austin@sqltutorial.org','590.423.4569','1997-06-25',9,4800.00,103,6); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (106,'Valli','Pataballa','valli.pataballa@sqltutorial.org','590.423.4560','1998-02-05',9,4800.00,103,6); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (107,'Diana','Lorentz','diana.lorentz@sqltutorial.org','590.423.5567','1999-02-07',9,4200.00,103,6); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (108,'Nancy','Greenberg','nancy.greenberg@sqltutorial.org','515.124.4569','1994-08-17',7,12000.00,101,10); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (109,'Daniel','Faviet','daniel.faviet@sqltutorial.org','515.124.4169','1994-08-16',6,9000.00,108,10); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (110,'John','Chen','john.chen@sqltutorial.org','515.124.4269','1997-09-28',6,8200.00,108,10); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (111,'Ismael','Sciarra','ismael.sciarra@sqltutorial.org','515.124.4369','1997-09-30',6,7700.00,108,10); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (112,'Jose Manuel','Urman','jose manuel.urman@sqltutorial.org','515.124.4469','1998-03-07',6,7800.00,108,10); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (113,'Luis','Popp','luis.popp@sqltutorial.org','515.124.4567','1999-12-07',6,6900.00,108,10); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (114,'Den','Raphaely','den.raphaely@sqltutorial.org','515.127.4561','1994-12-07',14,11000.00,100,3); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (115,'Alexander','Khoo','alexander.khoo@sqltutorial.org','515.127.4562','1995-05-18',13,3100.00,114,3); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (116,'Shelli','Baida','shelli.baida@sqltutorial.org','515.127.4563','1997-12-24',13,2900.00,114,3); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (117,'Sigal','Tobias','sigal.tobias@sqltutorial.org','515.127.4564','1997-07-24',13,2800.00,114,3); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (118,'Guy','Himuro','guy.himuro@sqltutorial.org','515.127.4565','1998-11-15',13,2600.00,114,3); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (119,'Karen','Colmenares','karen.colmenares@sqltutorial.org','515.127.4566','1999-08-10',13,2500.00,114,3); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (120,'Matthew','Weiss','matthew.weiss@sqltutorial.org','650.123.1234','1996-07-18',19,8000.00,100,5); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (121,'Adam','Fripp','adam.fripp@sqltutorial.org','650.123.2234','1997-04-10',19,8200.00,100,5); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (122,'Payam','Kaufling','payam.kaufling@sqltutorial.org','650.123.3234','1995-05-01',19,7900.00,100,5); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (123,'Shanta','Vollman','shanta.vollman@sqltutorial.org','650.123.4234','1997-10-10',19,6500.00,100,5); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (126,'Irene','Mikkilineni','irene.mikkilineni@sqltutorial.org','650.124.1224','1998-09-28',18,2700.00,120,5); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (145,'John','Russell','john.russell@sqltutorial.org',NULL,'1996-10-01',15,14000.00,100,8); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (146,'Karen','Partners','karen.partners@sqltutorial.org',NULL,'1997-01-05',15,13500.00,100,8); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (176,'Jonathon','Taylor','jonathon.taylor@sqltutorial.org',NULL,'1998-03-24',16,8600.00,100,8); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (177,'Jack','Livingston','jack.livingston@sqltutorial.org',NULL,'1998-04-23',16,8400.00,100,8); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (178,'Kimberely','Grant','kimberely.grant@sqltutorial.org',NULL,'1999-05-24',16,7000.00,100,8); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (179,'Charles','Johnson','charles.johnson@sqltutorial.org',NULL,'2000-01-04',16,6200.00,100,8); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (192,'Sarah','Bell','sarah.bell@sqltutorial.org','650.501.1876','1996-02-04',17,4000.00,123,5); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (193,'Britney','Everett','britney.everett@sqltutorial.org','650.501.2876','1997-03-03',17,3900.00,123,5); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (200,'Jennifer','Whalen','jennifer.whalen@sqltutorial.org','515.123.4444','1987-09-17',3,4400.00,101,1); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (201,'Michael','Hartstein','michael.hartstein@sqltutorial.org','515.123.5555','1996-02-17',10,13000.00,100,2); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (202,'Pat','Fay','pat.fay@sqltutorial.org','603.123.6666','1997-08-17',11,6000.00,201,2); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (203,'Susan','Mavris','susan.mavris@sqltutorial.org','515.123.7777','1994-06-07',8,6500.00,101,4); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (204,'Hermann','Baer','hermann.baer@sqltutorial.org','515.123.8888','1994-06-07',12,10000.00,101,7); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (205,'Shelley','Higgins','shelley.higgins@sqltutorial.org','515.123.8080','1994-06-07',2,12000.00,101,11); +-- INSERT INTO employees(employee_id,first_name,last_name,email,phone_number,hire_date,job_id,salary,manager_id,department_id) VALUES (206,'William','Gietz','william.gietz@sqltutorial.org','515.123.8181','1994-06-07',1,8300.00,205,11); + + +-- /*Data for the table dependents */ + +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (1,'Penelope','Gietz','Child',206); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (2,'Nick','Higgins','Child',205); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (3,'Ed','Whalen','Child',200); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (4,'Jennifer','King','Child',100); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (5,'Johnny','Kochhar','Child',101); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (6,'Bette','De Haan','Child',102); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (7,'Grace','Faviet','Child',109); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (8,'Matthew','Chen','Child',110); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (9,'Joe','Sciarra','Child',111); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (10,'Christian','Urman','Child',112); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (11,'Zero','Popp','Child',113); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (12,'Karl','Greenberg','Child',108); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (13,'Uma','Mavris','Child',203); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (14,'Vivien','Hunold','Child',103); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (15,'Cuba','Ernst','Child',104); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (16,'Fred','Austin','Child',105); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (17,'Helen','Pataballa','Child',106); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (18,'Dan','Lorentz','Child',107); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (19,'Bob','Hartstein','Child',201); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (20,'Lucille','Fay','Child',202); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (21,'Kirsten','Baer','Child',204); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (22,'Elvis','Khoo','Child',115); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (23,'Sandra','Baida','Child',116); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (24,'Cameron','Tobias','Child',117); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (25,'Kevin','Himuro','Child',118); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (26,'Rip','Colmenares','Child',119); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (27,'Julia','Raphaely','Child',114); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (28,'Woody','Russell','Child',145); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (29,'Alec','Partners','Child',146); +-- INSERT INTO dependents(dependent_id,first_name,last_name,relationship,employee_id) VALUES (30,'Sandra','Taylor','Child',176); diff --git a/internal/testutil/testdata/mysql/humanresources/teardown.sql b/internal/testutil/testdata/mysql/humanresources/teardown.sql new file mode 100644 index 0000000000..7db214c127 --- /dev/null +++ b/internal/testutil/testdata/mysql/humanresources/teardown.sql @@ -0,0 +1 @@ +DROP DATABASE IF EXISTS humanresources; diff --git a/worker/pkg/query-builder2/integration_test.go b/worker/pkg/query-builder2/integration_test.go index 80427ea060..9f8dddba77 100644 --- a/worker/pkg/query-builder2/integration_test.go +++ b/worker/pkg/query-builder2/integration_test.go @@ -3,21 +3,16 @@ package querybuilder2 import ( "context" "database/sql" - "fmt" - "log/slog" "os" "testing" - "time" - "github.com/jackc/pgx/v5/pgxpool" _ "github.com/microsoft/go-mssqldb" pg_queries "github.com/nucleuscloud/neosync/backend/gen/go/db/dbschemas/postgresql" sqlmanager_shared "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager/shared" + "github.com/nucleuscloud/neosync/internal/testutil" + tcpostgres "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/postgres" "github.com/stretchr/testify/suite" - "github.com/testcontainers/testcontainers-go" testmssql "github.com/testcontainers/testcontainers-go/modules/mssql" - testpg "github.com/testcontainers/testcontainers-go/modules/postgres" - "github.com/testcontainers/testcontainers-go/wait" ) type mssqlTest struct { @@ -28,7 +23,6 @@ type mssqlTest struct { type IntegrationTestSuite struct { suite.Suite - pgpool *pgxpool.Pool querier pg_queries.Querier setupSql string @@ -36,7 +30,7 @@ type IntegrationTestSuite struct { ctx context.Context - pgcontainer *testpg.PostgresContainer + pgcontainer *tcpostgres.PostgresTestContainer schema string @@ -81,40 +75,15 @@ func (s *IntegrationTestSuite) SetupSuite() { s.ctx = context.Background() s.schema = "genbenthosconfigs_querybuilder" - pgcontainer, err := testpg.Run( - s.ctx, - "postgres:15", - testcontainers.WithWaitStrategy( - wait.ForLog("database system is ready to accept connections"). - WithOccurrence(2).WithStartupTimeout(5*time.Second), - ), - ) + pgcontainer, err := tcpostgres.NewPostgresTestContainer(s.ctx) if err != nil { panic(err) } s.pgcontainer = pgcontainer - connstr, err := pgcontainer.ConnectionString(s.ctx) - if err != nil { - panic(err) - } - setupSql, err := os.ReadFile("./testdata/postgres/setup.sql") - if err != nil { - panic(err) - } - s.setupSql = string(setupSql) + s.setupSql = "testdata/postgres/setup.sql" + s.teardownSql = "testdata/postgres/teardown.sql" - teardownSql, err := os.ReadFile("./testdata/postgres/teardown.sql") - if err != nil { - panic(err) - } - s.teardownSql = string(teardownSql) - - pool, err := pgxpool.New(s.ctx, connstr) - if err != nil { - panic(err) - } - s.pgpool = pool s.querier = pg_queries.New() mssqlTest, err := s.SetupMssql() @@ -126,25 +95,22 @@ func (s *IntegrationTestSuite) SetupSuite() { // Runs before each test func (s *IntegrationTestSuite) SetupTest() { - _, err := s.pgpool.Exec(s.ctx, s.setupSql) + err := s.pgcontainer.RunSqlFiles(s.ctx, nil, []string{s.setupSql}) if err != nil { panic(err) } } func (s *IntegrationTestSuite) TearDownTest() { - _, err := s.pgpool.Exec(s.ctx, s.teardownSql) + err := s.pgcontainer.RunSqlFiles(s.ctx, nil, []string{s.teardownSql}) if err != nil { panic(err) } } func (s *IntegrationTestSuite) TearDownSuite() { - if s.pgpool != nil { - s.pgpool.Close() - } if s.pgcontainer != nil { - err := s.pgcontainer.Terminate(s.ctx) + err := s.pgcontainer.TearDown(s.ctx) if err != nil { panic(err) } @@ -163,10 +129,8 @@ func (s *IntegrationTestSuite) TearDownSuite() { } func TestIntegrationTestSuite(t *testing.T) { - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } suite.Run(t, new(IntegrationTestSuite)) diff --git a/worker/pkg/query-builder2/query-builder_integration_test.go b/worker/pkg/query-builder2/query-builder_integration_test.go index f8b177bc41..6d305bdd4a 100644 --- a/worker/pkg/query-builder2/query-builder_integration_test.go +++ b/worker/pkg/query-builder2/query-builder_integration_test.go @@ -81,7 +81,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_DoubleReference() { for table, selectQueryRunType := range sqlMap { sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) assert.NoError(s.T(), err) columnDescriptions := rows.FieldDescriptions() @@ -178,7 +178,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_DoubleRootSubset() { sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) assert.NoError(s.T(), err) columnDescriptions := rows.FieldDescriptions() @@ -369,7 +369,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_MultipleRoots() { sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) assert.NoError(s.T(), err) columnDescriptions := rows.FieldDescriptions() @@ -499,7 +499,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_MultipleSubsets() { sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) assert.NoError(s.T(), err) columnDescriptions := rows.FieldDescriptions() @@ -628,7 +628,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_MultipleSubsets_SubsetsByForei sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) assert.NoError(s.T(), err) columnDescriptions := rows.FieldDescriptions() @@ -753,7 +753,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_CircularDependency() { sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) assert.NoError(s.T(), err) columnDescriptions := rows.FieldDescriptions() @@ -859,7 +859,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_NoForeignKeys() { sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) assert.NoError(s.T(), err) columnDescriptions := rows.FieldDescriptions() @@ -948,7 +948,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_NoForeignKeys_NoSubsets() { sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) assert.NoError(s.T(), err) rowCount := 0 @@ -1038,7 +1038,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_SubsetCompositeKeys() { sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) assert.NoError(s.T(), err) columnDescriptions := rows.FieldDescriptions() @@ -1308,7 +1308,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_ComplexSubset_Postgres() { sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) if rows != nil { allrows = append(allrows, rows) } @@ -1437,7 +1437,7 @@ func (s *IntegrationTestSuite) Test_BuildQueryMap_Pruned_Joins() { sql := selectQueryRunType[tabledependency.RunTypeInsert] assert.NotEmpty(s.T(), sql, "table %s", table) - rows, err := s.pgpool.Query(s.ctx, sql) + rows, err := s.pgcontainer.DB.Query(s.ctx, sql) if rows != nil { allrows = append(allrows, rows) } diff --git a/worker/pkg/workflows/datasync/workflow/integration_test.go b/worker/pkg/workflows/datasync/workflow/integration_test.go index 14f98d5105..f38df777b3 100644 --- a/worker/pkg/workflows/datasync/workflow/integration_test.go +++ b/worker/pkg/workflows/datasync/workflow/integration_test.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "log/slog" "os" "testing" @@ -15,6 +14,7 @@ import ( mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" sqlmanager_shared "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager/shared" awsmanager "github.com/nucleuscloud/neosync/internal/aws" + "github.com/nucleuscloud/neosync/internal/testutil" tcmysql "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/mysql" tcpostgres "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/postgres" "github.com/stretchr/testify/suite" @@ -565,10 +565,8 @@ func (s *IntegrationTestSuite) TearDownSuite() { } func TestIntegrationTestSuite(t *testing.T) { - evkey := "INTEGRATION_TESTS_ENABLED" - shouldRun := os.Getenv(evkey) - if shouldRun != "1" { - slog.Warn(fmt.Sprintf("skipping integration tests, set %s=1 to enable", evkey)) + ok := testutil.ShouldRunIntegrationTest() + if !ok { return } suite.Run(t, new(IntegrationTestSuite))