Refactor fixture loading for testing (#33024)

To help binary size and testing performance

go.mod

@@ -61,7 +61,6 @@ require (
 	github.com/go-redsync/redsync/v4 v4.13.0
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/go-swagger/go-swagger v0.31.0
-	github.com/go-testfixtures/testfixtures/v3 v3.11.0
 	github.com/go-webauthn/webauthn v0.11.2
 	github.com/gobwas/glob v0.2.3
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f
@@ -145,8 +144,6 @@ require (
 	filippo.io/edwards25519 v1.1.0 // indirect
 	git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
-	github.com/ClickHouse/ch-go v0.63.1 // indirect
-	github.com/ClickHouse/clickhouse-go/v2 v2.24.0 // indirect
 	github.com/DataDog/zstd v1.5.6 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.3.0 // indirect
@@ -204,8 +201,6 @@ require (
 	github.com/go-ap/errors v0.0.0-20240910140019-1e9d33cc1568 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/go-enry/go-oniguruma v1.2.1 // indirect
-	github.com/go-faster/city v1.0.1 // indirect
-	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
@@ -270,7 +265,6 @@ require (
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
-	github.com/paulmach/orb v0.11.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
@@ -285,7 +279,6 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.6.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
@@ -310,8 +303,6 @@ require (
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	go.etcd.io/bbolt v1.3.11 // indirect
 	go.mongodb.org/mongo-driver v1.17.1 // indirect
-	go.opentelemetry.io/otel v1.31.0 // indirect
-	go.opentelemetry.io/otel/trace v1.31.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect

models/fixtures/action_run_job.yml

@@ -64,7 +64,7 @@
   name: job2
   attempt: 1
   job_id: job2
-  needs: [job1]
+  needs: '["job1"]'
   task_id: 51
   status: 5
   started: 1683636528

models/fixtures/protected_tag.yml

@@ -2,23 +2,23 @@
   id: 1
   repo_id: 4
   name_pattern: /v.+/
-  allowlist_user_i_ds: []
-  allowlist_team_i_ds: []
+  allowlist_user_i_ds: "[]"
+  allowlist_team_i_ds: "[]"
   created_unix: 1715596037
   updated_unix: 1715596037
 -
   id: 2
   repo_id: 1
   name_pattern: v-*
-  allowlist_user_i_ds: []
-  allowlist_team_i_ds: []
+  allowlist_user_i_ds: "[]"
+  allowlist_team_i_ds: "[]"
   created_unix: 1715596037
   updated_unix: 1715596037
 -
   id: 3
   repo_id: 1
   name_pattern: v-1.1
-  allowlist_user_i_ds: [2]
-  allowlist_team_i_ds: []
+  allowlist_user_i_ds: "[2]"
+  allowlist_team_i_ds: "[]"
   created_unix: 1715596037
   updated_unix: 1715596037

models/migrations/base/tests.go

@@ -76,7 +76,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...any) (*xorm.Engine, fu
 			t.Errorf("error whilst initializing fixtures from %s: %v", fixturesDir, err)
 			return x, deferFn
 		}
-		if err := unittest.LoadFixtures(x); err != nil {
+		if err := unittest.LoadFixtures(); err != nil {
 			t.Errorf("error whilst loading fixtures from %s: %v", fixturesDir, err)
 			return x, deferFn
 		}

models/unittest/fixtures.go

@@ -5,88 +5,29 @@ package unittest
 
 import (
 	"fmt"
-	"time"
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/modules/auth/password/hash"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
 
-	"github.com/go-testfixtures/testfixtures/v3"
 	"xorm.io/xorm"
 	"xorm.io/xorm/schemas"
 )
 
-var fixturesLoader *testfixtures.Loader
-
-// GetXORMEngine gets the XORM engine
-func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
-	if len(engine) == 1 {
-		return engine[0]
-	}
-	return db.GetEngine(db.DefaultContext).(*xorm.Engine)
+type FixturesLoader interface {
+	Load() error
 }
 
-// InitFixtures initialize test fixtures for a test database
-func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
-	e := GetXORMEngine(engine...)
-	var fixtureOptionFiles func(*testfixtures.Loader) error
-	if opts.Dir != "" {
-		fixtureOptionFiles = testfixtures.Directory(opts.Dir)
-	} else {
-		fixtureOptionFiles = testfixtures.Files(opts.Files...)
-	}
-	var dialect string
-	switch e.Dialect().URI().DBType {
-	case schemas.POSTGRES:
-		dialect = "postgres"
-	case schemas.MYSQL:
-		dialect = "mysql"
-	case schemas.MSSQL:
-		dialect = "mssql"
-	case schemas.SQLITE:
-		dialect = "sqlite3"
-	default:
-		return fmt.Errorf("unsupported RDBMS for integration tests: %q", e.Dialect().URI().DBType)
-	}
-	loaderOptions := []func(loader *testfixtures.Loader) error{
-		testfixtures.Database(e.DB().DB),
-		testfixtures.Dialect(dialect),
-		testfixtures.DangerousSkipTestDatabaseCheck(),
-		fixtureOptionFiles,
-	}
-
-	if e.Dialect().URI().DBType == schemas.POSTGRES {
-		loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
-	}
+var fixturesLoader FixturesLoader
 
-	fixturesLoader, err = testfixtures.New(loaderOptions...)
-	if err != nil {
-		return err
-	}
-
-	// register the dummy hash algorithm function used in the test fixtures
-	_ = hash.Register("dummy", hash.NewDummyHasher)
-	setting.PasswordHashAlgo, _ = hash.SetDefaultPasswordHashAlgorithm("dummy")
-	return err
+// GetXORMEngine gets the XORM engine
+func GetXORMEngine() (x *xorm.Engine) {
+	return db.GetEngine(db.DefaultContext).(*xorm.Engine)
 }
 
-// LoadFixtures load fixtures for a test database
-func LoadFixtures(engine ...*xorm.Engine) error {
-	e := GetXORMEngine(engine...)
-	var err error
-	// (doubt) database transaction conflicts could occur and result in ROLLBACK? just try for a few times.
-	for i := 0; i < 5; i++ {
-		if err = fixturesLoader.Load(); err == nil {
-			break
-		}
-		time.Sleep(200 * time.Millisecond)
-	}
-	if err != nil {
-		return fmt.Errorf("LoadFixtures failed after retries: %w", err)
-	}
-	// Now if we're running postgres we need to tell it to update the sequences
-	if e.Dialect().URI().DBType == schemas.POSTGRES {
-		results, err := e.QueryString(`SELECT 'SELECT SETVAL(' ||
+func loadFixtureResetSeqPgsql(e *xorm.Engine) error {
+	results, err := e.QueryString(`SELECT 'SELECT SETVAL(' ||
 		quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) ||
 		', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' ||
 		quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';'
@@ -102,19 +43,42 @@ func LoadFixtures(engine ...*xorm.Engine) error {
 	     AND D.refobjsubid = C.attnum
 	     AND T.relname = PGT.tablename
 	 ORDER BY S.relname;`)
-		if err != nil {
-			return fmt.Errorf("failed to generate sequence update: %w", err)
-		}
-		for _, r := range results {
-			for _, value := range r {
-				_, err = e.Exec(value)
-				if err != nil {
-					return fmt.Errorf("failed to update sequence: %s, error: %w", value, err)
-				}
+	if err != nil {
+		return fmt.Errorf("failed to generate sequence update: %w", err)
+	}
+	for _, r := range results {
+		for _, value := range r {
+			_, err = e.Exec(value)
+			if err != nil {
+				return fmt.Errorf("failed to update sequence: %s, error: %w", value, err)
 			}
 		}
 	}
+	return nil
+}
+
+// InitFixtures initialize test fixtures for a test database
+func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
+	xormEngine := util.IfZero(util.OptionalArg(engine), GetXORMEngine())
+	fixturesLoader, err = NewFixturesLoader(xormEngine, opts)
+	// fixturesLoader = NewFixturesLoaderVendor(xormEngine, opts)
+
+	// register the dummy hash algorithm function used in the test fixtures
 	_ = hash.Register("dummy", hash.NewDummyHasher)
 	setting.PasswordHashAlgo, _ = hash.SetDefaultPasswordHashAlgorithm("dummy")
+	return err
+}
+
+// LoadFixtures load fixtures for a test database
+func LoadFixtures() error {
+	if err := fixturesLoader.Load(); err != nil {
+		return err
+	}
+	// Now if we're running postgres we need to tell it to update the sequences
+	if GetXORMEngine().Dialect().URI().DBType == schemas.POSTGRES {
+		if err := loadFixtureResetSeqPgsql(GetXORMEngine()); err != nil {
+			return err
+		}
+	}
 	return nil
 }