feat: add repology exporter (#498)

Co-authored-by: Elsie <[email protected]>

server/Makefile

@@ -28,6 +28,7 @@ run:
 	PACSTALL_MATOMO_ENABLED="false" \
 	PACSTALL_REPOLOGY_ENABLED="false" \
 	go run bin/webserver/main.go
+
 dist/webserver: $(shell find . -not \( -path ./tmp -prune \) -not \( -path ./dist -prune \) -type f)
 	CGO_ENABLED=0 go build -o dist/webserver -ldflags "${LDFLAGS}" bin/webserver/main.go
 clean:

server/bin/webserver/main.go

@@ -7,6 +7,7 @@ import (
 	"github.com/fatih/color"
 	"pacstall.dev/webserver/config"
 	"pacstall.dev/webserver/log"
+	"pacstall.dev/webserver/repology"
 	"pacstall.dev/webserver/server"
 	ps_api "pacstall.dev/webserver/server/api/pacscripts"
 	repology_api "pacstall.dev/webserver/server/api/repology"
@@ -54,8 +55,6 @@ func main() {
 	}
 
 	startedAt := time.Now()
-	port := config.Port
-	refreshTimer := config.UpdateInterval
 
 	printLogo()
 
@@ -66,19 +65,20 @@ func main() {
 
 	server.OnServerOnline(func() {
 		log.NotifyCustom("🚀 Startup 🧑‍🚀", "Successfully started up.")
-		log.Info("Server is now online on port %v.\n", port)
+		log.Info("Server is now online on port %v.\n", config.Port)
 
 		log.Info("Booted in %v\n", color.GreenString("%v", time.Since(startedAt)))
 
-		log.Info("Attempting to parse existing pacscripts")
-		err := parser.ParseAll()
-		if err != nil {
-			log.Error("Failed to parse pacscripts: %v", err)
-		}
+		parser.ScheduleRefresh(config.UpdateInterval)
+		log.Info("Scheduled pacscripts to auto-refresh every %v", config.UpdateInterval)
 
-		parser.ScheduleRefresh(refreshTimer)
-		log.Info("Scheduled pacscripts to auto-refresh every %v", refreshTimer)
+		if config.Repology.Enabled {
+			repology.ScheduleRefresh(config.RepologyUpdateInterval)
+			log.Info("Scheduled repology to auto-refresh every %v", config.RepologyUpdateInterval)
+		} else {
+			log.Warn("Repology is disabled. Pacstall will not be able to fetch package data from Repology.")
+		}
 	})
 
-	server.Listen(port)
+	server.Listen(config.Port)
 }

server/config/build/vars.go

@@ -3,6 +3,7 @@ package build
 var Production = "false"
 
 var UpdateInterval = "900"
+var RepologyUpdateInterval = "43200"
 var TempDir = "./tmp"
 var MaxOpenFiles = "100"
 var GitURL = "https://github.com/pacstall/pacstall-programs.git"

server/config/vars.go

@@ -9,6 +9,7 @@ import (
 var Production = toBool(build.Production)
 
 var UpdateInterval = time.Duration(toInt(build.UpdateInterval)) * time.Second
+var RepologyUpdateInterval = time.Duration(toInt(build.RepologyUpdateInterval)) * time.Second
 var TempDir = build.TempDir
 var MaxOpenFiles = toInt(build.MaxOpenFiles)
 var GitURL = build.GitURL

server/go.sum

@@ -48,14 +48,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/mysql v1.5.0 h1:6hSAT5QcyIaty0jfnff0z0CLDjyRgZ8mlMHLqSt7uXM=
-gorm.io/driver/mysql v1.5.0/go.mod h1:FFla/fJuCvyTi7rJQd27qlNX2v3L6deTR1GgTjSOLPo=
 gorm.io/driver/mysql v1.5.1 h1:WUEH5VF9obL/lTtzjmML/5e6VfFR/788coz2uaVCAZw=
 gorm.io/driver/mysql v1.5.1/go.mod h1:Jo3Xu7mMhCyj8dlrb3WoCaRd1FhsVh+yMXb1jUInf5o=
-gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
-gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU=
-gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
-gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64=
 gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
 gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho=
 gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=

server/model/connection.go

@@ -5,6 +5,7 @@ import (
 
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
 	"pacstall.dev/webserver/config"
 )
 
@@ -22,7 +23,10 @@ func Instance() *gorm.DB {
 		return database
 	}
 
-	db, err := gorm.Open(mysql.Open(connectionString), &gorm.Config{})
+	db, err := gorm.Open(mysql.Open(connectionString), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+
 	if err != nil {
 		panic(fmt.Sprintf("failed to connect database: %v", err))
 	}

server/model/repology_project.go

@@ -0,0 +1,11 @@
+package model
+
+type RepologyProject struct {
+	Name string `gorm:"primaryKey"`
+}
+
+var RepologyProjectColumns = struct {
+	Name string
+}{
+	Name: "name",
+}

server/model/repology_project_provider.go

@@ -0,0 +1,47 @@
+package model
+
+const RepologyProjectProviderTableName = "repology_project_providers"
+
+type RepologyProjectProvider struct {
+	ID              uint            `gorm:"primarykey"`
+	ProjectName     string          `gorm:"index:"`
+	Project         RepologyProject `gorm:"foreignKey:Name"`
+	Repository      string
+	SubRepository   *string `gorm:"default:null"`
+	SourceName      *string `gorm:"index:,default:null"`
+	VisibleName     *string `gorm:"index:,default:null"`
+	BinaryName      *string `gorm:"index:,default:null"`
+	Version         string
+	OriginalVersion string
+	Status          string
+	Summary         string
+	Active          bool `gorm:"index:,default:false"`
+}
+
+var RepologyProjectProviderColumns = struct {
+	ID              string
+	ProjectName     string
+	Repository      string
+	SubRepository   string
+	SourceName      string
+	VisibleName     string
+	BinaryName      string
+	Version         string
+	OriginalVersion string
+	Status          string
+	Summary         string
+	Active          string
+}{
+	ID:              "id",
+	ProjectName:     "project_name",
+	Repository:      "repository",
+	SubRepository:   "sub_repository",
+	SourceName:      "source_name",
+	VisibleName:     "visible_name",
+	BinaryName:      "binary_name",
+	Version:         "version",
+	OriginalVersion: "original_version",
+	Status:          "status",
+	Summary:         "summary",
+	Active:          "active",
+}

server/repology/exporter.go

@@ -0,0 +1,5 @@
+package repology
+
+import "pacstall.dev/webserver/repology/internal"
+
+var ExportRepologyDatabase = internal.ExportRepologyDatabase