db cleanup (#341)

* db cleanup

* dev

* pg setup

* postgres uuid migration

* cleanup

* fk

* sqlite migration

* cleanup

* wal

* max open conns = 1

* fix panic

---------

Co-authored-by: BuckarooBanzay <[email protected]>

Author: BuckarooBanzay

.github/workflows/build.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/[email protected]
         with:
-          go-version: 1.18
+          go-version: "1.21"
 
       - name: Set up nodejs
         uses: actions/setup-node@v3

.github/workflows/go-test.yml

@@ -11,7 +11,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/[email protected]
       with:
-        go-version: '1.18'
+        go-version: '1.21'
 
     - name: test
       run: go test ./... -coverprofile=profile.cov

db/postgres/initialblocks.go

@@ -16,8 +16,8 @@ const (
 	SETTING_LAST_Y_BLOCK = "last_y_block"
 )
 
-func (this *PostgresAccessor) countBlocks(x1, y1, z1, x2, y2, z2 int) (int, error) {
-	rows, err := this.db.Query(getBlockCountByInitialTileQuery,
+func (a *PostgresAccessor) countBlocks(x1, y1, z1, x2, y2, z2 int) (int, error) {
+	rows, err := a.db.Query(getBlockCountByInitialTileQuery,
 		x1, y1, z1, x2, y2, z2,
 	)
 
@@ -27,7 +27,7 @@ func (this *PostgresAccessor) countBlocks(x1, y1, z1, x2, y2, z2 int) (int, erro
 
 	defer rows.Close()
 
-	for rows.Next() {
+	if rows.Next() {
 		var count int
 
 		err = rows.Scan(&count)
@@ -53,7 +53,7 @@ func (this *PostgresAccessor) countBlocks(x1, y1, z1, x2, y2, z2 int) (int, erro
 //Zoom 9:
 //10 mapblocks height * 16 * 16 == 2560
 
-func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers []*layer.Layer, limit int) (*db.InitialBlocksResult, error) {
+func (a *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers []*layer.Layer, limit int) (*db.InitialBlocksResult, error) {
 
 	lastlayer := s.GetInt(SETTING_LAST_LAYER, 0)
 	lastxblock := s.GetInt(SETTING_LAST_X_BLOCK, -129)
@@ -111,7 +111,7 @@ func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers
 
 	if lastxblock <= -128 {
 		//first x entry, check z stride
-		stridecount := this.intQuery(`
+		stridecount := a.intQuery(`
 			select count(*) from blocks
 			where posz >= $1 and posz <= $2
 			and posy >= $3 and posy <= $4`,
@@ -139,7 +139,7 @@ func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers
 		}
 	}
 
-	count, err := this.countBlocks(minX, minY, minZ, maxX, maxY, maxZ)
+	count, err := a.countBlocks(minX, minY, minZ, maxX, maxY, maxZ)
 
 	if err != nil {
 		return nil, err
@@ -150,7 +150,7 @@ func (this *PostgresAccessor) FindNextInitialBlocks(s settings.Settings, layers
 
 	if count > 0 {
 
-		rows, err := this.db.Query(getBlocksByInitialTileQuery,
+		rows, err := a.db.Query(getBlocksByInitialTileQuery,
 			minX, minY, minZ, maxX, maxY, maxZ,
 		)
 

public/sql/postgres_mapdb_migrate.sql renamed to db/postgres/migrations/postgres_mapdb_migrate.sql

@@ -2,9 +2,9 @@ package postgres
 
 import (
 	"database/sql"
+	"embed"
 	"mapserver/coords"
 	"mapserver/db"
-	"mapserver/public"
 	"time"
 
 	_ "github.com/lib/pq"
@@ -15,6 +15,9 @@ type PostgresAccessor struct {
 	db *sql.DB
 }
 
+//go:embed migrations/*.sql
+var migrations embed.FS
+
 func (db *PostgresAccessor) Migrate() error {
 	hasMtime := true
 	_, err := db.db.Query("select max(mtime) from blocks")
@@ -26,7 +29,7 @@ func (db *PostgresAccessor) Migrate() error {
 		log.Info("Migrating database, this might take a while depending on the mapblock-count")
 		start := time.Now()
 
-		sql, err := public.Files.ReadFile("sql/postgres_mapdb_migrate.sql")
+		sql, err := migrations.ReadFile("migrations/postgres_mapdb_migrate.sql")
 		if err != nil {
 			return err
 		}
@@ -48,10 +51,10 @@ func convertRows(posx, posy, posz int, data []byte, mtime int64) *db.Block {
 	return &db.Block{Pos: c, Data: data, Mtime: mtime}
 }
 
-func (this *PostgresAccessor) FindBlocksByMtime(gtmtime int64, limit int) ([]*db.Block, error) {
+func (a *PostgresAccessor) FindBlocksByMtime(gtmtime int64, limit int) ([]*db.Block, error) {
 	blocks := make([]*db.Block, 0)
 
-	rows, err := this.db.Query(getBlocksByMtimeQuery, gtmtime, limit)
+	rows, err := a.db.Query(getBlocksByMtimeQuery, gtmtime, limit)
 	if err != nil {
 		return nil, err
 	}
@@ -75,8 +78,8 @@ func (this *PostgresAccessor) FindBlocksByMtime(gtmtime int64, limit int) ([]*db
 	return blocks, nil
 }
 
-func (this *PostgresAccessor) CountBlocks(frommtime, tomtime int64) (int, error) {
-	rows, err := this.db.Query(countBlocksQuery, frommtime, tomtime)
+func (a *PostgresAccessor) CountBlocks(frommtime, tomtime int64) (int, error) {
+	rows, err := a.db.Query(countBlocksQuery, frommtime, tomtime)
 	if err != nil {
 		panic(err)
 	}
@@ -97,8 +100,8 @@ func (this *PostgresAccessor) CountBlocks(frommtime, tomtime int64) (int, error)
 	return 0, nil
 }
 
-func (db *PostgresAccessor) GetTimestamp() (int64, error) {
-	rows, err := db.db.Query(getTimestampQuery)
+func (a *PostgresAccessor) GetTimestamp() (int64, error) {
+	rows, err := a.db.Query(getTimestampQuery)
 	if err != nil {
 		return 0, err
 	}
@@ -119,8 +122,8 @@ func (db *PostgresAccessor) GetTimestamp() (int64, error) {
 	return 0, nil
 }
 
-func (this *PostgresAccessor) GetBlock(pos *coords.MapBlockCoords) (*db.Block, error) {
-	rows, err := this.db.Query(getBlockQuery, pos.X, pos.Y, pos.Z)
+func (a *PostgresAccessor) GetBlock(pos *coords.MapBlockCoords) (*db.Block, error) {
+	rows, err := a.db.Query(getBlockQuery, pos.X, pos.Y, pos.Z)
 	if err != nil {
 		return nil, err
 	}

db/postgres/postgres.go

@@ -1,7 +1,7 @@
 package postgres
 
-func (this *PostgresAccessor) intQuery(q string, params ...interface{}) int {
-	rows, err := this.db.Query(q, params...)
+func (a *PostgresAccessor) intQuery(q string, params ...interface{}) int {
+	rows, err := a.db.Query(q, params...)
 	if err != nil {
 		panic(err)
 	}

db/postgres/util.go

@@ -5,8 +5,6 @@ import (
 	"mapserver/db"
 	"mapserver/layer"
 	"mapserver/settings"
-
-	_ "modernc.org/sqlite"
 )
 
 const (
@@ -23,7 +21,7 @@ order by b.pos asc, b.mtime asc
 limit ?
 `
 
-func (this *Sqlite3Accessor) FindNextInitialBlocks(s settings.Settings, layers []*layer.Layer, limit int) (*db.InitialBlocksResult, error) {
+func (a *Sqlite3Accessor) FindNextInitialBlocks(s settings.Settings, layers []*layer.Layer, limit int) (*db.InitialBlocksResult, error) {
 	result := &db.InitialBlocksResult{}
 
 	blocks := make([]*db.Block, 0)
@@ -33,7 +31,7 @@ func (this *Sqlite3Accessor) FindNextInitialBlocks(s settings.Settings, layers [
 	totallegacycount := s.GetInt64(SETTING_TOTAL_LEGACY_COUNT, -1)
 	if totallegacycount == -1 {
 		//Query from db
-		totallegacycount, err := this.CountBlocks()
+		totallegacycount, err := a.CountBlocks()
 
 		if err != nil {
 			panic(err)
@@ -42,7 +40,7 @@ func (this *Sqlite3Accessor) FindNextInitialBlocks(s settings.Settings, layers [
 		s.SetInt64(SETTING_TOTAL_LEGACY_COUNT, int64(totallegacycount))
 	}
 
-	rows, err := this.db.Query(getLastBlockQuery, lastpos, limit)
+	rows, err := a.db.Query(getLastBlockQuery, lastpos, limit)
 	if err != nil {
 		return nil, err
 	}

db/sqlite/initialblocks.go

@@ -2,10 +2,10 @@ package sqlite
 
 import (
 	"database/sql"
+	"embed"
 	"errors"
 	"mapserver/coords"
 	"mapserver/db"
-	"mapserver/public"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -22,6 +22,9 @@ type Sqlite3Accessor struct {
 	filename string
 }
 
+//go:embed migrations/*.sql
+var migrations embed.FS
+
 func (db *Sqlite3Accessor) Migrate() error {
 
 	//RW connection
@@ -44,7 +47,7 @@ func (db *Sqlite3Accessor) Migrate() error {
 		}).Info("Migrating database, this might take a while depending on the mapblock-count")
 		start := time.Now()
 
-		sql, err := public.Files.ReadFile("sql/sqlite_mapdb_migrate.sql")
+		sql, err := migrations.ReadFile("migrations/sqlite_mapdb_migrate.sql")
 		if err != nil {
 			return err
 		}

public/sql/sqlite_mapdb_migrate.sql renamed to db/sqlite/migrations/sqlite_mapdb_migrate.sql

@@ -0,0 +1,17 @@
+gameid = minetest
+creative_mode = true
+enable_damage = true
+
+auth_backend = postgresql
+pgsql_auth_connection = host=postgres user=postgres password=enter dbname=postgres
+
+player_backend = postgresql
+pgsql_player_connection = host=postgres port=5432 user=postgres password=enter dbname=postgres
+
+backend = postgresql
+pgsql_connection = host=postgres port=5432 user=postgres password=enter dbname=postgres
+
+mod_storage_backend = postgresql
+pgsql_mod_storage_connection = host=postgres user=postgres password=enter dbname=postgres
+
+pgsql_mapserver_connection = host=postgres port=5432 user=postgres password=enter dbname=postgres

db/sqlite/sqlite.go

@@ -1,58 +1,37 @@
 
-# System overview
-
-<img src="./Overview.png">
-
 # Build dependencies
 
-* go >= 1.16
-* nodejs >= v17.4.0
-* npm >= 8.3.0
+* docker
+* docker-compose
 
 # Create the frontend bundle
 
 ```bash
-cd public
-npm ci
-npm run bundle
+docker-compose up mapserver_frontend
 ```
 
-# Development setup
-
-Working directory: `./server`
-
-## Preparing the files and map
-
-Copy your `map.sqlite` into the working directory if you want to test with
-a sqlite map database
-
-### world.mt
-
-You need a `world.mt` too in order to make the connection to the database.
-In the sqlite case:
+# Development setup (sqlite)
 
-```
-gameid = minetest
-backend = sqlite3
-creative_mode = false
-enable_damage = false
-player_backend = files
+```bash
+# start the engine in the first window/shell
+docker-compose up minetest
+# and the mapserver in another
+docker-compose up mapserver
 ```
 
-For postgres:
-```
-gameid = minetest
-backend = postgresql
-creative_mode = true
-enable_damage = true
-player_backend = postgresql
-pgsql_connection = host=localhost port=5432 user=postgres password=enter dbname=postgres
-pgsql_player_connection = host=localhost port=5432 user=postgres password=enter dbname=postgres
-```
+# Development setup (postgres)
 
-## Running the server
+```bash
+# start postgres in the background
+docker-compose -f docker-compose.yml -f docker-compose.postgres.yml up -d postgres
+# start the engine in the first window/shell
+docker-compose -f docker-compose.yml -f docker-compose.postgres.yml up minetest
+# and the mapserver in another
+docker-compose -f docker-compose.yml -f docker-compose.postgres.yml up mapserver
+```
 
-* Create a `mapserver.json` with `go run . -createconfig`
-* Change the value `webdev` in the `mapserver.json` to `true`
-* Start the server with `go run .` or with debug output: `go run . -debug`
-* The web files in `public/` can now be changed on the fly without restarting the server
+Utilities:
+```sh
+# psql
+docker-compose -f docker-compose.yml -f docker-compose.postgres.yml exec postgres psql -U postgres
+```

dev/world.postgres.mt

@@ -0,0 +1,21 @@
+version: "3.6"
+
+services:
+ postgres:
+  image: postgres:16.1
+  restart: always
+  environment:
+   POSTGRES_PASSWORD: enter
+  volumes:
+   - "pg_data:/var/lib/postgresql/data"
+
+ mapserver:
+  volumes:
+   - "./dev/world.postgres.mt:/data/world/world.mt"
+
+ minetest:
+  volumes:
+   - "./dev/world.postgres.mt:/root/.minetest/worlds/world/world.mt"
+
+volumes:
+ pg_data: {}

doc/Overview.png

@@ -2,21 +2,23 @@ version: "3.6"
 
 services:
  mapserver_frontend:
-  image: node:alpine3.13
+  image: node:21.5.0-alpine
   volumes:
    - "dev_home:/root"
    - ".:/data"
   working_dir: /data/public
-  command: ["npm", "i"]
+  command: ["npm", "ci"]
 
  mapserver:
-  image: golang:1.20.3
+  image: golang:1.21.5
   depends_on:
    - minetest
    - mapserver_frontend
   volumes:
    - "dev_home:/root"
    - "world_data:/data/world"
+   - "go_dir:/go"
+   - "go_cache:/.cache"
    - ".:/data"
    - "./public:/data/world/public"
    - "./dev/mapserver.json:/data/world/mapserver.json"
@@ -26,7 +28,7 @@ services:
   command: ["go", "run", ".."]
 
  minetest:
-  image: registry.gitlab.com/minetest/minetest/server:5.6.1
+  image: registry.gitlab.com/minetest/minetest/server:5.7.0
   user: root
   volumes:
    - "world_data:/root/.minetest/worlds/world"
@@ -38,4 +40,6 @@ services:
 
 volumes:
   world_data: {}
-  dev_home: {}
+  dev_home: {}
+  go_dir: {}
+  go_cache: {}