update tables

datastore/postgres/migrations/indexer/09-delete-scannerlist.sql

@@ -0,0 +1,2 @@
+-- This table is not used, so no need to keep it around.
+DROP TABLE IF EXISTS scannerlist CASCADE;

datastore/postgres/migrations/indexer/10-package_scanartifact-id.sql

@@ -0,0 +1,5 @@
+-- The package table only save one copy for each unique name+version of a package (and a few more fields).
+-- It is possible an image has several packages with the same name+version (and the other fields)
+-- in different locations. Without this migration, the IndexReport only tracks one of those packages
+-- instead of a package for each unique location.
+ALTER TABLE IF EXISTS package_scanartifact ADD COLUMN id BIGSERIAL;

datastore/postgres/migrations/migrations.go

@@ -61,6 +61,14 @@ var IndexerMigrations = []migrate.Migration{
 		ID: 8,
 		Up: runFile("indexer/08-index-manifest_layer.sql"),
 	},
+	{
+		ID: 9,
+		Up: runFile("indexer/09-delete-scannerlist.sql"),
+	},
+	{
+		ID: 10,
+		Up: runFile("indexer/10-package_scanartifact-id.sql"),
+	},
 }
 
 var MatcherMigrations = []migrate.Migration{

datastore/postgres/migrations/migrations_test.go

@@ -2,22 +2,65 @@ package migrations
 
 import (
 	"bufio"
+	"fmt"
 	iofs "io/fs"
 	"os"
-	"path"
 	"path/filepath"
 	"regexp"
 	"slices"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/remind101/migrate"
 )
 
+func TestBasicIndexerMigrations(t *testing.T) {
+	testMigrations(t, "indexer", IndexerMigrations)
+}
+
+func TestBasicMatchMigrations(t *testing.T) {
+	testMigrations(t, "matcher", MatcherMigrations)
+}
+
+func testMigrations(t *testing.T, root string, migrations []migrate.Migration) {
+	var fileMigrations []string
+	err := iofs.WalkDir(fs, root, func(path string, d iofs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.Name() == root {
+			return nil
+		}
+		if !d.Type().IsRegular() {
+			return fmt.Errorf("%s is not a regular file", path)
+		}
+		if filepath.Ext(d.Name()) != ".sql" {
+			return fmt.Errorf("%s is not a .sql file", path)
+		}
+
+		fileMigrations = append(fileMigrations, path)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(fileMigrations) != len(migrations) {
+		t.Error(cmp.Diff(len(fileMigrations), len(migrations)))
+	}
+
+	for i, m := range migrations {
+		if m.ID != i+1 {
+			t.Error(cmp.Diff(m.ID, i+1))
+		}
+	}
+}
+
 func TestMigrationsMismatch(t *testing.T) {
 	var migrations, files []string
 
 	// Get referenced migrations
-	migrationLine, err := regexp.Compile(`runFile\(\"(.*)\"\)`)
+	migrationLine, err := regexp.Compile(`runFile\("(.*)"\)`)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -34,16 +77,15 @@ func TestMigrationsMismatch(t *testing.T) {
 		case ms == nil, len(ms) == 1:
 			continue
 		case len(ms) == 2:
-			migrations = append(migrations, path.Clean(string(ms[1])))
+			migrations = append(migrations, filepath.Clean(string(ms[1])))
 		}
 	}
 	if err := s.Err(); err != nil {
 		t.Error(err)
 	}
 	slices.Sort(migrations)
-
 	// Get migration files
-	err = iofs.WalkDir(os.DirFS("."), ".", func(p string, d iofs.DirEntry, err error) error {
+	err = iofs.WalkDir(fs, ".", func(p string, d iofs.DirEntry, err error) error {
 		switch {
 		case err != nil:
 			return err

datastore/postgres/packagesbylayer.go

@@ -48,9 +48,11 @@ FROM
 WHERE
 	name = $1 AND version = $2 AND kind = $3;
 `
+		// TODO: It'd be nice to just use package_scanartifact.id, but what to do about the source_package ID?
+		// Just use the source package ID for now I guess?
 		query = `
 SELECT
-	package.id,
+	package_scanartifact.id,
 	package.name,
 	package.kind,
 	package.version,