fixup: clean up and add tests

alexbrdn · alexbrdn · commit ecdb6d03cf8f · 2024-12-12T15:41:27.000+01:00
diff --git a/api/tm-catalog.openapi.yaml b/api/tm-catalog.openapi.yaml
@@ -1571,6 +1571,7 @@ components:
         meta:
           page:
             elements: 2
+          lastUpdated: '2024-12-11T13:07:22+01:00'
         data:
           - 'links':
               'self': './inventory/.tmName/siemens/siemens/poc1000'
diff --git a/cmd/create_si.go b/cmd/create_si.go
@@ -11,15 +11,15 @@ import (
 // createSiCmd represents the createSi command
 var createSiCmd = &cobra.Command{
 	Use:   "create-si",
-	Short: "Creates or updates a bleve search index",
-	Long:  `Creates or updates a bleve search index for all TMs`,
-	Run:   executeCreateSearchIndex,
+	Short: "Create or update a bleve search index",
+	Long: `Create or update a bleve search index for deep searching TMs with '--search <query> --deep'. Usually needs to
+be called only once per repository. Afterwards, updates are performed automatically an outdated search index is detected.`,
+	Run: executeCreateSearchIndex,
 }
 
 func init() {
 	RootCmd.AddCommand(createSiCmd)
-	createSiCmd.Flags().StringP("repo", "r", "", "name of the remote to pull from")
-	createSiCmd.Flags().StringP("directory", "d", "", "TM repository directory to pull from")
+	AddRepoConstraintFlags(createSiCmd)
 }
 
 func executeCreateSearchIndex(cmd *cobra.Command, args []string) {
diff --git a/go.mod b/go.mod
@@ -5,10 +5,12 @@ go 1.23
 require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/MicahParks/keyfunc/v3 v3.2.5
+	github.com/blevesearch/bleve/v2 v2.4.2
 	github.com/buger/jsonparser v1.1.1
 	github.com/gofrs/flock v0.8.1
 	github.com/golang-jwt/jwt/v5 v5.2.0
 	github.com/google/renameio v1.0.1
+	github.com/google/uuid v1.5.0
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.1
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
@@ -27,7 +29,6 @@ require (
 	github.com/RoaringBitmap/roaring v1.9.3 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/bits-and-blooms/bitset v1.12.0 // indirect
-	github.com/blevesearch/bleve/v2 v2.4.2 // indirect
 	github.com/blevesearch/bleve_index_api v1.1.10 // indirect
 	github.com/blevesearch/geo v0.1.20 // indirect
 	github.com/blevesearch/go-faiss v1.0.20 // indirect
@@ -52,7 +53,6 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.3 // indirect
-	github.com/google/uuid v1.5.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
diff --git a/internal/app/cli/list.go b/internal/app/cli/list.go
@@ -50,9 +50,7 @@ func printIndex(res model.SearchResult) {
 		mpn := elideString(value.Mpn, colWidth)
 		auth := elideString(value.Author.Name, colWidth)
 		repo := elideString(fmt.Sprintf("%v", value.FoundIn), colWidth)
-
 		_, _ = fmt.Fprintf(table, "%s\t%s\t%s\t%s\t%s\n", name, auth, man, mpn, repo)
-
 	}
 	_ = table.Flush()
 }
diff --git a/internal/app/http/handler_test.go b/internal/app/http/handler_test.go
@@ -12,6 +12,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/wot-oss/tmc/internal/app/http/mocks"
 	"github.com/wot-oss/tmc/internal/commands"
@@ -212,6 +213,7 @@ func Test_Inventory(t *testing.T) {
 		assert.Equal(t, 2, len(response.Data))
 		assertInventoryEntry(t, listResult1.Entries[0], response.Data[0])
 		assertInventoryEntry(t, listResult1.Entries[1], response.Data[1])
+		assert.Equal(t, listResult1.LastUpdated.Format(time.RFC3339), response.Meta.LastUpdated)
 		// and then result is ordered ascending by name
 		isSorted := sort.SliceIsSorted(response.Data, func(i, j int) bool {
 			return response.Data[i].TmName < response.Data[j].TmName
@@ -1416,6 +1418,7 @@ func assertAttachments(t *testing.T, linkPrefix string, expAtts []model.Attachme
 
 var (
 	listResult1 = model.SearchResult{
+		LastUpdated: time.Date(2023, time.November, 10, 12, 32, 43, 0, time.UTC),
 		Entries: []model.FoundEntry{
 			{
 				Name:         "a-corp/eagle/bt2000",
diff --git a/internal/app/http/mocks/HandlerService.go b/internal/app/http/mocks/HandlerService.go
diff --git a/internal/model/model.go b/internal/model/model.go
@@ -8,7 +8,6 @@ import (
 	"regexp"
 	"slices"
 	"strings"
-	"time"
 
 	"github.com/Masterminds/semver/v3"
 	"github.com/wot-oss/tmc/internal/utils"
@@ -233,7 +232,6 @@ type RepoDescription struct {
 	Name        string
 	Type        string
 	Description string
-	LastUpdated time.Time
 }
 
 type CheckResultType int
diff --git a/internal/model/search.go b/internal/model/search.go
@@ -24,7 +24,6 @@ var ErrSearchIndexNotFound = errors.New("search index not found. Use `tmc create
 type SearchResult struct {
 	LastUpdated time.Time
 	Entries     []FoundEntry
-	indexPath   string
 }
 
 type FoundEntry struct {
@@ -110,6 +109,10 @@ type SearchParams struct {
 	Options      SearchOptions
 }
 
+func (p *SearchParams) SetIndexPath(indexPath string) {
+	p.Options.indexPath = indexPath
+}
+
 func (p *SearchParams) Sanitize() {
 	p.Author = sanitizeList(p.Author)
 	p.Manufacturer = sanitizeList(p.Manufacturer)
@@ -124,6 +127,8 @@ type SearchOptions struct {
 	NameFilterType FilterType
 	// UseBleve indicates that the search query uses bleve query syntax
 	UseBleve bool
+
+	indexPath string
 }
 
 func (sr *SearchResult) Filter(search *SearchParams) error {
@@ -161,7 +166,7 @@ func (sr *SearchResult) Filter(search *SearchParams) error {
 		return nil
 	}
 
-	del, err := getSearchExclusionFunction(search, sr.indexPath)
+	del, err := getSearchExclusionFunction(search, search.Options.indexPath)
 	if err != nil {
 		return err
 	}
@@ -171,11 +176,6 @@ func (sr *SearchResult) Filter(search *SearchParams) error {
 	return nil
 }
 
-func (sr *SearchResult) WithSearchIndex(indexPath string) *SearchResult {
-	sr.indexPath = indexPath
-	return sr
-}
-
 func getSearchExclusionFunction(search *SearchParams, indexPath string) (func(e FoundEntry) bool, error) {
 	if search.Query == "" {
 		return nil, nil
diff --git a/internal/repos/fs_test.go b/internal/repos/fs_test.go
@@ -352,6 +352,8 @@ func TestFileRepo_Index(t *testing.T) {
 
 		idx, err := r.readIndex()
 		assert.NoError(t, err)
+		zeroTime := time.Time{}
+		assert.True(t, idx.Meta.Created.After(zeroTime))
 		assert.Equal(t, 1, len(idx.Data))
 		assert.Equal(t, "omnicorp-tm-department/omnicorp/omnilamp/subfolder", idx.Data[0].Name)
 		assert.Equal(t, 1, len(idx.Data[0].Versions))
diff --git a/internal/repos/tmc_test.go b/internal/repos/tmc_test.go
@@ -9,7 +9,9 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"reflect"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/wot-oss/tmc/internal/model"
@@ -222,6 +224,9 @@ func TestTmcRepo_List(t *testing.T) {
 			if test.expErr == "" {
 				assert.NoError(t, err)
 				assert.Equal(t, test.expRes, len(sr.Entries))
+				if reflect.DeepEqual(test.body, inventory) {
+					assert.Equal(t, time.Date(2024, 12, 1, 10, 0, 0, 0, time.UTC), sr.LastUpdated)
+				}
 				for _, e := range sr.Entries {
 					for _, v := range e.Versions {
 						assert.NotEmpty(t, v.TMID)
diff --git a/internal/repos/union.go b/internal/repos/union.go
@@ -141,16 +141,17 @@ func listRepoWithDeepSearch(ctx context.Context, r Repo, search *model.SearchPar
 		}
 	}
 
-	// refine with bleve
+	// refine the result with bleve
 	filtered := &searchResult
 	search.Query = query
 	search.Options.UseBleve = true
-	err = filtered.WithSearchIndex(indexPath).Filter(search)
+	search.SetIndexPath(indexPath)
+	err = filtered.Filter(search)
 	if err != nil {
 		return model.SearchResult{}, err
 	}
 
-	return searchResult, err
+	return *filtered, err
 }
 
 func (u *Union) GetTMMetadata(ctx context.Context, tmID string) ([]model.FoundVersion, []*RepoAccessError) {
diff --git a/internal/repos/union_test.go b/internal/repos/union_test.go
@@ -0,0 +1,151 @@
+package repos
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/wot-oss/tmc/internal/config"
+	"github.com/wot-oss/tmc/internal/model"
+	"github.com/wot-oss/tmc/internal/testutils"
+	"github.com/wot-oss/tmc/internal/utils"
+)
+
+func TestUnion_List_WithSearch(t *testing.T) {
+	t.Run("simple search", func(t *testing.T) {
+		tempDir, _ := os.MkdirTemp("", "tmc-export")
+		defer os.RemoveAll(tempDir)
+		old := config.ConfigDir
+		config.ConfigDir = filepath.Join(tempDir, "config")
+		defer func() { config.ConfigDir = old }()
+		repoRoot := filepath.Join(tempDir, "repo")
+		r := &FileRepo{
+			root: repoRoot,
+			spec: model.NewRepoSpec("repo"),
+		}
+		u := NewUnion(r)
+		err := testutils.CopyDir("../../test/data/repos/file/attachments", repoRoot)
+		assert.NoError(t, err)
+
+		sp := &model.SearchParams{
+			Query: "query",
+		}
+		lu := time.Date(2024, 5, 3, 15, 20, 26, 0, time.UTC)
+
+		res, errs := u.List(context.Background(), sp)
+
+		assert.Empty(t, errs)
+		assert.Equal(t, lu, res.LastUpdated)
+		assert.Len(t, res.Entries, 0)
+	})
+	t.Run("no bleve index", func(t *testing.T) {
+		tempDir, _ := os.MkdirTemp("", "tmc-export")
+		defer os.RemoveAll(tempDir)
+		old := config.ConfigDir
+		config.ConfigDir = filepath.Join(tempDir, "config")
+		defer func() { config.ConfigDir = old }()
+		repoRoot := filepath.Join(tempDir, "repo")
+		r := &FileRepo{
+			root: repoRoot,
+			spec: model.NewRepoSpec("repo"),
+		}
+		u := NewUnion(r)
+		err := testutils.CopyDir("../../test/data/repos/file/attachments", repoRoot)
+		assert.NoError(t, err)
+
+		sp := &model.SearchParams{
+			Query: "query",
+			Options: model.SearchOptions{
+				UseBleve: true,
+			},
+		}
+
+		_, errs := u.List(context.Background(), sp)
+		if assert.Len(t, errs, 1) {
+			assert.ErrorIs(t, errs[0], model.ErrSearchIndexNotFound)
+		}
+	})
+	t.Run("existing bleve index", func(t *testing.T) {
+		tempDir, _ := os.MkdirTemp("", "tmc-export")
+		defer os.RemoveAll(tempDir)
+		old := config.ConfigDir
+		config.ConfigDir = filepath.Join(tempDir, "config")
+		defer func() { config.ConfigDir = old }()
+		repoRoot := filepath.Join(tempDir, "repo")
+		r := &FileRepo{
+			root: repoRoot,
+			spec: model.NewRepoSpec("repo"),
+		}
+		u := NewUnion(r)
+
+		err := testutils.CopyDir("../../test/data/repos/file/attachments", repoRoot)
+		assert.NoError(t, err)
+
+		err = UpdateRepoIndex(context.Background(), r)
+		assert.NoError(t, err)
+
+		t.Run("with no match", func(t *testing.T) {
+			sp := &model.SearchParams{
+				Query: "query",
+				Options: model.SearchOptions{
+					UseBleve: true,
+				},
+			}
+
+			res, errs := u.List(context.Background(), sp)
+			assert.Len(t, errs, 0)
+			assert.Len(t, res.Entries, 0)
+		})
+		t.Run("with match", func(t *testing.T) {
+			sp := &model.SearchParams{
+				Query: "\"Lamp reaches a critical temperature\"",
+				Options: model.SearchOptions{
+					UseBleve: true,
+				},
+			}
+
+			res, errs := u.List(context.Background(), sp)
+			assert.Len(t, errs, 0)
+			assert.Len(t, res.Entries, 1)
+		})
+
+	})
+	t.Run("outdated bleve index", func(t *testing.T) {
+		tempDir, _ := os.MkdirTemp("", "tmc-export")
+		defer os.RemoveAll(tempDir)
+		old := config.ConfigDir
+		config.ConfigDir = filepath.Join(tempDir, "config")
+		defer func() { config.ConfigDir = old }()
+		repoRoot := filepath.Join(tempDir, "repo")
+		r := &FileRepo{
+			root: repoRoot,
+			spec: model.NewRepoSpec("repo"),
+		}
+		u := NewUnion(r)
+
+		err := testutils.CopyDir("../../test/data/repos/file/attachments", repoRoot)
+		assert.NoError(t, err)
+
+		indexPath := BleveIndexPath(r)
+		_ = os.MkdirAll(indexPath, defaultDirPermissions)
+		_ = utils.WriteFileLines(
+			[]string{time.Date(2024, 1, 1, 1, 1, 1, 0, time.UTC).Format(time.RFC3339)},
+			filepath.Join(indexPath, "updated"),
+			defaultFilePermissions)
+
+		sp := &model.SearchParams{
+			Query: "\"Lamp reaches a critical temperature\"",
+			Options: model.SearchOptions{
+				UseBleve: true,
+			},
+		}
+
+		res, errs := u.List(context.Background(), sp)
+		assert.Len(t, errs, 0)
+		assert.Len(t, res.Entries, 1)
+
+	})
+}
diff --git a/test/data/repos/file/attachments/.tmc/tm-catalog.toc.json b/test/data/repos/file/attachments/.tmc/tm-catalog.toc.json
@@ -1,6 +1,6 @@
 {
   "meta": {
-    "created": "2024-05-03T15:20:26.96061534+03:00"
+    "created": "2024-05-03T15:20:26Z"
   },
   "data": [
     {
diff --git a/test/data/repos/inventory_response.json b/test/data/repos/inventory_response.json
@@ -94,6 +94,6 @@
     }
   ],
   "meta": {
-    "created": "0001-01-01T00:00:00Z"
+    "lastUpdated": "2024-12-01T10:00:00Z"
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -50,9 +50,7 @@ func printIndex(res model.SearchResult) {`
`50`	`50`	`mpn := elideString(value.Mpn, colWidth)`
`51`	`51`	`auth := elideString(value.Author.Name, colWidth)`
`52`	`52`	`repo := elideString(fmt.Sprintf("%v", value.FoundIn), colWidth)`
`53`		`-`
`54`	`53`	`_, _ = fmt.Fprintf(table, "%s\t%s\t%s\t%s\t%s\n", name, auth, man, mpn, repo)`
`55`		`-`
`56`	`54`	`}`
`57`	`55`	`_ = table.Flush()`
`58`	`56`	`}`