diff --git a/query/commons.go b/query/commons.go index 6bfd2b79..87370f78 100644 --- a/query/commons.go +++ b/query/commons.go @@ -33,13 +33,13 @@ func parseAndBuildFilteringQuery(query string, field string) ([]string, map[stri in, notIN, prefixes, suffixes := ParseFilteringQuery(query) if len(in) > 0 { - clauses = append(clauses, fmt.Sprintf("%s IN @field_in", field)) - args["field_in"] = in + clauses = append(clauses, fmt.Sprintf("%s IN @%s_field_in", field, field)) + args[fmt.Sprintf("%s_field_in", field)] = in } if len(notIN) > 0 { - clauses = append(clauses, fmt.Sprintf("%s NOT IN @field_not_in", field)) - args["field_not_in"] = notIN + clauses = append(clauses, fmt.Sprintf("%s NOT IN @%s_field_not_in", field, field)) + args[fmt.Sprintf("%s_field_not_in", field)] = notIN } for i, p := range prefixes { diff --git a/query/config_changes.go b/query/config_changes.go index abaf203b..4b7e02e7 100644 --- a/query/config_changes.go +++ b/query/config_changes.go @@ -20,16 +20,44 @@ const ( CatalogChangeRecursiveBoth = "both" ) +var allowedConfigChangesSortColumns = []string{"catalog_name", "change_type", "summary", "source", "created_at"} + type CatalogChangesSearchRequest struct { - CatalogID uuid.UUID `query:"id"` - ConfigType string `query:"config_type"` - ChangeType string `query:"type"` - From string `query:"from"` + CatalogID uuid.UUID `query:"id"` + ConfigType string `query:"config_type"` + ChangeType string `query:"type"` + Severity string `query:"severity"` + IncludeDeletedConfigs bool `query:"include_deleted_configs"` + + // From date in datemath format + From string `query:"from"` + // To date in datemath format + To string `query:"to"` + + PageSize int `query:"page_size"` + Page int `query:"page"` + SortBy string `query:"sort_by"` + sortOrder string // upstream | downstream | both Recursive string `query:"recursive"` fromParsed time.Time + toParsed time.Time +} + +func (t *CatalogChangesSearchRequest) SetDefaults() { + if t.PageSize <= 0 { + t.PageSize = 50 + } + + if t.Page <= 0 { + t.Page = 1 + } + + if t.From == "" && t.To == "" { + t.From = "now-2d" + } } func (t *CatalogChangesSearchRequest) Validate() error { @@ -49,12 +77,41 @@ func (t *CatalogChangesSearchRequest) Validate() error { } } + if t.To != "" { + if expr, err := datemath.Parse(t.To); err != nil { + return fmt.Errorf("invalid 'to' param: %w", err) + } else { + t.toParsed = expr.Time() + } + } + + if !t.fromParsed.IsZero() && !t.toParsed.IsZero() && !t.fromParsed.Before(t.toParsed) { + return fmt.Errorf("'from' must be before 'to'") + } + + if t.SortBy != "" { + if strings.HasPrefix(t.SortBy, "-") { + t.sortOrder = "desc" + t.SortBy = strings.TrimPrefix(t.SortBy, "-") + } + + if !lo.Contains(allowedConfigChangesSortColumns, t.SortBy) { + return fmt.Errorf("invalid 'sort_by' param: %s. allowed sort fields are: %s", t.SortBy, strings.Join(allowedConfigChangesSortColumns, ", ")) + } + } + return nil } +type ConfigChangeRow struct { + models.ConfigChange `json:",inline"` + CatalogName string `json:"catalog_name"` +} + type CatalogChangesSearchResponse struct { - Summary map[string]int `json:"summary,omitempty"` - Changes []models.ConfigChange `json:"changes,omitempty"` + Summary map[string]int `json:"summary,omitempty"` + Total int `json:"total,omitempty"` + Changes []ConfigChangeRow `json:"changes,omitempty"` } func (t *CatalogChangesSearchResponse) Summarize() { @@ -65,25 +122,31 @@ func (t *CatalogChangesSearchResponse) Summarize() { } func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (*CatalogChangesSearchResponse, error) { + req.SetDefaults() if err := req.Validate(); err != nil { return nil, api.Errorf(api.EINVALID, "bad request: %v", err) } args := map[string]any{ - "catalog_id": req.CatalogID, - "recursive": req.Recursive, + "catalog_id": req.CatalogID, + "recursive": req.Recursive, + "include_deleted_configs": req.IncludeDeletedConfigs, } - var clauses []string - query := "SELECT cc.* FROM related_changes_recursive(@catalog_id, @recursive) cc" + var ( + clauses []string + selectColumns = "cc.*, config_items.name as catalog_name" + from = "related_changes_recursive(@catalog_id, @recursive, @include_deleted_configs) cc" + ) + if req.Recursive == "" { - query = "SELECT cc.* FROM config_changes cc" + from = "config_changes cc" clauses = append(clauses, "cc.config_id = @catalog_id") } - if req.ConfigType != "" { - query += " LEFT JOIN config_items ON cc.config_id = config_items.id" + from += " LEFT JOIN config_items ON cc.config_id = config_items.id" + if req.ConfigType != "" { _clauses, _args := parseAndBuildFilteringQuery(req.ConfigType, "config_items.type") clauses = append(clauses, _clauses...) args = collections.MergeMap(args, _args) @@ -95,20 +158,51 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (* args = collections.MergeMap(args, _args) } + if req.Severity != "" { + _clauses, _args := parseAndBuildFilteringQuery(req.Severity, "cc.severity") + clauses = append(clauses, _clauses...) + args = collections.MergeMap(args, _args) + } + if !req.fromParsed.IsZero() { - clauses = append(clauses, "cc.created_at >= @from") + clauses = append(clauses, "cc.created_at > @from") args["from"] = req.fromParsed } + if !req.toParsed.IsZero() { + clauses = append(clauses, "cc.created_at < @to") + args["to"] = req.toParsed + } + + query := fmt.Sprintf(`SELECT %s FROM %s`, selectColumns, from) if len(clauses) > 0 { query += fmt.Sprintf(" WHERE %s", strings.Join(clauses, " AND ")) } + if req.SortBy != "" { + query += fmt.Sprintf(" ORDER BY %s %s", req.SortBy, req.sortOrder) + } + + query += " LIMIT @page_size OFFSET @offset" + args["page_size"] = req.PageSize + args["offset"] = (req.Page - 1) * req.PageSize + var output CatalogChangesSearchResponse if err := ctx.DB().Raw(query, args).Find(&output.Changes).Error; err != nil { return nil, err } + { + totalQuery := fmt.Sprintf(`SELECT count(*) FROM %s`, from) + if len(clauses) > 0 { + totalQuery += fmt.Sprintf(" WHERE %s", strings.Join(clauses, " AND ")) + } + + if err := ctx.DB().Raw(totalQuery, args).Find(&output.Total).Error; err != nil { + return nil, err + } + } + output.Summarize() return &output, nil } diff --git a/tests/config_changes_test.go b/tests/config_changes_test.go index ef2d1274..a73faac3 100644 --- a/tests/config_changes_test.go +++ b/tests/config_changes_test.go @@ -44,12 +44,12 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { // Create changes for each config var ( - UChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now()), ConfigID: U.ID.String(), Summary: ".name.U", ChangeType: "RegisterNode", Source: "test-changes"} - VChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour)), ConfigID: V.ID.String(), Summary: ".name.V", ChangeType: "diff", Source: "test-changes"} - WChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 2)), ConfigID: W.ID.String(), Summary: ".name.W", ChangeType: "Pulled", Source: "test-changes"} - XChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 3)), ConfigID: X.ID.String(), Summary: ".name.X", ChangeType: "diff", Source: "test-changes"} - YChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 4)), ConfigID: Y.ID.String(), Summary: ".name.Y", ChangeType: "diff", Source: "test-changes"} - ZChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 5)), ConfigID: Z.ID.String(), Summary: ".name.Z", ChangeType: "Pulled", Source: "test-changes"} + UChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now()), Severity: "info", ConfigID: U.ID.String(), Summary: ".name.U", ChangeType: "RegisterNode", Source: "test-changes"} + VChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour)), Severity: "warn", ConfigID: V.ID.String(), Summary: ".name.V", ChangeType: "diff", Source: "test-changes"} + WChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 2)), Severity: "low", ConfigID: W.ID.String(), Summary: ".name.W", ChangeType: "Pulled", Source: "test-changes"} + XChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 3)), Severity: "info", ConfigID: X.ID.String(), Summary: ".name.X", ChangeType: "diff", Source: "test-changes"} + YChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 4)), Severity: "warn", ConfigID: Y.ID.String(), Summary: ".name.Y", ChangeType: "diff", Source: "test-changes"} + ZChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 5)), Severity: "info", ConfigID: Z.ID.String(), Summary: ".name.Z", ChangeType: "Pulled", Source: "test-changes"} changes = []models.ConfigChange{UChange, VChange, WChange, XChange, YChange, ZChange} ) @@ -162,6 +162,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(1)) + Expect(response.Total).To(Equal(1)) Expect(response.Summary[UChange.ChangeType]).To(Equal(1)) }) @@ -173,6 +174,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ConfigType: "Kubernetes::Pod,Kubernetes::ReplicaSet", }) Expect(err).To(BeNil()) + Expect(response.Total).To(Equal(3)) Expect(len(response.Changes)).To(Equal(3)) Expect(response.Summary["Pulled"]).To(Equal(2)) Expect(response.Summary["diff"]).To(Equal(1)) @@ -185,6 +187,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ConfigType: "!Kubernetes::ReplicaSet", }) Expect(err).To(BeNil()) + Expect(response.Total).To(Equal(5)) Expect(len(response.Changes)).To(Equal(5)) Expect(response.Summary["diff"]).To(Equal(2)) Expect(response.Summary["Pulled"]).To(Equal(2)) @@ -200,6 +203,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ChangeType: "diff", }) Expect(err).To(BeNil()) + Expect(response.Total).To(Equal(2)) Expect(len(response.Changes)).To(Equal(2)) Expect(response.Summary["diff"]).To(Equal(2)) }) @@ -211,11 +215,56 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ChangeType: "!diff,!Pulled", }) Expect(err).To(BeNil()) + Expect(response.Total).To(Equal(1)) Expect(len(response.Changes)).To(Equal(1)) Expect(response.Summary["RegisterNode"]).To(Equal(1)) }) }) + ginkgo.It("Severity filter", func() { + response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ + CatalogID: U.ID, + Recursive: query.CatalogChangeRecursiveDownstream, + Severity: "!info", + }) + Expect(err).To(BeNil()) + Expect(response.Total).To(Equal(3)) + Expect(len(response.Changes)).To(Equal(3)) + Expect(response.Summary["Pulled"]).To(Equal(1)) + Expect(response.Summary["diff"]).To(Equal(2)) + }) + + ginkgo.Context("Pagination", func() { + ginkgo.It("Page size", func() { + response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ + CatalogID: U.ID, + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "summary", + PageSize: 2, + }) + Expect(err).To(BeNil()) + Expect(response.Total).To(Equal(6)) + Expect(len(response.Changes)).To(Equal(2)) + changes := lo.Map(response.Changes, func(c query.ConfigChangeRow, _ int) string { return c.Summary }) + Expect(changes).To(Equal([]string{".name.U", ".name.V"})) + }) + + ginkgo.It("Page number", func() { + response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ + CatalogID: U.ID, + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "summary", + PageSize: 2, + Page: 2, + }) + Expect(err).To(BeNil()) + Expect(response.Total).To(Equal(6)) + Expect(len(response.Changes)).To(Equal(2)) + changes := lo.Map(response.Changes, func(c query.ConfigChangeRow, _ int) string { return c.Summary }) + Expect(changes).To(Equal([]string{".name.W", ".name.X"})) + }) + }) + ginkgo.Context("recursive mode", func() { ginkgo.It("upstream", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ @@ -224,6 +273,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(2)) + Expect(response.Total).To(Equal(2)) Expect(response.Summary[UChange.ChangeType]).To(Equal(1)) Expect(response.Summary[WChange.ChangeType]).To(Equal(1)) }) @@ -235,6 +285,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(4)) + Expect(response.Total).To(Equal(4)) Expect(response.Summary["diff"]).To(Equal(3)) Expect(response.Summary["Pulled"]).To(Equal(1)) }) @@ -246,6 +297,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(5)) + Expect(response.Total).To(Equal(5)) Expect(response.Summary["diff"]).To(Equal(3)) Expect(response.Summary["Pulled"]).To(Equal(1)) Expect(response.Summary["RegisterNode"]).To(Equal(1)) @@ -258,12 +310,42 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { CatalogID: U.ID, Recursive: query.CatalogChangeRecursiveDownstream, From: "now-65m", + To: "now-1s", }) Expect(err).To(BeNil()) + Expect(response.Total).To(Equal(2)) Expect(len(response.Changes)).To(Equal(2)) Expect(response.Summary["diff"]).To(Equal(1)) Expect(response.Summary["RegisterNode"]).To(Equal(1)) }) }) + + ginkgo.Context("Sorting", func() { + ginkgo.It("Descending", func() { + response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ + CatalogID: U.ID, + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "-catalog_name", + }) + Expect(err).To(BeNil()) + Expect(len(response.Changes)).To(Equal(6)) + Expect(response.Total).To(Equal(6)) + changes := lo.Map(response.Changes, func(c query.ConfigChangeRow, _ int) string { return c.CatalogName }) + Expect(changes).To(Equal([]string{"Z", "Y", "X", "W", "V", "U"})) + }) + + ginkgo.It("Ascending", func() { + response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ + CatalogID: U.ID, + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "catalog_name", + }) + Expect(err).To(BeNil()) + Expect(response.Total).To(Equal(6)) + Expect(len(response.Changes)).To(Equal(6)) + changes := lo.Map(response.Changes, func(c query.ConfigChangeRow, _ int) string { return c.CatalogName }) + Expect(changes).To(Equal([]string{"U", "V", "W", "X", "Y", "Z"})) + }) + }) }) })