Skip to content

Commit

Permalink
feat: add pagination and more filtering in catalog changes (#593)
Browse files Browse the repository at this point in the history 
* feat: add pagination and more filtering in catalog changes

* fix: sorting

* feat: return the config name also

* feat: added total results
  • Loading branch information
@adityathebe
adityathebe authored Mar 12, 2024
1 parent d6e51ad commit 2f36131
Show file tree
Hide file tree
Showing 3 changed files with 200 additions and 24 deletions.
8 changes: 4 additions & 4 deletions query/commons.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ func parseAndBuildFilteringQuery(query string, field string) ([]string, map[stri

in, notIN, prefixes, suffixes := ParseFilteringQuery(query)
if len(in) > 0 {
clauses = append(clauses, fmt.Sprintf("%s IN @field_in", field))
args["field_in"] = in
clauses = append(clauses, fmt.Sprintf("%s IN @%s_field_in", field, field))
args[fmt.Sprintf("%s_field_in", field)] = in
}

if len(notIN) > 0 {
clauses = append(clauses, fmt.Sprintf("%s NOT IN @field_not_in", field))
args["field_not_in"] = notIN
clauses = append(clauses, fmt.Sprintf("%s NOT IN @%s_field_not_in", field, field))
args[fmt.Sprintf("%s_field_not_in", field)] = notIN
}

for i, p := range prefixes {
Expand Down
122 changes: 108 additions & 14 deletions query/config_changes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,44 @@ const (
CatalogChangeRecursiveBoth = "both"
)

var allowedConfigChangesSortColumns = []string{"catalog_name", "change_type", "summary", "source", "created_at"}

type CatalogChangesSearchRequest struct {
CatalogID uuid.UUID `query:"id"`
ConfigType string `query:"config_type"`
ChangeType string `query:"type"`
From string `query:"from"`
CatalogID uuid.UUID `query:"id"`
ConfigType string `query:"config_type"`
ChangeType string `query:"type"`
Severity string `query:"severity"`
IncludeDeletedConfigs bool `query:"include_deleted_configs"`

// From date in datemath format
From string `query:"from"`
// To date in datemath format
To string `query:"to"`

PageSize int `query:"page_size"`
Page int `query:"page"`
SortBy string `query:"sort_by"`
sortOrder string

// upstream | downstream | both
Recursive string `query:"recursive"`

fromParsed time.Time
toParsed time.Time
}

func (t *CatalogChangesSearchRequest) SetDefaults() {
if t.PageSize <= 0 {
t.PageSize = 50
}

if t.Page <= 0 {
t.Page = 1
}

if t.From == "" && t.To == "" {
t.From = "now-2d"
}
}

func (t *CatalogChangesSearchRequest) Validate() error {
Expand All @@ -49,12 +77,41 @@ func (t *CatalogChangesSearchRequest) Validate() error {
}
}

if t.To != "" {
if expr, err := datemath.Parse(t.To); err != nil {
return fmt.Errorf("invalid 'to' param: %w", err)
} else {
t.toParsed = expr.Time()
}
}

if !t.fromParsed.IsZero() && !t.toParsed.IsZero() && !t.fromParsed.Before(t.toParsed) {
return fmt.Errorf("'from' must be before 'to'")
}

if t.SortBy != "" {
if strings.HasPrefix(t.SortBy, "-") {
t.sortOrder = "desc"
t.SortBy = strings.TrimPrefix(t.SortBy, "-")
}

if !lo.Contains(allowedConfigChangesSortColumns, t.SortBy) {
return fmt.Errorf("invalid 'sort_by' param: %s. allowed sort fields are: %s", t.SortBy, strings.Join(allowedConfigChangesSortColumns, ", "))
}
}

return nil
}

type ConfigChangeRow struct {
models.ConfigChange `json:",inline"`
CatalogName string `json:"catalog_name"`
}

type CatalogChangesSearchResponse struct {
Summary map[string]int `json:"summary,omitempty"`
Changes []models.ConfigChange `json:"changes,omitempty"`
Summary map[string]int `json:"summary,omitempty"`
Total int `json:"total,omitempty"`
Changes []ConfigChangeRow `json:"changes,omitempty"`
}

func (t *CatalogChangesSearchResponse) Summarize() {
Expand All @@ -65,25 +122,31 @@ func (t *CatalogChangesSearchResponse) Summarize() {
}

func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (*CatalogChangesSearchResponse, error) {
req.SetDefaults()
if err := req.Validate(); err != nil {
return nil, api.Errorf(api.EINVALID, "bad request: %v", err)
}

args := map[string]any{
"catalog_id": req.CatalogID,
"recursive": req.Recursive,
"catalog_id": req.CatalogID,
"recursive": req.Recursive,
"include_deleted_configs": req.IncludeDeletedConfigs,
}

var clauses []string
query := "SELECT cc.* FROM related_changes_recursive(@catalog_id, @recursive) cc"
var (
clauses []string
selectColumns = "cc.*, config_items.name as catalog_name"
from = "related_changes_recursive(@catalog_id, @recursive, @include_deleted_configs) cc"
)

if req.Recursive == "" {
query = "SELECT cc.* FROM config_changes cc"
from = "config_changes cc"
clauses = append(clauses, "cc.config_id = @catalog_id")
}

if req.ConfigType != "" {
query += " LEFT JOIN config_items ON cc.config_id = config_items.id"
from += " LEFT JOIN config_items ON cc.config_id = config_items.id"

if req.ConfigType != "" {
_clauses, _args := parseAndBuildFilteringQuery(req.ConfigType, "config_items.type")
clauses = append(clauses, _clauses...)
args = collections.MergeMap(args, _args)
Expand All @@ -95,20 +158,51 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (*
args = collections.MergeMap(args, _args)
}

if req.Severity != "" {
_clauses, _args := parseAndBuildFilteringQuery(req.Severity, "cc.severity")
clauses = append(clauses, _clauses...)
args = collections.MergeMap(args, _args)
}

if !req.fromParsed.IsZero() {
clauses = append(clauses, "cc.created_at >= @from")
clauses = append(clauses, "cc.created_at > @from")
args["from"] = req.fromParsed
}

if !req.toParsed.IsZero() {
clauses = append(clauses, "cc.created_at < @to")
args["to"] = req.toParsed
}

query := fmt.Sprintf(`SELECT %s FROM %s`, selectColumns, from)
if len(clauses) > 0 {
query += fmt.Sprintf(" WHERE %s", strings.Join(clauses, " AND "))
}

if req.SortBy != "" {
query += fmt.Sprintf(" ORDER BY %s %s", req.SortBy, req.sortOrder)
}

query += " LIMIT @page_size OFFSET @offset"
args["page_size"] = req.PageSize
args["offset"] = (req.Page - 1) * req.PageSize

var output CatalogChangesSearchResponse
if err := ctx.DB().Raw(query, args).Find(&output.Changes).Error; err != nil {
return nil, err
}

{
totalQuery := fmt.Sprintf(`SELECT count(*) FROM %s`, from)
if len(clauses) > 0 {
totalQuery += fmt.Sprintf(" WHERE %s", strings.Join(clauses, " AND "))
}

if err := ctx.DB().Raw(totalQuery, args).Find(&output.Total).Error; err != nil {
return nil, err
}
}

output.Summarize()
return &output, nil
}
94 changes: 88 additions & 6 deletions tests/config_changes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,12 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {

// Create changes for each config
var (
UChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now()), ConfigID: U.ID.String(), Summary: ".name.U", ChangeType: "RegisterNode", Source: "test-changes"}
VChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour)), ConfigID: V.ID.String(), Summary: ".name.V", ChangeType: "diff", Source: "test-changes"}
WChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 2)), ConfigID: W.ID.String(), Summary: ".name.W", ChangeType: "Pulled", Source: "test-changes"}
XChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 3)), ConfigID: X.ID.String(), Summary: ".name.X", ChangeType: "diff", Source: "test-changes"}
YChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 4)), ConfigID: Y.ID.String(), Summary: ".name.Y", ChangeType: "diff", Source: "test-changes"}
ZChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 5)), ConfigID: Z.ID.String(), Summary: ".name.Z", ChangeType: "Pulled", Source: "test-changes"}
UChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now()), Severity: "info", ConfigID: U.ID.String(), Summary: ".name.U", ChangeType: "RegisterNode", Source: "test-changes"}
VChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour)), Severity: "warn", ConfigID: V.ID.String(), Summary: ".name.V", ChangeType: "diff", Source: "test-changes"}
WChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 2)), Severity: "low", ConfigID: W.ID.String(), Summary: ".name.W", ChangeType: "Pulled", Source: "test-changes"}
XChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 3)), Severity: "info", ConfigID: X.ID.String(), Summary: ".name.X", ChangeType: "diff", Source: "test-changes"}
YChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 4)), Severity: "warn", ConfigID: Y.ID.String(), Summary: ".name.Y", ChangeType: "diff", Source: "test-changes"}
ZChange = models.ConfigChange{ID: uuid.New().String(), CreatedAt: lo.ToPtr(time.Now().Add(-time.Hour * 5)), Severity: "info", ConfigID: Z.ID.String(), Summary: ".name.Z", ChangeType: "Pulled", Source: "test-changes"}

changes = []models.ConfigChange{UChange, VChange, WChange, XChange, YChange, ZChange}
)
Expand Down Expand Up @@ -162,6 +162,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(1))
Expect(response.Total).To(Equal(1))
Expect(response.Summary[UChange.ChangeType]).To(Equal(1))
})

Expand All @@ -173,6 +174,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
ConfigType: "Kubernetes::Pod,Kubernetes::ReplicaSet",
})
Expect(err).To(BeNil())
Expect(response.Total).To(Equal(3))
Expect(len(response.Changes)).To(Equal(3))
Expect(response.Summary["Pulled"]).To(Equal(2))
Expect(response.Summary["diff"]).To(Equal(1))
Expand All @@ -185,6 +187,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
ConfigType: "!Kubernetes::ReplicaSet",
})
Expect(err).To(BeNil())
Expect(response.Total).To(Equal(5))
Expect(len(response.Changes)).To(Equal(5))
Expect(response.Summary["diff"]).To(Equal(2))
Expect(response.Summary["Pulled"]).To(Equal(2))
Expand All @@ -200,6 +203,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
ChangeType: "diff",
})
Expect(err).To(BeNil())
Expect(response.Total).To(Equal(2))
Expect(len(response.Changes)).To(Equal(2))
Expect(response.Summary["diff"]).To(Equal(2))
})
Expand All @@ -211,11 +215,56 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
ChangeType: "!diff,!Pulled",
})
Expect(err).To(BeNil())
Expect(response.Total).To(Equal(1))
Expect(len(response.Changes)).To(Equal(1))
Expect(response.Summary["RegisterNode"]).To(Equal(1))
})
})

ginkgo.It("Severity filter", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
Severity: "!info",
})
Expect(err).To(BeNil())
Expect(response.Total).To(Equal(3))
Expect(len(response.Changes)).To(Equal(3))
Expect(response.Summary["Pulled"]).To(Equal(1))
Expect(response.Summary["diff"]).To(Equal(2))
})

ginkgo.Context("Pagination", func() {
ginkgo.It("Page size", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
SortBy: "summary",
PageSize: 2,
})
Expect(err).To(BeNil())
Expect(response.Total).To(Equal(6))
Expect(len(response.Changes)).To(Equal(2))
changes := lo.Map(response.Changes, func(c query.ConfigChangeRow, _ int) string { return c.Summary })
Expect(changes).To(Equal([]string{".name.U", ".name.V"}))
})

ginkgo.It("Page number", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
SortBy: "summary",
PageSize: 2,
Page: 2,
})
Expect(err).To(BeNil())
Expect(response.Total).To(Equal(6))
Expect(len(response.Changes)).To(Equal(2))
changes := lo.Map(response.Changes, func(c query.ConfigChangeRow, _ int) string { return c.Summary })
Expect(changes).To(Equal([]string{".name.W", ".name.X"}))
})
})

ginkgo.Context("recursive mode", func() {
ginkgo.It("upstream", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
Expand All @@ -224,6 +273,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(2))
Expect(response.Total).To(Equal(2))
Expect(response.Summary[UChange.ChangeType]).To(Equal(1))
Expect(response.Summary[WChange.ChangeType]).To(Equal(1))
})
Expand All @@ -235,6 +285,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(4))
Expect(response.Total).To(Equal(4))
Expect(response.Summary["diff"]).To(Equal(3))
Expect(response.Summary["Pulled"]).To(Equal(1))
})
Expand All @@ -246,6 +297,7 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(5))
Expect(response.Total).To(Equal(5))
Expect(response.Summary["diff"]).To(Equal(3))
Expect(response.Summary["Pulled"]).To(Equal(1))
Expect(response.Summary["RegisterNode"]).To(Equal(1))
Expand All @@ -258,12 +310,42 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() {
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
From: "now-65m",
To: "now-1s",
})
Expect(err).To(BeNil())
Expect(response.Total).To(Equal(2))
Expect(len(response.Changes)).To(Equal(2))
Expect(response.Summary["diff"]).To(Equal(1))
Expect(response.Summary["RegisterNode"]).To(Equal(1))
})
})

ginkgo.Context("Sorting", func() {
ginkgo.It("Descending", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
SortBy: "-catalog_name",
})
Expect(err).To(BeNil())
Expect(len(response.Changes)).To(Equal(6))
Expect(response.Total).To(Equal(6))
changes := lo.Map(response.Changes, func(c query.ConfigChangeRow, _ int) string { return c.CatalogName })
Expect(changes).To(Equal([]string{"Z", "Y", "X", "W", "V", "U"}))
})

ginkgo.It("Ascending", func() {
response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{
CatalogID: U.ID,
Recursive: query.CatalogChangeRecursiveDownstream,
SortBy: "catalog_name",
})
Expect(err).To(BeNil())
Expect(response.Total).To(Equal(6))
Expect(len(response.Changes)).To(Equal(6))
changes := lo.Map(response.Changes, func(c query.ConfigChangeRow, _ int) string { return c.CatalogName })
Expect(changes).To(Equal([]string{"U", "V", "W", "X", "Y", "Z"}))
})
})
})
})

0 comments on commit 2f36131

Please sign in to comment.