Merge branch 'main' into config-traversal

Makefile

@@ -52,9 +52,13 @@ download-openapi-schemas:
 fmt_json:
 	ls fixtures/expectations/*.json | while read -r jf; do \
 		cat <<< $$(jq . $$jf) > $$jf; \
-	done; 
+	done;
 
 fmt_sql:
 	ls views/*.sql | while read -r sqlf; do \
 		sql-formatter -l postgresql --fix $$sqlf; \
 	done;
+
+tidy:
+	go mod tidy
+	cd hack/migrate && go mod tidy

context/context.go

@@ -7,14 +7,14 @@ import (
 	commons "github.com/flanksource/commons/context"
 	dutyGorm "github.com/flanksource/duty/gorm"
 	"github.com/flanksource/duty/models"
+	"github.com/flanksource/duty/tracing"
 	"github.com/flanksource/duty/types"
 	"github.com/flanksource/kommons"
 	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/samber/lo"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
@@ -172,12 +172,10 @@ func (k Context) WithDBLogLevel(level string) Context {
 	}
 }
 
-// FastDB returns a db suitable for high-performance usage, with limited logging
+// FastDB returns a db suitable for high-performance usage, with limited logging and tracing
 func (k Context) FastDB() *gorm.DB {
-	db := k.DB().Session(&gorm.Session{
-		NewDB: true,
-	})
-	db.Logger.LogMode(logger.Error)
+	db := k.WithAnyValue(tracing.TracePaused, true).DB()
+	db.Logger = dutyGorm.NewGormLogger("warn")
 	return db
 }
 

db.go

@@ -14,13 +14,13 @@ import (
 	"github.com/flanksource/duty/drivers"
 	dutyGorm "github.com/flanksource/duty/gorm"
 	"github.com/flanksource/duty/migrate"
+	"github.com/flanksource/duty/tracing"
 	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/jackc/pgx/v5/stdlib"
 	"github.com/spf13/pflag"
 	gormpostgres "gorm.io/driver/postgres"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
-	"gorm.io/plugin/opentelemetry/tracing"
 )
 
 var pool *pgxpool.Pool

go.mod

@@ -41,7 +41,6 @@ require (
 	go.opentelemetry.io/otel/trace v1.21.0
 	gorm.io/driver/postgres v1.5.3
 	gorm.io/gorm v1.25.5
-	gorm.io/plugin/opentelemetry v0.1.4
 	k8s.io/api v0.28.2
 	k8s.io/apimachinery v0.28.2
 	k8s.io/client-go v0.28.2

go.sum

@@ -1150,7 +1150,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
@@ -2208,11 +2207,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/postgres v1.5.3 h1:qKGY5CPHOuj47K/VxbCXJfFvIUeqMSXXadqdCY+MbBU=
 gorm.io/driver/postgres v1.5.3/go.mod h1:F+LtvlFhZT7UBiA81mC9W6Su3D4WUhSboc/36QZU0gk=
-gorm.io/driver/sqlite v1.5.0 h1:zKYbzRCpBrT1bNijRnxLDJWPjVfImGEn0lSnUY5gZ+c=
 gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
 gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
-gorm.io/plugin/opentelemetry v0.1.4 h1:7p0ocWELjSSRI7NCKPW2mVe6h43YPini99sNJcbsTuc=
-gorm.io/plugin/opentelemetry v0.1.4/go.mod h1:tndJHOdvPT0pyGhOb8E2209eXJCUxhC5UpKw7bGVWeI=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=

hack/migrate/go.mod

@@ -155,7 +155,6 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/postgres v1.5.3 // indirect
 	gorm.io/gorm v1.25.5 // indirect
-	gorm.io/plugin/opentelemetry v0.1.4 // indirect
 	k8s.io/api v0.28.2 // indirect
 	k8s.io/apiextensions-apiserver v0.24.4 // indirect
 	k8s.io/apimachinery v0.28.2 // indirect

hack/migrate/go.sum

@@ -1134,7 +1134,6 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
@@ -2172,11 +2171,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/postgres v1.5.3 h1:qKGY5CPHOuj47K/VxbCXJfFvIUeqMSXXadqdCY+MbBU=
 gorm.io/driver/postgres v1.5.3/go.mod h1:F+LtvlFhZT7UBiA81mC9W6Su3D4WUhSboc/36QZU0gk=
-gorm.io/driver/sqlite v1.5.0 h1:zKYbzRCpBrT1bNijRnxLDJWPjVfImGEn0lSnUY5gZ+c=
 gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
 gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
-gorm.io/plugin/opentelemetry v0.1.4 h1:7p0ocWELjSSRI7NCKPW2mVe6h43YPini99sNJcbsTuc=
-gorm.io/plugin/opentelemetry v0.1.4/go.mod h1:tndJHOdvPT0pyGhOb8E2209eXJCUxhC5UpKw7bGVWeI=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=

job/job.go

@@ -29,6 +29,20 @@ var RetentionHour = Retention{
 	Interval: 5 * time.Minute,
 }
 
+var RetentionFailed = Retention{
+	Success:  0,
+	Failed:   1,
+	Age:      time.Hour,
+	Interval: 15 * time.Minute,
+}
+
+var RetentionShort = Retention{
+	Success:  1,
+	Failed:   1,
+	Age:      time.Hour,
+	Interval: 5 * time.Minute,
+}
+
 var RetentionDay = Retention{
 	Success:  3,
 	Failed:   3,
@@ -98,7 +112,7 @@ func (j *JobRuntime) start() {
 		j.History.ResourceType = j.Job.ResourceType
 	}
 	if j.Job.JobHistory && j.Job.Retention.Success > 0 {
-		if err := j.History.Persist(j.DB()); err != nil {
+		if err := j.History.Persist(j.FastDB()); err != nil {
 			j.Warnf("failed to persist history: %v", err)
 		}
 	}
@@ -107,7 +121,7 @@ func (j *JobRuntime) start() {
 func (j *JobRuntime) end() {
 	j.History.End()
 	if j.Job.JobHistory && (j.Job.Retention.Success > 0 || len(j.History.Errors) > 0) {
-		if err := j.History.Persist(j.DB()); err != nil {
+		if err := j.History.Persist(j.FastDB()); err != nil {
 			j.Warnf("failed to persist history: %v", err)
 		}
 	}
@@ -166,9 +180,11 @@ func (j *Job) SetID(id string) *Job {
 
 func (j *Job) cleanupHistory() int {
 	j.Context.Logger.V(4).Infof("running cleanup: %v", j.Retention)
-	db := j.Context.WithDBLogLevel("warn").DB()
+	ctx, span := j.Context.StartSpan("CleanupHistory")
+	defer span.End()
+	db := ctx.FastDB()
 	if err := db.Exec("DELETE FROM job_history WHERE name = ? AND now() - created_at >  interval '1 minute' * ?", j.Name, j.Retention.Age.Minutes()).Error; err != nil {
-		j.Context.Warnf("failed to cleanup history %v", err)
+		ctx.Warnf("failed to cleanup history %v", err)
 	}
 	query := `WITH ordered_history AS (
       SELECT
@@ -195,10 +211,10 @@ func (j *Job) cleanupHistory() int {
 		tx := db.Exec(query, j.Name, r.statuses, r.count)
 		count += int(tx.RowsAffected)
 		if tx.Error != nil {
-			j.Context.Warnf("failed to cleanup history: %v", tx.Error)
+			ctx.Warnf("failed to cleanup history: %v", tx.Error)
 		}
 	}
-	j.Context.Logger.V(3).Infof("cleaned up %d records", count)
+	ctx.Logger.V(3).Infof("cleaned up %d records", count)
 	return count
 }
 
@@ -246,9 +262,11 @@ func (j *Job) Run() {
 	}
 
 	err := j.Fn(r)
-	ctx.Tracef("finished duration=%s, error=%s", time.Since(r.History.TimeStart), err)
 	if err != nil {
+		ctx.Tracef("finished duration=%s, error=%s", time.Since(r.History.TimeStart), err)
 		r.History.AddErrorWithSkipReportLevel(err.Error(), 1)
+	} else {
+		ctx.Tracef("finished duration=%s", time.Since(r.History.TimeStart))
 	}
 }
 
@@ -330,8 +348,12 @@ func (j *Job) init() {
 	if obj.Annotations == nil {
 		obj.Annotations = make(map[string]string)
 	}
-	obj.Annotations["debug"] = lo.Ternary(j.Debug, "true", "false")
-	obj.Annotations["trace"] = lo.Ternary(j.Trace, "true", "false")
+	if _, exists := obj.Annotations["debug"]; !exists {
+		obj.Annotations["debug"] = lo.Ternary(j.Debug, "true", "false")
+	}
+	if _, exists := obj.Annotations["trace"]; !exists {
+		obj.Annotations["trace"] = lo.Ternary(j.Trace, "true", "false")
+	}
 
 	j.Context = j.Context.WithObject(obj)
 

query/check_details.go

@@ -3,7 +3,6 @@ package query
 import (
 	"fmt"
 	"net/url"
-	"strconv"
 	"strings"
 	"time"
 
@@ -15,9 +14,6 @@ import (
 )
 
 var (
-	// Maximum number of past checks in the in-memory cache
-	DefaultCacheCount = 5
-
 	// Default search window
 	DefaultCheckQueryWindow = "1h"
 )
@@ -210,22 +206,10 @@ ORDER BY time
 }
 
 func (q CheckQueryParams) String() string {
-	return fmt.Sprintf("check:=%s, start=%s, end=%s, count=%d", q.Check, q.Start, q.End, q.StatusCount)
+	return fmt.Sprintf("check:=%s, start=%s, end=%s", q.Check, q.Start, q.End)
 }
 
 func (q *CheckQueryParams) Init(queryParams url.Values) error {
-	count := queryParams.Get("count")
-	var cacheCount int
-	if count != "" {
-		if c, err := strconv.ParseInt(count, 10, 32); err != nil {
-			return fmt.Errorf("count must be a number: %s", count)
-		} else {
-			cacheCount = int(c)
-		}
-	} else {
-		cacheCount = DefaultCacheCount
-	}
-
 	since := queryParams.Get("since")
 	if since == "" {
 		since = queryParams.Get("start")
@@ -248,7 +232,6 @@ func (q *CheckQueryParams) Init(queryParams url.Values) error {
 		IncludeMessages: isTrue(queryParams.Get("includeMessages")),
 		IncludeDetails:  isTrue(queryParams.Get("includeDetails")),
 		Check:           queryParams.Get("check"),
-		StatusCount:     int(cacheCount),
 		Trace:           isTrue(queryParams.Get("trace")),
 		CanaryID:        queryParams.Get("canary_id"),
 	}