Merge branch 'master' into dependabot/go_modules/go.opentelemetry.io/…

…otel/trace-1.29.0

api/v1/checks.go

@@ -972,6 +972,16 @@ type KubernetesResourceCheck struct {
 	WaitFor KubernetesResourceCheckWaitFor `json:"waitFor,omitempty"`
 }
 
+func (c KubernetesResourceCheck) GetDisplayTemplate() Template {
+	if !c.Templatable.Display.IsEmpty() {
+		return c.Templatable.Display
+	}
+
+	return Template{
+		Expression: "display.keys().map(k, k + ': ' + display[k]).join('\n')",
+	}
+}
+
 func (c KubernetesResourceCheck) TotalResources() int {
 	return len(c.Resources) + len(c.StaticResources)
 }

build/full/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-bookworm@sha256:6d71b7c3f884e7b9552bffa852d938315ecca843dcc75a86ee7000567da0923d AS builder
+FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-bookworm@sha256:f020456572fc292e9627b3fb435c6de5dfb8020fbcef1fd7b65dd092c0ac56bb AS builder
 WORKDIR /app
 
 ARG TARGETOS

build/minimal/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-bookworm@sha256:6d71b7c3f884e7b9552bffa852d938315ecca843dcc75a86ee7000567da0923d AS builder
+FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-bookworm@sha256:f020456572fc292e9627b3fb435c6de5dfb8020fbcef1fd7b65dd092c0ac56bb AS builder
 WORKDIR /app
 
 ARG TARGETOS

canary-checker.properties

@@ -5,3 +5,4 @@
 # check.disabled.tcp=false
 
 # topology.runNow=true
+log.level.db=warn

checks/aws_config.go

@@ -4,9 +4,7 @@ package checks
 
 import (
 	"github.com/aws/aws-sdk-go-v2/service/configservice"
-	awsUtil "github.com/flanksource/artifacts/clients/aws"
 	"github.com/flanksource/canary-checker/api/context"
-	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
 	"github.com/flanksource/duty/connection"
@@ -30,8 +28,7 @@ func (c *AwsConfigChecker) Type() string {
 	return "awsconfig"
 }
 
-func (c *AwsConfigChecker) Check(ctx *context.Context, extConfig external.Check) pkg.Results {
-	check := extConfig.(v1.AwsConfigCheck)
+func (c *AwsConfigChecker) Check(ctx *context.Context, check v1.AwsConfigCheck) pkg.Results {
 	result := pkg.Success(check, ctx.Canary)
 	var results pkg.Results
 	results = append(results, result)
@@ -44,12 +41,17 @@ func (c *AwsConfigChecker) Check(ctx *context.Context, extConfig external.Check)
 		}
 	}
 
-	cfg, err := awsUtil.NewSession(ctx.Context, *check.AWSConnection)
+	cfg, err := check.AWSConnection.Client(ctx.Context)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
 
-	client := configservice.NewFromConfig(*cfg)
+	client := configservice.NewFromConfig(cfg, func(o *configservice.Options) {
+		if check.AWSConnection.Endpoint != "" {
+			o.BaseEndpoint = &check.AWSConnection.Endpoint
+		}
+	})
+
 	if check.AggregatorName != nil {
 		output, err := client.SelectAggregateResourceConfig(ctx, &configservice.SelectAggregateResourceConfigInput{
 			ConfigurationAggregatorName: check.AggregatorName,

checks/aws_config_rule.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/configservice"
 	"github.com/aws/aws-sdk-go-v2/service/configservice/types"
-	awsUtil "github.com/flanksource/artifacts/clients/aws"
 	"github.com/flanksource/canary-checker/api/context"
 	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
@@ -45,15 +44,16 @@ func (c *AwsConfigRuleChecker) Check(ctx *context.Context, extConfig external.Ch
 		return results.Failf("failed to populate aws connection: %v", err)
 	}
 
-	cfg, err := awsUtil.NewSession(ctx.Context, *check.AWSConnection)
+	cfg, err := check.AWSConnection.Client(ctx.Context)
 	if err != nil {
 		return results.Failf("failed to create a session: %v", err)
 	}
 
-	client := configservice.NewFromConfig(*cfg)
-	if err != nil {
-		return results.Failf("failed to describe compliance rules: %v", err)
-	}
+	client := configservice.NewFromConfig(cfg, func(o *configservice.Options) {
+		if check.AWSConnection.Endpoint != "" {
+			o.BaseEndpoint = &check.AWSConnection.Endpoint
+		}
+	})
 
 	var complianceTypes = []types.ComplianceType{}
 	for _, i := range check.ComplianceTypes {

checks/cloudwatch.go

@@ -7,9 +7,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
-	awsUtil "github.com/flanksource/artifacts/clients/aws"
 	"github.com/flanksource/canary-checker/api/context"
-	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
 )
@@ -32,8 +30,7 @@ func (c *CloudWatchChecker) Type() string {
 	return "cloudwatch"
 }
 
-func (c *CloudWatchChecker) Check(ctx *context.Context, extConfig external.Check) pkg.Results {
-	check := extConfig.(v1.CloudWatchCheck)
+func (c *CloudWatchChecker) Check(ctx *context.Context, check v1.CloudWatchCheck) pkg.Results {
 	result := pkg.Success(check, ctx.Canary)
 	var results pkg.Results
 	results = append(results, result)
@@ -42,11 +39,17 @@ func (c *CloudWatchChecker) Check(ctx *context.Context, extConfig external.Check
 		return results.Failf("failed to populate aws connection: %v", err)
 	}
 
-	cfg, err := awsUtil.NewSession(ctx.Context, check.AWSConnection)
+	cfg, err := check.AWSConnection.Client(ctx.Context)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
-	client := cloudwatch.NewFromConfig(*cfg)
+
+	client := cloudwatch.NewFromConfig(cfg, func(o *cloudwatch.Options) {
+		if check.AWSConnection.Endpoint != "" {
+			o.BaseEndpoint = &check.AWSConnection.Endpoint
+		}
+	})
+
 	maxRecords := int32(100)
 	alarms, err := client.DescribeAlarms(ctx, &cloudwatch.DescribeAlarmsInput{
 		AlarmNames:      check.CloudWatchFilter.Alarms,

checks/folder.go

@@ -6,9 +6,10 @@ import (
 	"os"
 	"strings"
 
+	"github.com/flanksource/artifacts"
+	artifactFS "github.com/flanksource/artifacts/fs"
 	"github.com/prometheus/client_golang/prometheus"
 
-	"github.com/flanksource/artifacts"
 	"github.com/flanksource/canary-checker/api/context"
 	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
@@ -111,18 +112,18 @@ func checkLocalFolder(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 	return results
 }
 
-func genericFolderCheck(dirFS artifacts.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
+func genericFolderCheck(dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
 	return _genericFolderCheck(true, dirFS, path, recursive, filter)
 }
 
 // genericFolderCheckWithoutPrecheck is used for those filesystems that do not support fetching the stat of a directory.
 // Eg: s3, gcs.
 // It will not pre check whether the given path is a directory.
-func genericFolderCheckWithoutPrecheck(dirFS artifacts.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
+func genericFolderCheckWithoutPrecheck(dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
 	return _genericFolderCheck(false, dirFS, path, recursive, filter)
 }
 
-func _genericFolderCheck(supportsDirStat bool, dirFS artifacts.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
+func _genericFolderCheck(supportsDirStat bool, dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
 	result := FolderCheck{}
 	_filter, err := filter.New()
 	if err != nil {
@@ -170,7 +171,7 @@ func _genericFolderCheck(supportsDirStat bool, dirFS artifacts.Filesystem, path
 
 // getFolderContents walks the folder and returns all files.
 // Also supports recursively fetching contents
-func getFolderContents(dirFs artifacts.Filesystem, path string, filter *v1.FolderFilterContext) ([]fs.FileInfo, error) {
+func getFolderContents(dirFs artifactFS.Filesystem, path string, filter *v1.FolderFilterContext) ([]fs.FileInfo, error) {
 	files, err := dirFs.ReadDir(path)
 	if err != nil {
 		return nil, err

checks/folder_s3.go

@@ -8,10 +8,10 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/flanksource/artifacts"
+	artifactFS "github.com/flanksource/artifacts/fs"
 	"github.com/flanksource/canary-checker/api/context"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
-	"github.com/flanksource/duty/models"
 )
 
 type S3 struct {
@@ -31,26 +31,22 @@ func CheckS3Bucket(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 	var bucket string
 	bucket, check.Path = parseS3Path(check.Path)
 
-	connection, err := ctx.HydrateConnectionByURL(check.AWSConnection.ConnectionName)
-	if err != nil {
-		return results.Failf("failed to populate AWS connection: %v", err)
-	} else if connection == nil {
-		connection = &models.Connection{Type: models.ConnectionTypeS3}
-		if check.S3Connection.Bucket == "" {
-			check.S3Connection.Bucket = bucket
-		}
-
-		connection, err = connection.Merge(ctx, check.S3Connection)
-		if err != nil {
-			return results.Failf("failed to populate AWS connection: %v", err)
-		}
+	if err := check.S3Connection.Populate(ctx); err != nil {
+		return results.ErrorMessage(err)
 	}
 
-	fs, err := artifacts.GetFSForConnection(ctx.Context, *connection)
+	conn := check.S3Connection.ToModel()
+	conn.SetProperty("bucket", bucket)
+
+	fs, err := artifacts.GetFSForConnection(ctx.Context, conn)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
 
+	if limitFS, ok := fs.(artifactFS.ListItemLimiter); ok {
+		limitFS.SetMaxListItems(ctx.Properties().Int("s3.list.max-objects", 50_000))
+	}
+
 	folders, err := genericFolderCheckWithoutPrecheck(fs, check.Path, check.Recursive, check.Filter)
 	if err != nil {
 		return results.ErrorMessage(err)

checks/folder_sftp.go

@@ -1,11 +1,7 @@
 package checks
 
 import (
-	"fmt"
-
 	"github.com/flanksource/artifacts"
-	"github.com/flanksource/artifacts/clients/sftp"
-
 	"github.com/flanksource/canary-checker/api/context"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
@@ -16,27 +12,16 @@ func CheckSFTP(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 	var results pkg.Results
 	results = append(results, result)
 
-	foundConn, err := check.SFTPConnection.HydrateConnection(ctx)
-	if err != nil {
+	if err := check.SFTPConnection.HydrateConnection(ctx); err != nil {
 		return results.Failf("failed to populate SFTP connection: %v", err)
 	}
 
-	auth := check.SFTPConnection.Authentication
-	if !foundConn {
-		auth, err = ctx.GetAuthValues(check.SFTPConnection.Authentication)
-		if err != nil {
-			return results.ErrorMessage(err)
-		}
-	}
-
-	client, err := sftp.SSHConnect(fmt.Sprintf("%s:%d", check.SFTPConnection.Host, check.SFTPConnection.GetPort()), auth.GetUsername(), auth.GetPassword())
+	fs, err := artifacts.GetFSForConnection(ctx.Context, check.SFTPConnection.ToModel())
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
-	defer client.Close()
 
-	session := artifacts.Filesystem(client)
-	folders, err := genericFolderCheck(session, check.Path, check.Recursive, check.Filter)
+	folders, err := genericFolderCheck(fs, check.Path, check.Recursive, check.Filter)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}

checks/folder_smb.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/flanksource/artifacts/clients/smb"
+	"github.com/flanksource/artifacts"
 	"github.com/flanksource/canary-checker/api/context"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
@@ -15,34 +15,30 @@ func CheckSmb(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 	var results pkg.Results
 	results = append(results, result)
 
-	var serverPath = strings.TrimPrefix(check.Path, "smb://")
-	server, sharename, path, err := extractServerDetails(serverPath)
+	serverPath := strings.TrimPrefix(check.Path, "smb://")
+	server, share, path, err := extractServerDetails(serverPath)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
 
-	foundConn, err := check.SMBConnection.HydrateConnection(ctx)
-	if err != nil {
+	if err := check.SMBConnection.Populate(ctx); err != nil {
 		return results.Failf("failed to populate SMB connection: %v", err)
 	}
 
-	auth := check.SMBConnection.Authentication
-	if !foundConn {
-		auth, err = ctx.GetAuthValues(check.SMBConnection.Authentication)
-		if err != nil {
-			return results.ErrorMessage(err)
-		}
+	if server != "" {
+		check.SMBConnection.Domain = server
+	}
+
+	if share != "" {
+		check.SMBConnection.Share = share
 	}
 
-	session, err := smb.SMBConnect(server, fmt.Sprintf("%d", check.SMBConnection.GetPort()), sharename, auth)
+	fs, err := artifacts.GetFSForConnection(ctx.Context, check.SMBConnection.ToModel())
 	if err != nil {
 		return results.ErrorMessage(err)
 	}
-	if session != nil {
-		defer session.Close()
-	}
 
-	folders, err := genericFolderCheck(session, path, check.Recursive, check.Filter)
+	folders, err := genericFolderCheck(fs, path, check.Recursive, check.Filter)
 	if err != nil {
 		return results.ErrorMessage(err)
 	}

checks/http.go

@@ -14,7 +14,6 @@ import (
 	"github.com/flanksource/commons/http/middlewares"
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty/models"
-	"github.com/flanksource/gomplate/v3"
 
 	"github.com/flanksource/canary-checker/api/external"
 	"github.com/prometheus/client_golang/prometheus"
@@ -231,7 +230,7 @@ func (c *HTTPChecker) Check(ctx *context.Context, extConfig external.Check) pkg.
 
 	body := check.Body
 	if check.TemplateBody {
-		body, err = ctx.RunTemplate(gomplate.Template{Template: body}, ctx.Environment)
+		body, err = template(ctx, v1.Template{Template: body})
 		if err != nil {
 			return results.WithError(oops.Wrap(err)).Invalidf("failed to template request body: %v", err)
 		}
@@ -252,7 +251,7 @@ func (c *HTTPChecker) Check(ctx *context.Context, extConfig external.Check) pkg.
 
 	start := time.Now()
 
-	ctx.Infof("%s	%s", console.Greenf(check.GetMethod()), check.URL)
+	ctx.Tracef("%s	%s", console.Greenf(check.GetMethod()), check.URL)
 
 	response, err := request.Do(check.GetMethod(), check.URL)
 	if err != nil {

checks/kubernetes_resource.go

@@ -109,6 +109,8 @@ func (c *KubernetesResourceChecker) Check(ctx context.Context, check v1.Kubernet
 	}
 
 	ctx.Logger.V(4).Infof("found %d checks to run", len(check.Checks))
+
+	displayPerCheck := map[string]string{}
 	for _, c := range check.Checks {
 		virtualCanary := v1.Canary{
 			ObjectMeta: ctx.Canary.ObjectMeta,
@@ -167,13 +169,19 @@ func (c *KubernetesResourceChecker) Check(ctx context.Context, check v1.Kubernet
 				}
 			}
 
+			for _, r := range res {
+				displayPerCheck[r.Check.GetName()] = r.Message
+			}
 			return nil
 		})
 		if retryErr != nil {
 			return results.Failf(retryErr.Error())
 		}
 	}
 
+	result.AddData(map[string]any{
+		"display": displayPerCheck,
+	})
 	return results
 }