feat: Support looking up values into headers and body

api/v1/checks.go

@@ -55,6 +55,11 @@ func (c Check) GetLabels() map[string]string {
 	return c.Labels
 }
 
+type Oauth2Config struct {
+	Scopes   []string `json:"scope,omitempty" yaml:"scope,omitempty"`
+	TokenURL string   `json:"tokenURL,omitempty" yaml:"tokenURL,omitempty"`
+}
+
 type HTTPCheck struct {
 	Description `yaml:",inline" json:",inline"`
 	Templatable `yaml:",inline" json:",inline"`
@@ -85,6 +90,10 @@ type HTTPCheck struct {
 	Headers []types.EnvVar `yaml:"headers,omitempty" json:"headers,omitempty"`
 	//Template the request body
 	TemplateBody bool `yaml:"templateBody,omitempty" json:"templateBody,omitempty"`
+	// EnvVars are the environment variables that are accesible to templated body
+	EnvVars []types.EnvVar `yaml:"env,omitempty" json:"env,omitempty"`
+	// Oauth2 Configuration. The client ID & Client secret should go to username & password respectively.
+	Oauth2 *Oauth2Config `yaml:"oauth2,omitempty" json:"oauth2,omitempty"`
 }
 
 func (c HTTPCheck) GetType() string {

checks/http.go

@@ -1,22 +1,23 @@
 package checks
 
 import (
+	"fmt"
 	"net/url"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/PaesslerAG/jsonpath"
 	"github.com/flanksource/canary-checker/api/context"
-	"github.com/flanksource/commons/text"
+	"github.com/flanksource/commons/http"
 	"github.com/flanksource/duty/models"
-	"github.com/pkg/errors"
+	gomplate "github.com/flanksource/gomplate/v3"
 
 	"github.com/flanksource/canary-checker/api/external"
 	"github.com/prometheus/client_golang/prometheus"
 
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
-	"github.com/flanksource/canary-checker/pkg/http"
 	"github.com/flanksource/canary-checker/pkg/metrics"
 	"github.com/flanksource/canary-checker/pkg/utils"
 )
@@ -61,28 +62,39 @@ func (c *HTTPChecker) Run(ctx *context.Context) pkg.Results {
 	return results
 }
 
-func (c *HTTPChecker) configure(req *http.HTTPRequest, ctx *context.Context, check v1.HTTPCheck, connection *models.Connection) error {
+func (c *HTTPChecker) generateHTTPRequest(ctx *context.Context, check v1.HTTPCheck, connection *models.Connection) (*http.Request, error) {
+	client := http.NewClient()
+
 	for _, header := range check.Headers {
 		value, err := ctx.GetEnvValueFromCache(header)
 		if err != nil {
-			return errors.WithMessagef(err, "failed getting header: %v", header)
+			return nil, fmt.Errorf("failed getting header (%v): %w", header, err)
 		}
-		req.Header(header.Name, value)
+
+		client.Header(header.Name, value)
 	}
 
 	if connection.Username != "" || connection.Password != "" {
-		req.Auth(connection.Username, connection.Password)
+		client.Auth(connection.Username, connection.Password)
+	}
+
+	if check.Oauth2 != nil {
+		client.OAuth(connection.Username, connection.Password, check.Oauth2.TokenURL, check.Oauth2.Scopes...)
 	}
 
-	req.NTLM(check.NTLM)
-	req.NTLMv2(check.NTLMv2)
+	client.NTLM(check.NTLM)
+	client.NTLMV2(check.NTLMv2)
 
 	if check.ThresholdMillis > 0 {
-		req.Timeout(time.Duration(check.ThresholdMillis) * time.Millisecond)
+		client.Timeout(time.Duration(check.ThresholdMillis) * time.Millisecond)
 	}
 
-	req.Trace(ctx.IsTrace()).Debug(ctx.IsDebug())
-	return nil
+	// TODO: Add finer controls over tracing to the canary
+	if ctx.IsTrace() {
+		client.Trace(http.TraceConfig{MaxBodyLength: 512, Body: true, Headers: true, ResponseHeaders: true})
+	}
+
+	return client.R(ctx), nil
 }
 
 func truncate(text string, max int) string {
@@ -129,25 +141,46 @@ func (c *HTTPChecker) Check(ctx *context.Context, extConfig external.Check) pkg.
 		return results.Failf("failed to parse url: %v", err)
 	}
 
+	templateEnv := map[string]any{
+		"canary": ctx.Canary,
+	}
+	for _, env := range check.EnvVars {
+		if val, err := ctx.GetEnvValueFromCache(env); err != nil {
+			return results.Failf("failed to get env value: %v", err)
+		} else {
+			templateEnv[env.Name] = val
+		}
+	}
+
 	body := check.Body
 	if check.TemplateBody {
-		body, err = text.Template(body, ctx.Canary)
+		body, err = gomplate.RunTemplate(templateEnv, gomplate.Template{Template: body})
 		if err != nil {
 			return results.ErrorMessage(err)
 		}
 	}
 
-	req := http.NewRequest(connection.URL).Method(check.GetMethod())
-
-	if err := c.configure(req, ctx, check, connection); err != nil {
+	request, err := c.generateHTTPRequest(ctx, check, connection)
+	if err != nil {
 		return results.ErrorMessage(err)
 	}
 
+	if body != "" {
+		if err := request.Body(body); err != nil {
+			return results.ErrorMessage(err)
+		}
+	}
+
 	start := time.Now()
 
-	resp := req.Do(body)
+	response, err := request.Do(check.GetMethod(), connection.URL)
+	if err != nil {
+		return results.ErrorMessage(err)
+	}
+
 	elapsed := time.Since(start)
-	status := resp.GetStatusCode()
+	status := response.StatusCode
+
 	result.AddMetric(pkg.Metric{
 		Name: "response_code",
 		Type: metrics.CounterType,
@@ -156,30 +189,29 @@ func (c *HTTPChecker) Check(ctx *context.Context, extConfig external.Check) pkg.
 			"url":  check.URL,
 		},
 	})
+
 	result.Duration = elapsed.Milliseconds()
 	responseStatus.WithLabelValues(strconv.Itoa(status), statusCodeToClass(status), check.URL).Inc()
-	age := resp.GetSSLAge()
+	age := response.GetSSLAge()
 	if age != nil {
 		sslExpiration.WithLabelValues(check.URL).Set(age.Hours() * 24)
 	}
 
-	body, _ = resp.AsString()
-
 	data := map[string]interface{}{
 		"code":    status,
-		"headers": resp.GetHeaders(),
+		"headers": response.Header,
 		"elapsed": time.Since(start),
-		"content": body,
 		"sslAge":  utils.Deref(age),
 		"json":    make(map[string]any),
 	}
 
-	if resp.IsJSON() {
-		json, err := resp.AsJSON()
+	if response.IsJSON() {
+		json, err := response.AsJSON()
+		data["json"] = json
 		if err == nil {
-			data["json"] = json.Value
+			data["json"] = json
 			if check.ResponseJSONContent != nil && check.ResponseJSONContent.Path != "" {
-				err := resp.CheckJSONContent(json.Value, check.ResponseJSONContent)
+				err := checkJSONContent(json, check.ResponseJSONContent)
 				if err != nil {
 					return results.ErrorMessage(err)
 				}
@@ -189,15 +221,14 @@ func (c *HTTPChecker) Check(ctx *context.Context, extConfig external.Check) pkg.
 		} else {
 			ctx.Tracef("ignoring invalid json response %v", err)
 		}
+	} else {
+		responseBody, _ := response.AsString()
+		data["content"] = responseBody
 	}
 
 	result.AddData(data)
 
-	if status == -1 {
-		return results.Failf("%v", truncate(resp.Error.Error(), 500))
-	}
-
-	if ok := resp.IsOK(check.ResponseCodes...); !ok {
+	if ok := response.IsOK(check.ResponseCodes...); !ok {
 		return results.Failf("response code invalid %d != %v", status, check.ResponseCodes)
 	}
 
@@ -209,14 +240,15 @@ func (c *HTTPChecker) Check(ctx *context.Context, extConfig external.Check) pkg.
 		return results.Failf("expected %v, found %v", check.ResponseContent, truncate(body, 100))
 	}
 
-	if req.URL.Scheme == "https" && check.MaxSSLExpiry > 0 {
+	if check.MaxSSLExpiry > 0 {
 		if age == nil {
 			return results.Failf("No certificate found to check age")
 		}
 		if *age < time.Duration(check.MaxSSLExpiry)*time.Hour*24 {
 			return results.Failf("SSL certificate expires soon %s > %d", utils.Age(*age), check.MaxSSLExpiry)
 		}
 	}
+
 	return results
 }
 
@@ -235,3 +267,29 @@ func statusCodeToClass(statusCode int) string {
 		return "unknown"
 	}
 }
+
+func checkJSONContent(jsonContent map[string]any, jsonCheck *v1.JSONCheck) error {
+	if jsonCheck == nil {
+		return nil
+	}
+
+	jsonResult, err := jsonpath.Get(jsonCheck.Path, jsonContent)
+	if err != nil {
+		return fmt.Errorf("error getting jsonPath: %w", err)
+	}
+
+	switch s := jsonResult.(type) {
+	case string:
+		if s != jsonCheck.Value {
+			return fmt.Errorf("%v not equal to %v", s, jsonCheck.Value)
+		}
+	case fmt.Stringer:
+		if s.String() != jsonCheck.Value {
+			return fmt.Errorf("%v not equal to %v", s.String(), jsonCheck.Value)
+		}
+	default:
+		return fmt.Errorf("json response could not be parsed back to string")
+	}
+
+	return nil
+}

config/deploy/crd.yaml

@@ -3048,6 +3048,37 @@ spec:
                       endpoint:
                         description: 'Deprecated: Use url instead'
                         type: string
+                      env:
+                        description: EnvVars are the environment variables that are accesible to templated body
+                        items:
+                          properties:
+                            name:
+                              type: string
+                            value:
+                              type: string
+                            valueFrom:
+                              properties:
+                                configMapKeyRef:
+                                  properties:
+                                    key:
+                                      type: string
+                                    name:
+                                      type: string
+                                  required:
+                                    - key
+                                  type: object
+                                secretKeyRef:
+                                  properties:
+                                    key:
+                                      type: string
+                                    name:
+                                      type: string
+                                  required:
+                                    - key
+                                  type: object
+                              type: object
+                          type: object
+                        type: array
                       headers:
                         description: Header fields to be used in the query
                         items:
@@ -3130,6 +3161,16 @@ spec:
                       ntlmv2:
                         description: NTLM when set to true will do authentication using NTLM v2 protocol
                         type: boolean
+                      oauth2:
+                        description: Oauth2 Configuration. The client ID & Client secret should go to username & password respectively.
+                        properties:
+                          scope:
+                            items:
+                              type: string
+                            type: array
+                          tokenURL:
+                            type: string
+                        type: object
                       password:
                         properties:
                           name: