feat(cmd): log test errors

cmd/substation/main.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/google/go-jsonnet"
@@ -17,6 +19,10 @@ var rootCmd = &cobra.Command{
 	Long: "'substation' is a tool for managing Substation configurations.",
 }
 
+// transformRe captures the transform ID from a Substation error message.
+// Example: `transform 324f1035-10a51b9a: object_target_key: missing required option` -> `324f1035-10a51b9a`
+var transformRe = regexp.MustCompile(`transform ([a-f0-9-]+):`)
+
 const (
 	// confStdout is the default configuration used by
 	// read-like commands. It prints any results (<= 100MB)
@@ -156,6 +162,45 @@ func pathVars(p string) (string, string) {
 	return dir, fn
 }
 
+// transformErrStr returns a formatted string for transform errors.
+//
+// If the error is not a transform error, then the error message
+// is returned as is.
+func transformErrStr(err error, arg string, cfg customConfig) string {
+	r := transformRe.FindStringSubmatch(err.Error())
+
+	// Cannot determine which transform failed. This should almost
+	// never happen, unless something has modified the configuration
+	// after it was compiled by Jsonnet.
+	if len(r) == 0 {
+		// Substation uses the transform name as a static transform ID.
+		//
+		// Example: `vet.json: transform hash_sha256: object_target_key: missing required option``
+		return fmt.Sprintf("%s: %v\n", arg, err)
+	}
+
+	tfID := r[1] // The transform ID (e.g., `324f1035-10a51b9a`).
+
+	// Prioritize returning test errors.
+	for _, test := range cfg.Tests {
+		for idx, tf := range test.Transforms {
+			if tf.Settings["id"] == tfID {
+				// Example: `vet.json:3 transform 324f1035-10a51b9a: object_target_key: missing required option``
+				return fmt.Sprintf("%s:%d %v\n", arg, idx+1, err) + fmt.Sprintf("        %s\n\n", tf) // The line number is 1-based.
+			}
+		}
+	}
+
+	for idx, tf := range cfg.Config.Transforms {
+		if tf.Settings["id"] == tfID {
+			return fmt.Sprintf("%s:%d %v\n", arg, idx+1, err) + fmt.Sprintf("        %s\n\n", tf)
+		}
+	}
+
+	// This happens if the input is not a transform error.
+	return fmt.Sprintf("%s: %v\n", arg, err)
+}
+
 func main() {
 	err := rootCmd.Execute()
 	if err != nil {

cmd/substation/playground.go

@@ -175,14 +175,16 @@ func handleTest(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	var output strings.Builder
 
+	arg := "config.jsonnet" // Used for consistency with the test and vet commands.
+
 	if len(cfg.Transforms) == 0 {
-		output.WriteString("?\t[config error]\n")
+		output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
 		sendJSONResponse(w, map[string]string{"output": output.String()})
 		return
 	}
 
 	if len(cfg.Tests) == 0 {
-		output.WriteString("?\t[no tests]\n")
+		output.WriteString(fmt.Sprintf("?\t%s\t[no tests]\n", arg))
 		sendJSONResponse(w, map[string]string{"output": output.String()})
 		return
 	}
@@ -193,7 +195,9 @@ func handleTest(w http.ResponseWriter, r *http.Request) {
 	for _, test := range cfg.Tests {
 		cnd, err := condition.New(ctx, test.Condition)
 		if err != nil {
-			output.WriteString("?\t[test error]\n")
+			output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
+			output.WriteString(transformErrStr(err, fmt.Sprintf("%s:tests:%s:condition", arg, test.Name), cfg))
+
 			sendJSONResponse(w, map[string]string{"output": output.String()})
 			return
 		}
@@ -202,28 +206,44 @@ func handleTest(w http.ResponseWriter, r *http.Request) {
 			Transforms: test.Transforms,
 		})
 		if err != nil {
-			output.WriteString("?\t[test error]\n")
+			output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
+			if len(transformRe.FindStringSubmatch(err.Error())) == 0 {
+				output.WriteString(transformErrStr(err, fmt.Sprintf("%s:tests:%s", arg, test.Name), cfg))
+			} else {
+				output.WriteString(transformErrStr(err, fmt.Sprintf("%s:tests:%s:transforms", arg, test.Name), cfg))
+			}
+
 			sendJSONResponse(w, map[string]string{"output": output.String()})
 			return
 		}
 
 		tester, err := substation.New(ctx, cfg.Config)
 		if err != nil {
-			output.WriteString("?\t[config error]\n")
+			output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
+			if len(transformRe.FindStringSubmatch(err.Error())) == 0 {
+				output.WriteString(transformErrStr(err, arg, cfg))
+			} else {
+				output.WriteString(transformErrStr(err, fmt.Sprintf("%s:transforms", arg), cfg))
+			}
+
 			sendJSONResponse(w, map[string]string{"output": output.String()})
 			return
 		}
 
 		sMsgs, err := setup.Transform(ctx, message.New().AsControl())
 		if err != nil {
-			output.WriteString("?\t[test error]\n")
+			output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
+			output.WriteString(transformErrStr(err, fmt.Sprintf("%s:tests:%s:transforms", arg, test.Name), cfg))
+
 			sendJSONResponse(w, map[string]string{"output": output.String()})
 			return
 		}
 
 		tMsgs, err := tester.Transform(ctx, sMsgs...)
 		if err != nil {
-			output.WriteString("?\t[config error]\n")
+			output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
+			output.WriteString(transformErrStr(err, fmt.Sprintf("%s:transforms", arg), cfg))
+
 			sendJSONResponse(w, map[string]string{"output": output.String()})
 			return
 		}
@@ -236,7 +256,9 @@ func handleTest(w http.ResponseWriter, r *http.Request) {
 
 			ok, err := cnd.Condition(ctx, msg)
 			if err != nil {
-				output.WriteString("?\t[test error]\n")
+				output.WriteString(fmt.Sprintf("?\t%s\t[error]\n", arg))
+				output.WriteString(transformErrStr(err, fmt.Sprintf("%s:tests:%s:condition", arg, test.Name), cfg))
+
 				sendJSONResponse(w, map[string]string{"output": output.String()})
 				return
 			}

cmd/substation/test.go

@@ -17,15 +17,16 @@ import (
 	"github.com/brexhq/substation/v2/message"
 )
 
+type testConfig struct {
+	Name       string          `json:"name"`
+	Transforms []config.Config `json:"transforms"`
+	Condition  config.Config   `json:"condition"`
+}
+
 // customConfig wraps the Substation config with support for tests.
 type customConfig struct {
 	substation.Config
-
-	Tests []struct {
-		Name       string          `json:"name"`
-		Transforms []config.Config `json:"transforms"`
-		Condition  config.Config   `json:"condition"`
-	} `json:"tests"`
+	Tests []testConfig `json:"tests"`
 }
 
 func init() {
@@ -197,9 +198,9 @@ func testFile(arg string, extVars map[string]string) error {
 		// If the Jsonnet cannot compile, then the file is invalid.
 		mem, err := compileFile(arg, extVars)
 		if err != nil {
-			fmt.Printf("?\t%s\t[config error]\n", arg)
+			fmt.Printf("?\t%s\t[error]\n", arg)
+			fmt.Fprint(os.Stderr, transformErrStr(err, arg, cfg))
 
-			//nolint:nilerr  // errors should not disrupt the test.
 			return nil
 		}
 
@@ -220,7 +221,7 @@ func testFile(arg string, extVars map[string]string) error {
 
 	start := time.Now()
 
-	// These configurations are not valid.
+	// These are not valid Substation configs and must be ignored.
 	if len(cfg.Transforms) == 0 {
 		return nil
 	}
@@ -238,9 +239,9 @@ func testFile(arg string, extVars map[string]string) error {
 		// cnd asserts that the test is successful.
 		cnd, err := condition.New(ctx, test.Condition)
 		if err != nil {
-			fmt.Printf("FAIL\t%s\t[test error]\n", arg)
+			fmt.Printf("?\t%s\t[error]\n", arg)
+			fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:tests:%s:condition", arg, test.Name), cfg))
 
-			//nolint:nilerr  // errors should not disrupt the test.
 			return nil
 		}
 
@@ -249,9 +250,13 @@ func testFile(arg string, extVars map[string]string) error {
 			Transforms: test.Transforms,
 		})
 		if err != nil {
-			fmt.Printf("?\t%s\t[test error]\n", arg)
+			fmt.Printf("?\t%s\t[error]\n", arg)
+			if len(transformRe.FindStringSubmatch(err.Error())) == 0 {
+				fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:tests:%s", arg, test.Name), cfg))
+			} else {
+				fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:tests:%s:transforms", arg, test.Name), cfg))
+			}
 
-			//nolint:nilerr  // errors should not disrupt the test.
 			return nil
 		}
 
@@ -260,25 +265,29 @@ func testFile(arg string, extVars map[string]string) error {
 		// that there is no state shared between tests.
 		tester, err := substation.New(ctx, cfg.Config)
 		if err != nil {
-			fmt.Printf("?\t%s\t[config error]\n", arg)
+			fmt.Printf("?\t%s\t[error]\n", arg)
+			if len(transformRe.FindStringSubmatch(err.Error())) == 0 {
+				fmt.Fprint(os.Stderr, transformErrStr(err, arg, cfg))
+			} else {
+				fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:transforms", arg), cfg))
+			}
 
-			//nolint:nilerr  // errors should not disrupt the test.
 			return nil
 		}
 
 		sMsgs, err := setup.Transform(ctx, message.New().AsControl())
 		if err != nil {
-			fmt.Printf("?\t%s\t[test error]\n", arg)
+			fmt.Printf("?\t%s\t[error]\n", arg)
+			fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:tests:%s:transforms", arg, test.Name), cfg))
 
-			//nolint:nilerr  // errors should not disrupt the test.
 			return nil
 		}
 
 		tMsgs, err := tester.Transform(ctx, sMsgs...)
 		if err != nil {
-			fmt.Printf("?\t%s\t[config error]\n", arg)
+			fmt.Printf("?\t%s\t[error]\n", arg)
+			fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:transforms", arg), cfg))
 
-			//nolint:nilerr  // errors should not disrupt the test.
 			return nil
 		}
 
@@ -290,9 +299,9 @@ func testFile(arg string, extVars map[string]string) error {
 
 			ok, err := cnd.Condition(ctx, msg)
 			if err != nil {
-				fmt.Printf("?\t%s\t[test error]\n", arg)
+				fmt.Printf("?\t%s\t[error]\n", arg)
+				fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:tests:%s:condition", arg, test.Name), cfg))
 
-				//nolint:nilerr  // errors should not disrupt the test.
 				return nil
 			}
 

cmd/substation/vet.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
 
 	"github.com/spf13/cobra"
 
@@ -20,10 +19,6 @@ func init() {
 	vetCmd.PersistentFlags().SortFlags = false
 }
 
-// vetTransformRe captures the transform ID from a Substation error message.
-// Example: `transform 324f1035-10a51b9a: object_target_key: missing required option` -> `324f1035-10a51b9a`
-var vetTransformRe = regexp.MustCompile(`transform ([a-f0-9-]+):`)
-
 var vetCmd = &cobra.Command{
 	Use:   "vet [path]",
 	Short: "report config errors",
@@ -127,30 +122,13 @@ func vetFile(arg string, extVars map[string]string) error {
 
 	ctx := context.Background() // This doesn't need to be canceled.
 	if _, err := substation.New(ctx, cfg.Config); err != nil {
-		r := vetTransformRe.FindStringSubmatch(err.Error())
-
-		// Cannot determine which transform failed. This should almost
-		// never happen, unless something has modified the configuration
-		// after it was compiled by Jsonnet.
-		if len(r) == 0 {
-			// Substation uses the transform name as a static transform ID.
-			//
-			// Example: `vet.json: transform hash_sha256: object_target_key: missing required option``
-			fmt.Printf("%s: %v\n", arg, err)
-
-			return nil
+		if len(transformRe.FindStringSubmatch(err.Error())) == 0 {
+			fmt.Fprint(os.Stderr, transformErrStr(err, arg, cfg))
+		} else {
+			fmt.Fprint(os.Stderr, transformErrStr(err, fmt.Sprintf("%s:transforms", arg), cfg))
 		}
 
-		tfID := r[1] // The transform ID (e.g., `324f1035-10a51b9a`).
-		for idx, tf := range cfg.Config.Transforms {
-			if tf.Settings["id"] == tfID {
-				// Example: `vet.jsonnet:3 transform 324f1035-10a51b9a: object_target_key: missing required option``
-				fmt.Printf("%s:%d %v\n", arg, idx+1, err) // The line number is 1-based.
-				fmt.Printf("\n    %s\n\n", tf)
-
-				return nil
-			}
-		}
+		return nil
 	}
 
 	// No errors were found.