Run --raw-output (#4023)

Re earthly/earthly#3713

There are two ways to solve this, one is adding RawOutput bool to
DeltaLog and wiring it through. That would change the logstream api.

This is the other solution, which is simple but hacky. We just check for
`--raw-output` on the command before printing. @alexcb suggested
starting here.

ast/commandflag/flags.go

@@ -38,6 +38,7 @@ type RunOpts struct {
 	Secrets         []string `long:"secret" description:"Make available a secret"`
 	Mounts          []string `long:"mount" description:"Mount a file or directory"`
 	Network         string   `long:"network" description:"Network to use; currently network=none is only supported"`
+	RawOutput       bool     `long:"raw-output" description:"Do not prefix output with target. Print Raw"`
 }
 
 type FromOpts struct {

earthfile2llb/converter.go

@@ -645,6 +645,7 @@ type ConvertRunOpts struct {
 	InteractiveKeep      bool
 	InteractiveSaveFiles []debuggercommon.SaveFilesSettings
 	WithAWSCredentials   bool
+	RawOutput            bool
 
 	// Internal.
 	shellWrap    shellWrapFun
@@ -2166,8 +2167,9 @@ func (c *Converter) internalRun(ctx context.Context, opts ConvertRunOpts) (pllb.
 
 	runOpts = append(runOpts, mountRunOpts...)
 	commandStr := fmt.Sprintf(
-		"%s %s%s%s%s%s%s%s%s",
+		"%s %s%s%s%s%s%s%s%s%s",
 		opts.CommandName, // e.g. "RUN", "IF", "FOR", "ARG"
+		strIf(opts.RawOutput, "--raw-output "),
 		strIf(opts.Privileged, "--privileged "),
 		strIf(opts.Push, "--push "),
 		strIf(opts.WithSSH, "--ssh "),
@@ -2176,7 +2178,6 @@ func (c *Converter) internalRun(ctx context.Context, opts ConvertRunOpts) (pllb.
 		strIf(opts.Interactive, "--interactive "),
 		strIf(opts.InteractiveKeep, "--interactive-keep "),
 		strings.Join(opts.Args, " "))
-
 	prefix, _, err := c.newVertexMeta(ctx, opts.Locally, isInteractive, false, opts.Secrets)
 	if err != nil {
 		return pllb.State{}, err
@@ -2745,7 +2746,6 @@ func (c *Converter) newVertexMeta(ctx context.Context, local, interactive, inter
 		Internal:            internal,
 		Runner:              c.opt.Runner,
 	}
-
 	return vm.ToVertexPrefix(), cmdID, nil
 }
 

earthfile2llb/interpreter.go

@@ -730,6 +730,10 @@ func (i *Interpreter) handleRun(ctx context.Context, cmd spec.Command) error {
 		return i.errorf(cmd.SourceLocation, "RUN --aws requires the --run-with-aws feature flag")
 	}
 
+	if opts.RawOutput && !i.converter.opt.Features.RawOutput {
+		return i.errorf(cmd.SourceLocation, "RUN --raw-output requires the --raw-output feature flag")
+	}
+
 	if i.withDocker == nil {
 		if opts.WithDocker {
 			return i.errorf(cmd.SourceLocation, "--with-docker is obsolete. Please use WITH DOCKER ... RUN ... END instead")
@@ -751,6 +755,7 @@ func (i *Interpreter) handleRun(ctx context.Context, cmd spec.Command) error {
 			InteractiveKeep:      opts.InteractiveKeep,
 			InteractiveSaveFiles: i.interactiveSaveFiles,
 			WithAWSCredentials:   opts.WithAWS,
+			RawOutput:            opts.RawOutput,
 		}
 		err = i.converter.Run(ctx, opts)
 		if err != nil {

features/features.go

@@ -72,6 +72,7 @@ type Features struct {
 	AllowPrivilegedFromDockerfile bool `long:"allow-privileged-from-dockerfile" description:"Allow the use of the --allow-privileged flag in the FROM DOCKERFILE command"`
 	RunWithAWS                    bool `long:"run-with-aws" description:"make AWS credentials in the environment or ~/.aws available to RUN commands"`
 	WildcardCopy                  bool `long:"wildcard-copy" description:"allow for the expansion of wildcard (glob) paths for COPY commands"`
+	RawOutput                     bool `long:"raw-output" description:"allow for --raw-output on RUN commands"`
 	GitAuthorEmailNameArgs        bool `long:"git-author-email-name-args" description:"includes EARTHLY_GIT_AUTHOR_EMAIL and EARTHLY_GIT_AUTHOR_NAME builtin ARGs"`
 
 	Major int

logbus/formatter/formatter.go

@@ -261,7 +261,18 @@ func (f *Formatter) getCommand(commandID string) *command {
 }
 
 func (f *Formatter) handleDeltaLog(dl *logstream.DeltaLog) error {
-	c, verboseOnly := f.targetConsole(dl.GetTargetId(), dl.GetCommandId())
+	commandID := dl.GetCommandId()
+	targetID := dl.GetTargetId()
+
+	//lookup --raw-output from the command manifest
+	cm := f.manifest.GetCommands()[dl.GetCommandId()]
+	rawOutput := false
+	if cm != nil {
+		//commandStr building order in converter.internalRun means
+		// --raw-output is always first after run
+		rawOutput = strings.Contains(cm.Name, "RUN --raw-output")
+	}
+	c, verboseOnly := f.targetConsole(targetID, commandID, rawOutput)
 	if verboseOnly && !f.verbose {
 		return nil
 	}
@@ -340,7 +351,7 @@ func (f *Formatter) processOngoingTick(ctx context.Context) error {
 }
 
 func (f *Formatter) printHeader(targetID string, commandID string, tm *logstream.TargetManifest, cm *logstream.CommandManifest, failure bool) {
-	c, verboseOnly := f.targetConsole(targetID, commandID)
+	c, verboseOnly := f.targetConsole(targetID, commandID, false)
 	if verboseOnly && !f.verbose {
 		return
 	}
@@ -371,7 +382,7 @@ func (f *Formatter) printHeader(targetID string, commandID string, tm *logstream
 }
 
 func (f *Formatter) printProgress(targetID string, commandID string, cm *logstream.CommandManifest) {
-	c, verboseOnly := f.targetConsole(targetID, commandID)
+	c, verboseOnly := f.targetConsole(targetID, commandID, false)
 	if verboseOnly && !f.verbose {
 		return
 	}
@@ -416,7 +427,7 @@ func (f *Formatter) shouldPrintProgress(targetID string, commandID string, cm *l
 }
 
 func (f *Formatter) printError(targetID string, commandID string, tm *logstream.TargetManifest, cm *logstream.CommandManifest) {
-	c, _ := f.targetConsole(targetID, commandID)
+	c, _ := f.targetConsole(targetID, commandID, false)
 	c.Printf("%s\n", cm.GetErrorMessage())
 	c.VerbosePrintf("Overriding args used: %s\n", strings.Join(tm.GetOverrideArgs(), " "))
 	f.lastOutputWasOngoingUpdate = false
@@ -437,7 +448,7 @@ func (f *Formatter) printBuildFailure() {
 	if failure.GetCommandId() != "" {
 		cm = f.manifest.GetCommands()[failure.GetCommandId()]
 	}
-	c, _ := f.targetConsole(failure.GetTargetId(), failure.GetCommandId())
+	c, _ := f.targetConsole(failure.GetTargetId(), failure.GetCommandId(), false)
 	c = c.WithFailed(true)
 	msgPrefix := "Error: " // print this prefix only when the command id is set
 	if failure.GetCommandId() != "" && failure.GetCommandId() != logbus.GenericDefault {
@@ -471,7 +482,7 @@ func (f *Formatter) commandName(commandID string) string {
 	return "unknown"
 }
 
-func (f *Formatter) targetConsole(targetID string, commandID string) (conslogging.ConsoleLogger, bool) {
+func (f *Formatter) targetConsole(targetID string, commandID string, rawOutput bool) (conslogging.ConsoleLogger, bool) {
 	var targetName string
 	var writerTargetID string
 	verboseOnly := false
@@ -517,6 +528,11 @@ func (f *Formatter) targetConsole(targetID string, commandID string) (consloggin
 		targetName = "_unknown"
 		writerTargetID = "_unknown"
 	}
+	if rawOutput {
+		return f.console.
+			WithWriter(f.bus.FormattedWriter(writerTargetID, commandID)), verboseOnly
+	}
+
 	return f.console.
 		WithWriter(f.bus.FormattedWriter(writerTargetID, commandID)).
 		WithPrefixAndSalt(targetName, writerTargetID), verboseOnly

tests/Earthfile

@@ -201,6 +201,7 @@ ga-no-qemu-group8:
     BUILD --pass-args ./pass-args-default-via-function+test
     BUILD --pass-args ./pass-args-global-via-function+test
     BUILD --pass-args ./pass-args-via-function-with-override+test
+    BUILD --pass-args ./raw-output+test-all
 
 ga-no-qemu-group9:
     BUILD --pass-args ./autoskip+test-group1

tests/raw-output/Earthfile

@@ -0,0 +1,12 @@
+VERSION 0.8 
+FROM --pass-args ..+base
+
+IMPORT .. AS tests
+
+WORKDIR /test
+
+test-all:
+    DO tests+RUN_EARTHLY --earthfile=manifest.earth --target=+gha
+    RUN acbgrep '^::group::' earthly.output
+    RUN acbgrep -v '^should have prefix' earthly.output
+    RUN acbgrep '^::endgroup::' earthly.output

tests/raw-output/manifest.earth

@@ -0,0 +1,8 @@
+VERSION --raw-output 0.8
+
+FROM alpine
+
+gha: 
+    RUN --no-cache --raw-output echo "::group:: build something group 1"
+    RUN echo "should have prefix"
+    RUN --no-cache --raw-output echo "::endgroup::"