cmd/catalyst: boot from a temporary mist config file

Makefile

@@ -214,6 +214,8 @@ box-dev: scripts
 	-v $$(realpath config):/etc/livepeer:ro \
 	-v $$(realpath ./coredumps):$$(realpath ./coredumps) \
 	-e CORE_DUMP_DIR=$$(realpath ./coredumps) \
+	-e CATALYST_CONFIG=/etc/livepeer/full-stack.yaml \
+	-e CATALYST_DOWNLOAD=false \
 	$(shell for line in $$(cat .env 2>/dev/null || echo ''); do printf -- "-e $$line "; done) \
 	--rm \
 	-it \

cmd/catalyst/catalyst.go

@@ -1,17 +1,15 @@
 package main
 
 import (
-	"encoding/json"
-	"fmt"
 	"os"
+	"os/exec"
 	"syscall"
 
-	"github.com/icza/dyno"
-	"github.com/livepeer/catalyst/cmd/downloader/cli"
+	"github.com/livepeer/catalyst/cmd/catalyst/cli"
+	"github.com/livepeer/catalyst/cmd/catalyst/config"
 	"github.com/livepeer/catalyst/cmd/downloader/downloader"
 	"github.com/livepeer/catalyst/cmd/downloader/types"
 	glog "github.com/magicsong/color-glog"
-	"gopkg.in/yaml.v3"
 )
 
 var Version = "undefined"
@@ -25,27 +23,41 @@ func main() {
 		glog.Fatalf("error parsing cli flags: %s", err)
 		return
 	}
-	err = downloader.Run(cliFlags)
+	if cliFlags.Download {
+		err = downloader.Run(cliFlags)
+		if err != nil {
+			glog.Fatalf("error running downloader: %s", err)
+		}
+	}
+	err = execNext(cliFlags)
 	if err != nil {
-		glog.Fatalf("error running downloader: %s", err)
+		glog.Fatalf("error executing MistController: %s", err)
 	}
-	execNext(cliFlags)
 }
 
 // Done! Move on to the provided next application, if it exists.
-func execNext(cliFlags types.CliFlags) {
-	if len(cliFlags.ExecCommand) == 0 {
-		// Nothing to do.
-		return
+func execNext(cliFlags types.CliFlags) error {
+	jsonBytes, err := config.HandleConfigStack(cliFlags.ConfigStack)
+	if err != nil {
+		return err
+	}
+	f, err := os.CreateTemp("", "catalyst-generated-*.json")
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(jsonBytes)
+	if err != nil {
+		return err
 	}
-	configStr, err := handleConfigFile("/home/iameli/code/catalyst/config/full-stack.yaml")
+	glog.Infof("downloader complete, now we will exec %v", cliFlags.MistController)
+	binary, err := exec.LookPath(cliFlags.MistController)
 	if err != nil {
-		panic(err)
+		return err
 	}
-	panic(configStr)
-	glog.Infof("downloader complete, now we will exec %v", cliFlags.ExecCommand)
-	execErr := syscall.Exec(cliFlags.ExecCommand[0], cliFlags.ExecCommand, os.Environ())
+	args := []string{binary, "-c", f.Name()}
+	execErr := syscall.Exec(binary, args, os.Environ())
 	if execErr != nil {
 		glog.Fatalf("error running next command: %s", execErr)
 	}
+	return nil
 }

cmd/catalyst/cli/cli.go

@@ -0,0 +1,48 @@
+package cli
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	downloaderCli "github.com/livepeer/catalyst/cmd/downloader/cli"
+	"github.com/livepeer/catalyst/cmd/downloader/constants"
+	"github.com/livepeer/catalyst/cmd/downloader/types"
+	"github.com/peterbourgon/ff/v3"
+)
+
+// GetCliFlags reads command-line arguments and generates a struct
+// with useful values set after parsing the same.
+func GetCliFlags(buildFlags types.BuildFlags) (types.CliFlags, error) {
+	cliFlags := types.CliFlags{}
+	flag.Set("logtostderr", "true")
+	vFlag := flag.Lookup("v")
+	fs := flag.NewFlagSet(constants.AppName, flag.ExitOnError)
+
+	downloaderCli.AddDownloaderFlags(fs, &cliFlags)
+
+	fs.StringVar(&cliFlags.MistController, "mist-controller", "MistController", "Path to MistController binary to exec when done")
+	fs.StringVar(&cliFlags.ConfigStack, "config", "/etc/livepeer/catalyst.yaml", "Path to multiple Catalyst config files to use. Can contain multiple entries e.g. /conf1:/conf2")
+
+	version := fs.Bool("version", false, "Get version information")
+
+	if *version {
+		fmt.Printf("catalyst version: %s\n", buildFlags.Version)
+		os.Exit(0)
+	}
+
+	ff.Parse(
+		fs, os.Args[1:],
+		ff.WithConfigFileParser(ff.PlainParser),
+		ff.WithEnvVarPrefix("CATALYST"),
+		ff.WithEnvVarSplit(","),
+	)
+	flag.CommandLine.Parse(nil)
+	vFlag.Value.Set(cliFlags.Verbosity)
+
+	err := downloaderCli.ValidateFlags(&cliFlags)
+	if err != nil {
+		return cliFlags, err
+	}
+	return cliFlags, err
+}

cmd/downloader/cli/cli.go

@@ -1,7 +1,6 @@
 package cli
 
 import (
-	"errors"
 	"flag"
 	"fmt"
 	"net/url"
@@ -15,7 +14,7 @@ import (
 	"github.com/peterbourgon/ff/v3"
 )
 
-func validateFlags(flags *types.CliFlags) error {
+func ValidateFlags(flags *types.CliFlags) error {
 	if !utils.IsSupportedPlatformArch(flags.Platform, flags.Architecture) {
 		return fmt.Errorf(
 			"invalid combination of platform+architecture detected: %s+%s",
@@ -30,8 +29,6 @@ func validateFlags(flags *types.CliFlags) error {
 		}
 		if manifestURL.Scheme == "https" {
 			flags.ManifestURL = true
-		} else if len(flags.ExecCommand) == 0 {
-			return errors.New("invalid path/url to manifest file")
 		}
 	}
 	if info, err := os.Stat(flags.DownloadPath); !(err == nil && info.IsDir()) {
@@ -47,38 +44,12 @@ func validateFlags(flags *types.CliFlags) error {
 // with useful values set after parsing the same.
 func GetCliFlags(buildFlags types.BuildFlags) (types.CliFlags, error) {
 	cliFlags := types.CliFlags{}
-	args := []string{}
-	// Handle post-exec string
-	for i, arg := range os.Args[1:] {
-		if arg == "--" {
-			cliFlags.ExecCommand = os.Args[i+2:]
-			break
-		}
-		args = append(args, arg)
-	}
+
 	flag.Set("logtostderr", "true")
 	vFlag := flag.Lookup("v")
 	fs := flag.NewFlagSet(constants.AppName, flag.ExitOnError)
 
-	goos := runtime.GOOS
-	if os.Getenv("GOOS") != "" {
-		goos = os.Getenv("GOOS")
-	}
-
-	goarch := runtime.GOARCH
-	if os.Getenv("GOARCH") != "" {
-		goarch = os.Getenv("GOARCH")
-	}
-
-	fs.StringVar(&cliFlags.Verbosity, "v", "3", "Log verbosity. Integer value from 0 to 9")
-	fs.StringVar(&cliFlags.Platform, "platform", goos, "One of linux/windows/darwin")
-	fs.StringVar(&cliFlags.Architecture, "architecture", goarch, "System architecture (amd64/arm64)")
-	fs.StringVar(&cliFlags.DownloadPath, "path", fmt.Sprintf(".%sbin", string(os.PathSeparator)), "Path to store binaries")
-	fs.StringVar(&cliFlags.ManifestFile, "manifest", "manifest.yaml", "Path (or URL) to manifest yaml file")
-	fs.BoolVar(&cliFlags.SkipDownloaded, "skip-downloaded", false, "Skip already downloaded archive (if found)")
-	fs.BoolVar(&cliFlags.Cleanup, "cleanup", true, "Cleanup downloaded archives after extraction")
-	fs.BoolVar(&cliFlags.UpdateManifest, "update-manifest", false, "Update the manifest file commit shas from releases prior to downloading")
-	fs.BoolVar(&cliFlags.Download, "download", true, "Actually do a download. Only useful for -update-manifest=true -download=false")
+	AddDownloaderFlags(fs, &cliFlags)
 
 	version := fs.Bool("version", false, "Get version information")
 
@@ -88,18 +59,40 @@ func GetCliFlags(buildFlags types.BuildFlags) (types.CliFlags, error) {
 	}
 
 	ff.Parse(
-		fs, args,
-		ff.WithConfigFileFlag("config"),
+		fs, os.Args[1:],
 		ff.WithConfigFileParser(ff.PlainParser),
 		ff.WithEnvVarPrefix("CATALYST_DOWNLOADER"),
 		ff.WithEnvVarSplit(","),
 	)
 	flag.CommandLine.Parse(nil)
 	vFlag.Value.Set(cliFlags.Verbosity)
 
-	err := validateFlags(&cliFlags)
+	err := ValidateFlags(&cliFlags)
 	if err != nil {
 		glog.Fatal(err)
 	}
 	return cliFlags, err
 }
+
+// Populate a provided flagset with downloader flags
+func AddDownloaderFlags(fs *flag.FlagSet, cliFlags *types.CliFlags) {
+	goos := runtime.GOOS
+	if os.Getenv("GOOS") != "" {
+		goos = os.Getenv("GOOS")
+	}
+
+	goarch := runtime.GOARCH
+	if os.Getenv("GOARCH") != "" {
+		goarch = os.Getenv("GOARCH")
+	}
+
+	fs.StringVar(&cliFlags.Verbosity, "v", "3", "Log verbosity. Integer value from 0 to 9")
+	fs.StringVar(&cliFlags.Platform, "platform", goos, "One of linux/windows/darwin")
+	fs.StringVar(&cliFlags.Architecture, "architecture", goarch, "System architecture (amd64/arm64)")
+	fs.StringVar(&cliFlags.DownloadPath, "path", fmt.Sprintf(".%sbin", string(os.PathSeparator)), "Path to store binaries")
+	fs.StringVar(&cliFlags.ManifestFile, "manifest", "manifest.yaml", "Path (or URL) to manifest yaml file")
+	fs.BoolVar(&cliFlags.SkipDownloaded, "skip-downloaded", false, "Skip already downloaded archive (if found)")
+	fs.BoolVar(&cliFlags.Cleanup, "cleanup", true, "Cleanup downloaded archives after extraction")
+	fs.BoolVar(&cliFlags.UpdateManifest, "update-manifest", false, "Update the manifest file commit shas from releases prior to downloading")
+	fs.BoolVar(&cliFlags.Download, "download", true, "Actually do a download. Only useful for -update-manifest=true -download=false")
+}

cmd/downloader/types/types.go

@@ -43,9 +43,10 @@ type CliFlags struct {
 	Architecture   string
 	ManifestFile   string
 	Verbosity      string
-	ExecCommand    []string
 
-	ManifestURL bool
+	ManifestURL    bool
+	MistController string
+	ConfigStack    string
 }
 
 type DownloadStrategy struct {