From 79fac5d7714289c77a930d15c18b13a413bea192 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Thu, 28 Sep 2023 07:21:54 -0700 Subject: [PATCH] cmd/catalyst: boot from a temporary mist config file --- Makefile | 2 ++ cmd/catalyst/catalyst.go | 46 +++++++++++++++---------- cmd/catalyst/cli/cli.go | 48 ++++++++++++++++++++++++++ cmd/downloader/cli/cli.go | 63 ++++++++++++++++------------------- cmd/downloader/types/types.go | 5 +-- 5 files changed, 110 insertions(+), 54 deletions(-) create mode 100644 cmd/catalyst/cli/cli.go diff --git a/Makefile b/Makefile index 6dea07281..f6af4d12a 100644 --- a/Makefile +++ b/Makefile @@ -219,6 +219,8 @@ box-dev: scripts -v $$(realpath config):/etc/livepeer:ro \ -v $$(realpath ./coredumps):$$(realpath ./coredumps) \ -e CORE_DUMP_DIR=$$(realpath ./coredumps) \ + -e CATALYST_CONFIG=/etc/livepeer/full-stack.yaml \ + -e CATALYST_DOWNLOAD=false \ $(shell for line in $$(cat .env 2>/dev/null || echo ''); do printf -- "-e $$line "; done) \ --rm \ -it \ diff --git a/cmd/catalyst/catalyst.go b/cmd/catalyst/catalyst.go index fa6f1ef74..4f6df6476 100644 --- a/cmd/catalyst/catalyst.go +++ b/cmd/catalyst/catalyst.go @@ -1,17 +1,15 @@ package main import ( - "encoding/json" - "fmt" "os" + "os/exec" "syscall" - "github.com/icza/dyno" - "github.com/livepeer/catalyst/cmd/downloader/cli" + "github.com/livepeer/catalyst/cmd/catalyst/cli" + "github.com/livepeer/catalyst/cmd/catalyst/config" "github.com/livepeer/catalyst/cmd/downloader/downloader" "github.com/livepeer/catalyst/cmd/downloader/types" glog "github.com/magicsong/color-glog" - "gopkg.in/yaml.v3" ) var Version = "undefined" @@ -25,27 +23,41 @@ func main() { glog.Fatalf("error parsing cli flags: %s", err) return } - err = downloader.Run(cliFlags) + if cliFlags.Download { + err = downloader.Run(cliFlags) + if err != nil { + glog.Fatalf("error running downloader: %s", err) + } + } + err = execNext(cliFlags) if err != nil { - glog.Fatalf("error running downloader: %s", err) + glog.Fatalf("error executing MistController: %s", err) } - execNext(cliFlags) } // Done! Move on to the provided next application, if it exists. -func execNext(cliFlags types.CliFlags) { - if len(cliFlags.ExecCommand) == 0 { - // Nothing to do. - return +func execNext(cliFlags types.CliFlags) error { + jsonBytes, err := config.HandleConfigStack(cliFlags.ConfigStack) + if err != nil { + return err + } + f, err := os.CreateTemp("", "catalyst-generated-*.json") + if err != nil { + return err + } + _, err = f.Write(jsonBytes) + if err != nil { + return err } - configStr, err := handleConfigFile("/home/iameli/code/catalyst/config/full-stack.yaml") + glog.Infof("downloader complete, now we will exec %v", cliFlags.MistController) + binary, err := exec.LookPath(cliFlags.MistController) if err != nil { - panic(err) + return err } - panic(configStr) - glog.Infof("downloader complete, now we will exec %v", cliFlags.ExecCommand) - execErr := syscall.Exec(cliFlags.ExecCommand[0], cliFlags.ExecCommand, os.Environ()) + args := []string{binary, "-c", f.Name()} + execErr := syscall.Exec(binary, args, os.Environ()) if execErr != nil { glog.Fatalf("error running next command: %s", execErr) } + return nil } diff --git a/cmd/catalyst/cli/cli.go b/cmd/catalyst/cli/cli.go new file mode 100644 index 000000000..191137c57 --- /dev/null +++ b/cmd/catalyst/cli/cli.go @@ -0,0 +1,48 @@ +package cli + +import ( + "flag" + "fmt" + "os" + + downloaderCli "github.com/livepeer/catalyst/cmd/downloader/cli" + "github.com/livepeer/catalyst/cmd/downloader/constants" + "github.com/livepeer/catalyst/cmd/downloader/types" + "github.com/peterbourgon/ff/v3" +) + +// GetCliFlags reads command-line arguments and generates a struct +// with useful values set after parsing the same. +func GetCliFlags(buildFlags types.BuildFlags) (types.CliFlags, error) { + cliFlags := types.CliFlags{} + flag.Set("logtostderr", "true") + vFlag := flag.Lookup("v") + fs := flag.NewFlagSet(constants.AppName, flag.ExitOnError) + + downloaderCli.AddDownloaderFlags(fs, &cliFlags) + + fs.StringVar(&cliFlags.MistController, "mist-controller", "MistController", "Path to MistController binary to exec when done") + fs.StringVar(&cliFlags.ConfigStack, "config", "/etc/livepeer/catalyst.yaml", "Path to multiple Catalyst config files to use. Can contain multiple entries e.g. /conf1:/conf2") + + version := fs.Bool("version", false, "Get version information") + + if *version { + fmt.Printf("catalyst version: %s\n", buildFlags.Version) + os.Exit(0) + } + + ff.Parse( + fs, os.Args[1:], + ff.WithConfigFileParser(ff.PlainParser), + ff.WithEnvVarPrefix("CATALYST"), + ff.WithEnvVarSplit(","), + ) + flag.CommandLine.Parse(nil) + vFlag.Value.Set(cliFlags.Verbosity) + + err := downloaderCli.ValidateFlags(&cliFlags) + if err != nil { + return cliFlags, err + } + return cliFlags, err +} diff --git a/cmd/downloader/cli/cli.go b/cmd/downloader/cli/cli.go index 7c890210d..90a9b97ea 100644 --- a/cmd/downloader/cli/cli.go +++ b/cmd/downloader/cli/cli.go @@ -1,7 +1,6 @@ package cli import ( - "errors" "flag" "fmt" "net/url" @@ -15,7 +14,7 @@ import ( "github.com/peterbourgon/ff/v3" ) -func validateFlags(flags *types.CliFlags) error { +func ValidateFlags(flags *types.CliFlags) error { if !utils.IsSupportedPlatformArch(flags.Platform, flags.Architecture) { return fmt.Errorf( "invalid combination of platform+architecture detected: %s+%s", @@ -30,8 +29,6 @@ func validateFlags(flags *types.CliFlags) error { } if manifestURL.Scheme == "https" { flags.ManifestURL = true - } else if len(flags.ExecCommand) == 0 { - return errors.New("invalid path/url to manifest file") } } if info, err := os.Stat(flags.DownloadPath); !(err == nil && info.IsDir()) { @@ -47,38 +44,12 @@ func validateFlags(flags *types.CliFlags) error { // with useful values set after parsing the same. func GetCliFlags(buildFlags types.BuildFlags) (types.CliFlags, error) { cliFlags := types.CliFlags{} - args := []string{} - // Handle post-exec string - for i, arg := range os.Args[1:] { - if arg == "--" { - cliFlags.ExecCommand = os.Args[i+2:] - break - } - args = append(args, arg) - } + flag.Set("logtostderr", "true") vFlag := flag.Lookup("v") fs := flag.NewFlagSet(constants.AppName, flag.ExitOnError) - goos := runtime.GOOS - if os.Getenv("GOOS") != "" { - goos = os.Getenv("GOOS") - } - - goarch := runtime.GOARCH - if os.Getenv("GOARCH") != "" { - goarch = os.Getenv("GOARCH") - } - - fs.StringVar(&cliFlags.Verbosity, "v", "3", "Log verbosity. Integer value from 0 to 9") - fs.StringVar(&cliFlags.Platform, "platform", goos, "One of linux/windows/darwin") - fs.StringVar(&cliFlags.Architecture, "architecture", goarch, "System architecture (amd64/arm64)") - fs.StringVar(&cliFlags.DownloadPath, "path", fmt.Sprintf(".%sbin", string(os.PathSeparator)), "Path to store binaries") - fs.StringVar(&cliFlags.ManifestFile, "manifest", "manifest.yaml", "Path (or URL) to manifest yaml file") - fs.BoolVar(&cliFlags.SkipDownloaded, "skip-downloaded", false, "Skip already downloaded archive (if found)") - fs.BoolVar(&cliFlags.Cleanup, "cleanup", true, "Cleanup downloaded archives after extraction") - fs.BoolVar(&cliFlags.UpdateManifest, "update-manifest", false, "Update the manifest file commit shas from releases prior to downloading") - fs.BoolVar(&cliFlags.Download, "download", true, "Actually do a download. Only useful for -update-manifest=true -download=false") + AddDownloaderFlags(fs, &cliFlags) version := fs.Bool("version", false, "Get version information") @@ -88,8 +59,7 @@ func GetCliFlags(buildFlags types.BuildFlags) (types.CliFlags, error) { } ff.Parse( - fs, args, - ff.WithConfigFileFlag("config"), + fs, os.Args[1:], ff.WithConfigFileParser(ff.PlainParser), ff.WithEnvVarPrefix("CATALYST_DOWNLOADER"), ff.WithEnvVarSplit(","), @@ -97,9 +67,32 @@ func GetCliFlags(buildFlags types.BuildFlags) (types.CliFlags, error) { flag.CommandLine.Parse(nil) vFlag.Value.Set(cliFlags.Verbosity) - err := validateFlags(&cliFlags) + err := ValidateFlags(&cliFlags) if err != nil { glog.Fatal(err) } return cliFlags, err } + +// Populate a provided flagset with downloader flags +func AddDownloaderFlags(fs *flag.FlagSet, cliFlags *types.CliFlags) { + goos := runtime.GOOS + if os.Getenv("GOOS") != "" { + goos = os.Getenv("GOOS") + } + + goarch := runtime.GOARCH + if os.Getenv("GOARCH") != "" { + goarch = os.Getenv("GOARCH") + } + + fs.StringVar(&cliFlags.Verbosity, "v", "3", "Log verbosity. Integer value from 0 to 9") + fs.StringVar(&cliFlags.Platform, "platform", goos, "One of linux/windows/darwin") + fs.StringVar(&cliFlags.Architecture, "architecture", goarch, "System architecture (amd64/arm64)") + fs.StringVar(&cliFlags.DownloadPath, "path", fmt.Sprintf(".%sbin", string(os.PathSeparator)), "Path to store binaries") + fs.StringVar(&cliFlags.ManifestFile, "manifest", "manifest.yaml", "Path (or URL) to manifest yaml file") + fs.BoolVar(&cliFlags.SkipDownloaded, "skip-downloaded", false, "Skip already downloaded archive (if found)") + fs.BoolVar(&cliFlags.Cleanup, "cleanup", true, "Cleanup downloaded archives after extraction") + fs.BoolVar(&cliFlags.UpdateManifest, "update-manifest", false, "Update the manifest file commit shas from releases prior to downloading") + fs.BoolVar(&cliFlags.Download, "download", true, "Actually do a download. Only useful for -update-manifest=true -download=false") +} diff --git a/cmd/downloader/types/types.go b/cmd/downloader/types/types.go index eaea40084..fe42f4746 100644 --- a/cmd/downloader/types/types.go +++ b/cmd/downloader/types/types.go @@ -43,9 +43,10 @@ type CliFlags struct { Architecture string ManifestFile string Verbosity string - ExecCommand []string - ManifestURL bool + ManifestURL bool + MistController string + ConfigStack string } type DownloadStrategy struct {