From 7147bfe6d0c102f7aa3c5de9d1973caa077f5aaa Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 6 May 2024 13:11:24 -0700 Subject: [PATCH 01/67] Add initial Exacloud page to documentation --- website/content/docs/interop/exacloud.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 website/content/docs/interop/exacloud.md diff --git a/website/content/docs/interop/exacloud.md b/website/content/docs/interop/exacloud.md new file mode 100644 index 000000000..8c9cd4c75 --- /dev/null +++ b/website/content/docs/interop/exacloud.md @@ -0,0 +1,15 @@ +--- +title: Exacloud +menu: + main: + parent: Interop +--- + +> ⚠️ Exacloud support is currently in development and requires a few additional steps to run which are included below. + +# Exacloud + +## Getting Started + +## Additional Resources + From af00d03c63a68b2982566d82ef77160c1d10e7e9 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 7 May 2024 16:54:47 -0700 Subject: [PATCH 02/67] Update Exacloud support --- cmd/server/run.go | 1 + compute/hpc_backend.go | 15 ++++++++------ config/config.go | 3 +++ examples/exacloud.config.yml | 40 ++++++++++++++++++++++++++++++++++++ server/tes.go | 2 ++ worker/docker.go | 18 ++++++++-------- 6 files changed, 65 insertions(+), 14 deletions(-) create mode 100644 examples/exacloud.config.yml diff --git a/cmd/server/run.go b/cmd/server/run.go index 2213cb04f..daf5dd45b 100644 --- a/cmd/server/run.go +++ b/cmd/server/run.go @@ -271,6 +271,7 @@ func NewServer(ctx context.Context, conf config.Config, log *logger.Logger) (*Se Compute: compute, Read: reader, Log: log, + HostName: conf.Server.HostName, }, Events: &events.Service{Writer: writer}, Nodes: nodes, diff --git a/compute/hpc_backend.go b/compute/hpc_backend.go index 1f09cee9e..0081584bf 100644 --- a/compute/hpc_backend.go +++ b/compute/hpc_backend.go @@ -48,7 +48,7 @@ type HPCBackend struct { func (b *HPCBackend) WriteEvent(ctx context.Context, ev *events.Event) error { switch ev.Type { case events.Type_TASK_CREATED: - return b.Submit(ev.GetTask()) + return b.Submit(ctx, ev.GetTask()) case events.Type_TASK_STATE: if ev.GetState() == tes.State_CANCELED { @@ -61,10 +61,8 @@ func (b *HPCBackend) WriteEvent(ctx context.Context, ev *events.Event) error { func (b *HPCBackend) Close() {} // Submit submits a task via "qsub", "condor_submit", "sbatch", etc. -func (b *HPCBackend) Submit(task *tes.Task) error { - ctx := context.Background() - - submitPath, err := b.setupTemplatedHPCSubmit(task) +func (b *HPCBackend) Submit(ctx context.Context, task *tes.Task) error { + submitPath, err := b.setupTemplatedHPCSubmit(ctx, task) if err != nil { return err } @@ -229,7 +227,7 @@ ReconcileLoop: // setupTemplatedHPCSubmit sets up a task submission in a HPC environment with // a shared file system. It generates a submission file based on a template for // schedulers such as SLURM, HTCondor, SGE, PBS/Torque, etc. -func (b *HPCBackend) setupTemplatedHPCSubmit(task *tes.Task) (string, error) { +func (b *HPCBackend) setupTemplatedHPCSubmit(ctx context.Context, task *tes.Task) (string, error) { var err error // TODO document that these working dirs need manual cleanup @@ -264,6 +262,7 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(task *tes.Task) (string, error) { zone = zones[0] } + args := fmt.Sprintf("--Server.HostName %v", ctx.Value("HostName")) err = submitTpl.Execute(f, map[string]interface{}{ "TaskId": task.Id, "WorkDir": workdir, @@ -271,12 +270,16 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(task *tes.Task) (string, error) { "RamGb": res.GetRamGb(), "DiskGb": res.GetDiskGb(), "Zone": zone, + "Args": args, }) if err != nil { + fmt.Println("DEBUG: err 3:", err) return "", err } + fmt.Println("DEBUG: f.Name()", f.Name()) f.Close() + fmt.Println("DEBUG: submitPath", submitPath) return submitPath, nil } diff --git a/config/config.go b/config/config.go index ecd387b0d..73d8e2bc7 100644 --- a/config/config.go +++ b/config/config.go @@ -151,6 +151,9 @@ type Worker struct { LeaveWorkDir bool // Limit the number of concurrent downloads/uploads MaxParallelTransfers int + // Container engine to use for executing tasks. + // Typically this is "docker". + Engine string } // HPCBackend describes the configuration for a HPC scheduler backend such as diff --git a/examples/exacloud.config.yml b/examples/exacloud.config.yml new file mode 100644 index 000000000..1ad0bd84b --- /dev/null +++ b/examples/exacloud.config.yml @@ -0,0 +1,40 @@ +LocalStorage: + # Whitelist of local directory paths which Funnel is allowed to access. + AllowedDirs: + - ./ + - /tmp + +Worker: + LeaveWorkDir: true + Engine: sudo /opt/acc/sbin/exadocker + +Server: + HostName: exahead1 + +Compute: slurm + +Slurm: + Template: | + #!/bin/bash + #SBATCH --job-name {{.TaskId}} + #SBATCH --ntasks 1 + #SBATCH --error {{.WorkDir}}/funnel-stderr + #SBATCH --output {{.WorkDir}}/funnel-stdout + #SBATCH --gres disk:1 + {{if ne .Cpus 0 -}} + {{printf "#SBATCH --cpus-per-task %d" .Cpus}} + {{- end}} + {{if ne .RamGb 0.0 -}} + {{printf "#SBATCH --mem %.0fGB" .RamGb}} + {{- end}} + {{if ne .DiskGb 0.0 -}} + {{printf "#SBATCH --tmp %.0fGB" .DiskGb}} + {{- end}} + + srun /usr/local/bin/mkdir-scratch.sh + SCRATCH_PATH="/mnt/scratch/${SLURM_JOB_ID}" + cd $SCRATCH_PATH + srun bash -c 'funnel worker run --taskID {{.TaskId}} {{.Args}}' --temp-dir $SCRATCH_PATH + cd - + srun /usr/local/bin/rmdir-scratch.sh + diff --git a/server/tes.go b/server/tes.go index 953dcefcb..399c4143a 100644 --- a/server/tes.go +++ b/server/tes.go @@ -29,6 +29,7 @@ type TaskService struct { Compute events.Computer Read tes.ReadOnlyServer Log *logger.Logger + HostName string } // CreateTask provides an HTTP/gRPC endpoint for creating a task. @@ -44,6 +45,7 @@ func (ts *TaskService) CreateTask(ctx context.Context, task *tes.Task) (*tes.Cre return nil, fmt.Errorf("error from backend: %s", err) } + ctx = context.WithValue(ctx, "HostName", ts.HostName) if err := ts.Event.WriteEvent(ctx, events.NewTaskCreated(task)); err != nil { return nil, fmt.Errorf("error creating task: %s", err) } diff --git a/worker/docker.go b/worker/docker.go index 5fc6b8dc2..94f9fbc94 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -26,6 +26,7 @@ type DockerCommand struct { Stdout io.Writer Stderr io.Writer Event *events.ExecutorWriter + Engine string } // Run runs the Docker command and blocks until done. @@ -36,13 +37,13 @@ func (dcmd DockerCommand) Run(ctx context.Context) error { dcmd.Event.Error("failed to sync docker client API version", err) } - pullcmd := exec.Command("docker", "pull", dcmd.Image) + pullcmd := exec.Command(dcmd.Engine, "pull", dcmd.Image) err = pullcmd.Run() if err != nil { dcmd.Event.Error("failed to pull docker image", err) } - args := []string{"run", "-i", "--read-only"} + args := []string{dcmd.Engine, "run", "-i", "--read-only"} if dcmd.RemoveContainer { args = append(args, "--rm") @@ -77,8 +78,9 @@ func (dcmd DockerCommand) Run(ctx context.Context) error { // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` err = dcmd.Event.Info(fmt.Sprintf("args: %s", args)) - dcmd.Event.Info("Running command", "cmd", "docker "+strings.Join(args, " ")) - cmd := exec.Command("docker", args...) + dcmd.Event.Info("Running command", "cmd", dcmd.Engine+" "+strings.Join(args, " ")) + strings.Split(dcmd.Engine, " ") + cmd := exec.Command("sudo", args...) if dcmd.Stdin != nil { cmd.Stdin = dcmd.Stdin @@ -99,7 +101,7 @@ func (dcmd DockerCommand) Run(ctx context.Context) error { func (dcmd DockerCommand) Stop() error { dcmd.Event.Info("Stopping container", "container", dcmd.ContainerName) // cmd := exec.Command("docker", "stop", dcmd.ContainerName) - cmd := exec.Command("docker", "rm", "-f", dcmd.ContainerName) //switching to this to be a bit more forceful + cmd := exec.Command("sudo", "opt/acc/sbin/exadocker", "rm", "-f", dcmd.ContainerName) //switching to this to be a bit more forceful return cmd.Run() } @@ -141,7 +143,7 @@ func (dcmd *DockerCommand) inspectContainer(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - cmd := exec.CommandContext(ctx, "docker", "inspect", dcmd.ContainerName) + cmd := exec.CommandContext(ctx, "sudo", "/opt/acc/sbin/exadocker", "inspect", dcmd.ContainerName) out, err := cmd.Output() if err == nil { meta := []metadata{} @@ -167,10 +169,10 @@ type dockerVersion struct { // the server. func SyncDockerAPIVersion() error { if os.Getenv("DOCKER_API_VERSION") == "" { - cmd := exec.Command("docker", "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) + cmd := exec.Command("sudo", "/opt/acc/sbin/exadocker", "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) out, err := cmd.Output() if err != nil { - return fmt.Errorf("docker version command failed: %v", err) + return fmt.Errorf("sudo", "/opt/acc/sbin/exadocker version command failed: %v", err) } version := &dockerVersion{} err = json.Unmarshal(out, version) From fc3d277f1f3e4165f68f02ae0f7018a03e8693e8 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 7 May 2024 16:55:08 -0700 Subject: [PATCH 03/67] Clean up console debug output [no ci] --- webdash/src/Pages.js | 1 - 1 file changed, 1 deletion(-) diff --git a/webdash/src/Pages.js b/webdash/src/Pages.js index b0841e0fb..99f4e0469 100644 --- a/webdash/src/Pages.js +++ b/webdash/src/Pages.js @@ -315,7 +315,6 @@ function ServiceInfo() { React.useEffect(() => { var url = new URL("/v1/service-info", window.location.origin); - console.log("DEBUG: ServiceInfo url:", url); get(url).then( (info) => { console.log("DEBUG: ServiceInfo info:", info); From 9dbef333de22b228d5380b39ccbfbd839097ef79 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 7 May 2024 17:00:26 -0700 Subject: [PATCH 04/67] Add default Worker engine ('docker') [no ci] --- compute/hpc_backend.go | 3 --- config/default.go | 1 + webdash/src/Pages.js | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/compute/hpc_backend.go b/compute/hpc_backend.go index 0081584bf..272b8a21c 100644 --- a/compute/hpc_backend.go +++ b/compute/hpc_backend.go @@ -273,13 +273,10 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(ctx context.Context, task *tes.Task "Args": args, }) if err != nil { - fmt.Println("DEBUG: err 3:", err) return "", err } - fmt.Println("DEBUG: f.Name()", f.Name()) f.Close() - fmt.Println("DEBUG: submitPath", submitPath) return submitPath, nil } diff --git a/config/default.go b/config/default.go index 44d680b6a..758cfdd26 100644 --- a/config/default.go +++ b/config/default.go @@ -61,6 +61,7 @@ func DefaultConfig() Config { LogUpdateRate: Duration(time.Second * 5), LogTailSize: 10000, MaxParallelTransfers: 10, + Engine: "docker", }, Logger: logger.DefaultConfig(), // databases / event handlers diff --git a/webdash/src/Pages.js b/webdash/src/Pages.js index 99f4e0469..cf7686b9e 100644 --- a/webdash/src/Pages.js +++ b/webdash/src/Pages.js @@ -317,7 +317,6 @@ function ServiceInfo() { var url = new URL("/v1/service-info", window.location.origin); get(url).then( (info) => { - console.log("DEBUG: ServiceInfo info:", info); setInfo(info); }); }, []); From a3a3445750e921715e60c32497c3196d41fdbaca Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 9 May 2024 11:27:09 -0700 Subject: [PATCH 05/67] Update configurable Container Engine commands --- worker/docker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/worker/docker.go b/worker/docker.go index 94f9fbc94..8155fe900 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -101,7 +101,7 @@ func (dcmd DockerCommand) Run(ctx context.Context) error { func (dcmd DockerCommand) Stop() error { dcmd.Event.Info("Stopping container", "container", dcmd.ContainerName) // cmd := exec.Command("docker", "stop", dcmd.ContainerName) - cmd := exec.Command("sudo", "opt/acc/sbin/exadocker", "rm", "-f", dcmd.ContainerName) //switching to this to be a bit more forceful + cmd := exec.Command(dcmd.Engine, "rm", "-f", dcmd.ContainerName) //switching to this to be a bit more forceful return cmd.Run() } @@ -169,10 +169,10 @@ type dockerVersion struct { // the server. func SyncDockerAPIVersion() error { if os.Getenv("DOCKER_API_VERSION") == "" { - cmd := exec.Command("sudo", "/opt/acc/sbin/exadocker", "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) + cmd := exec.Command("docker", "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) out, err := cmd.Output() if err != nil { - return fmt.Errorf("sudo", "/opt/acc/sbin/exadocker version command failed: %v", err) + return fmt.Errorf("docker version command failed: %v", err) } version := &dockerVersion{} err = json.Unmarshal(out, version) From 8ec817b55a366db83fd91b2528b7603c5bb73f92 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 9 May 2024 18:33:11 -0700 Subject: [PATCH 06/67] Revert changes to docker implementation and add engine interface [no ci] --- worker/docker.go | 29 ++++++----------------------- worker/engine.go | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 23 deletions(-) create mode 100644 worker/engine.go diff --git a/worker/docker.go b/worker/docker.go index 8155fe900..1dd928e6b 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -26,7 +26,6 @@ type DockerCommand struct { Stdout io.Writer Stderr io.Writer Event *events.ExecutorWriter - Engine string } // Run runs the Docker command and blocks until done. @@ -37,13 +36,13 @@ func (dcmd DockerCommand) Run(ctx context.Context) error { dcmd.Event.Error("failed to sync docker client API version", err) } - pullcmd := exec.Command(dcmd.Engine, "pull", dcmd.Image) + pullcmd := exec.Command("docker", "pull", dcmd.Image) err = pullcmd.Run() if err != nil { dcmd.Event.Error("failed to pull docker image", err) } - args := []string{dcmd.Engine, "run", "-i", "--read-only"} + args := []string{"run", "-i", "--read-only"} if dcmd.RemoveContainer { args = append(args, "--rm") @@ -69,18 +68,11 @@ func (dcmd DockerCommand) Run(ctx context.Context) error { } args = append(args, dcmd.Image) - // if dcmd.Command contains the ["/bin/bash", "-ue"] string, replace it with "/bin/bash -ue" - c := strings.Join(dcmd.Command, " ") - if strings.Contains(c, "[/bin/bash, -ue]") { - dcmd.Command = transformCommandSlice(dcmd.Command) - } args = append(args, dcmd.Command...) // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` - err = dcmd.Event.Info(fmt.Sprintf("args: %s", args)) - dcmd.Event.Info("Running command", "cmd", dcmd.Engine+" "+strings.Join(args, " ")) - strings.Split(dcmd.Engine, " ") - cmd := exec.Command("sudo", args...) + dcmd.Event.Info("Running command", "cmd", "docker "+strings.Join(args, " ")) + cmd := exec.Command("docker", args...) if dcmd.Stdin != nil { cmd.Stdin = dcmd.Stdin @@ -101,7 +93,7 @@ func (dcmd DockerCommand) Run(ctx context.Context) error { func (dcmd DockerCommand) Stop() error { dcmd.Event.Info("Stopping container", "container", dcmd.ContainerName) // cmd := exec.Command("docker", "stop", dcmd.ContainerName) - cmd := exec.Command(dcmd.Engine, "rm", "-f", dcmd.ContainerName) //switching to this to be a bit more forceful + cmd := exec.Command("docker", "rm", "-f", dcmd.ContainerName) //switching to this to be a bit more forceful return cmd.Run() } @@ -114,15 +106,6 @@ func formatVolumeArg(v Volume) string { return fmt.Sprintf("%s:%s:%s", v.HostPath, v.ContainerPath, mode) } -func transformCommandSlice(cmd []string) []string { - if len(cmd) == 0 { - return cmd // Handle empty slice - } - - cmd[2] = "/bin/bash -ue .command.run &> .command.log" - return cmd -} - type metadata struct { ID string Name string @@ -143,7 +126,7 @@ func (dcmd *DockerCommand) inspectContainer(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - cmd := exec.CommandContext(ctx, "sudo", "/opt/acc/sbin/exadocker", "inspect", dcmd.ContainerName) + cmd := exec.CommandContext(ctx, "docker", "inspect", dcmd.ContainerName) out, err := cmd.Output() if err == nil { meta := []metadata{} diff --git a/worker/engine.go b/worker/engine.go new file mode 100644 index 000000000..d648203d0 --- /dev/null +++ b/worker/engine.go @@ -0,0 +1,27 @@ +package worker + +import "context" + +type Engine interface { + Pull(ctx context.Context, image string) error + + Run(ctx context.Context, config ContainerConfig) error + + Stop(ctx context.Context, containerID string) error + + Inspect(ctx context.Context, containerID string) (ContainerInfo, error) +} + +type ContainerConfig struct { + Image string + Command []string + Args []string + EnvVars map[string]string + Volumes []Volume +} + +type ContainerInfo struct { + ID string + Name string + Image string +} From afc58258242ac7e072238c6d6b10631ca39da19f Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 14 May 2024 17:33:25 -0700 Subject: [PATCH 07/67] Update Exacloud/Container Engine implementation [no ci] --- examples/exacloud.config.yml | 3 +- util/dockerutil/docker.go | 4 +- worker/container_engine.go | 36 +++++++++ worker/engine.go | 27 ------- worker/exadocker.go | 144 +++++++++++++++++++++++++++++++++++ 5 files changed, 183 insertions(+), 31 deletions(-) create mode 100644 worker/container_engine.go delete mode 100644 worker/engine.go create mode 100644 worker/exadocker.go diff --git a/examples/exacloud.config.yml b/examples/exacloud.config.yml index 1ad0bd84b..7a126f608 100644 --- a/examples/exacloud.config.yml +++ b/examples/exacloud.config.yml @@ -6,7 +6,8 @@ LocalStorage: Worker: LeaveWorkDir: true - Engine: sudo /opt/acc/sbin/exadocker + ContainerEngine: exadocker + ContainerCommand: sudo /opt/acc/sbin/exadocker Server: HostName: exahead1 diff --git a/util/dockerutil/docker.go b/util/dockerutil/docker.go index e4ecd247b..6f25c49d1 100644 --- a/util/dockerutil/docker.go +++ b/util/dockerutil/docker.go @@ -34,9 +34,7 @@ func NewDockerClient() (*client.Client, error) { } // Error message example: // Error getting metadata for container: Error response from daemon: client is newer than server (client API version: 1.26, server API version: 1.24) - // Hardcoding Docker version until version[1] returns valid version - // os.Setenv("DOCKER_API_VERSION", version[1]) - os.Setenv("DOCKER_API_VERSION", "1.24") + os.Setenv("DOCKER_API_VERSION", version[1]) return NewDockerClient() } } diff --git a/worker/container_engine.go b/worker/container_engine.go new file mode 100644 index 000000000..264b24fbc --- /dev/null +++ b/worker/container_engine.go @@ -0,0 +1,36 @@ +package worker + +import ( + "context" + "io" + + "github.com/ohsu-comp-bio/funnel/events" +) + +type ContainerEngine interface { + Run(ctx context.Context) error + + Stop() error + + Inspect(ctx context.Context) (ContainerInfo, error) +} + +type ContainerCommand struct { + Image string + Command []string + Volumes []Volume + Workdir string + ContainerName string + RemoveContainer bool + Env map[string]string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + Event *events.ExecutorWriter +} + +type ContainerInfo struct { + Id string + Image string + Name string +} diff --git a/worker/engine.go b/worker/engine.go deleted file mode 100644 index d648203d0..000000000 --- a/worker/engine.go +++ /dev/null @@ -1,27 +0,0 @@ -package worker - -import "context" - -type Engine interface { - Pull(ctx context.Context, image string) error - - Run(ctx context.Context, config ContainerConfig) error - - Stop(ctx context.Context, containerID string) error - - Inspect(ctx context.Context, containerID string) (ContainerInfo, error) -} - -type ContainerConfig struct { - Image string - Command []string - Args []string - EnvVars map[string]string - Volumes []Volume -} - -type ContainerInfo struct { - ID string - Name string - Image string -} diff --git a/worker/exadocker.go b/worker/exadocker.go new file mode 100644 index 000000000..eb071bc3d --- /dev/null +++ b/worker/exadocker.go @@ -0,0 +1,144 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os/exec" + "strings" + "time" + + "github.com/ohsu-comp-bio/funnel/events" +) + +type Exadocker struct { + Image string + Command []string + Volumes []Volume + Workdir string + ContainerName string + RemoveContainer bool + Env map[string]string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + Event *events.ExecutorWriter +} + +type ExadockerInfo struct { + Id string + Image string + Name string +} + +var _ ContainerEngine = Exadocker{} + +// Run runs the Docker command and blocks until done. +func (exa Exadocker) Run(ctx context.Context) error { + // Sync docker API version info. + err := SyncDockerAPIVersion() + if err != nil { + exa.Event.Error("failed to sync docker client API version", err) + } + + pullcmd := exec.Command("docker", "pull", exa.Image) + err = pullcmd.Run() + if err != nil { + exa.Event.Error("failed to pull docker image", err) + } + + args := []string{"run", "-i", "--read-only"} + + if exa.RemoveContainer { + args = append(args, "--rm") + } + + if exa.Env != nil { + for k, v := range exa.Env { + args = append(args, "-e", fmt.Sprintf("%s=%s", k, v)) + } + } + + if exa.ContainerName != "" { + args = append(args, "--name", exa.ContainerName) + } + + if exa.Workdir != "" { + args = append(args, "-w", exa.Workdir) + } + + for _, vol := range exa.Volumes { + arg := formatVolumeArg(vol) + args = append(args, "-v", arg) + } + + args = append(args, exa.Image) + args = append(args, exa.Command...) + + // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` + exa.Event.Info("Running command", "cmd", "docker "+strings.Join(args, " ")) + cmd := exec.Command("docker", args...) + + if exa.Stdin != nil { + cmd.Stdin = exa.Stdin + } + if exa.Stdout != nil { + cmd.Stdout = exa.Stdout + } + if exa.Stderr != nil { + cmd.Stderr = exa.Stderr + } + go exa.inspectContainer(ctx) + out := cmd.Run() + exa.Event.Info("Command %s Complete exit=%s", strings.Join(args, " "), out) + return out +} + +// Stop stops the container. +func (exa Exadocker) Stop() error { + exa.Event.Info("Stopping container", "container", exa.ContainerName) + // cmd := exec.Command("docker", "stop", exa.ContainerName) + cmd := exec.Command("docker", "rm", "-f", exa.ContainerName) //switching to this to be a bit more forceful + return cmd.Run() +} + +func (exa Exadocker) Inspect(ctx context.Context) (ContainerInfo, error) { + info := ContainerInfo{ + Id: "1234", + Image: "image", + Name: "container", + } + return info, nil +} + +// inspectContainer inspects the docker container for metadata. +func (exa *Exadocker) inspectContainer(ctx context.Context) { + // Give the container time to start. + time.Sleep(2 * time.Second) + + // Inspect the container for metadata + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for i := 0; i < 5; i++ { + select { + case <-ctx.Done(): + return + case <-ticker.C: + cmd := exec.CommandContext(ctx, "docker", "inspect", exa.ContainerName) + out, err := cmd.Output() + if err == nil { + meta := []metadata{} + err := json.Unmarshal(out, &meta) + if err == nil && len(meta) == 1 { + exa.Event.Info("container metadata", + "containerID", meta[0].ID, + "containerName", meta[0].Name, + "containerImageHash", meta[0].Image) + return + } + } + } + } +} From 28905c07e4826f79ef35a0f469d74fce7a37eec0 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 16 May 2024 16:13:01 -0700 Subject: [PATCH 08/67] Update ContainerEngine interface [no ci] --- examples/exacloud.config.yml | 2 +- worker/container_engine.go | 41 ++++++++++++-- worker/exadocker.go | 107 +++++++++++++++++++---------------- worker/step.go | 14 ++--- worker/worker.go | 47 +++++++++------ 5 files changed, 132 insertions(+), 79 deletions(-) diff --git a/examples/exacloud.config.yml b/examples/exacloud.config.yml index 7a126f608..91b7052f6 100644 --- a/examples/exacloud.config.yml +++ b/examples/exacloud.config.yml @@ -7,7 +7,7 @@ LocalStorage: Worker: LeaveWorkDir: true ContainerEngine: exadocker - ContainerCommand: sudo /opt/acc/sbin/exadocker + Driver: sudo /opt/acc/sbin/exadocker Server: HostName: exahead1 diff --git a/worker/container_engine.go b/worker/container_engine.go index 264b24fbc..d8fae6259 100644 --- a/worker/container_engine.go +++ b/worker/container_engine.go @@ -2,6 +2,7 @@ package worker import ( "context" + "fmt" "io" "github.com/ohsu-comp-bio/funnel/events" @@ -12,11 +13,20 @@ type ContainerEngine interface { Stop() error - Inspect(ctx context.Context) (ContainerInfo, error) + Inspect(ctx context.Context) (ContainerConfig, error) + + GetImage() string + + GetIO() (io.Reader, io.Writer, io.Writer) + + SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) } -type ContainerCommand struct { +type ContainerConfig struct { + Id string Image string + Name string + Driver []string Command []string Volumes []Volume Workdir string @@ -29,8 +39,27 @@ type ContainerCommand struct { Event *events.ExecutorWriter } -type ContainerInfo struct { - Id string - Image string - Name string +type ContainerEngineFactory struct{} + +func (f *ContainerEngineFactory) NewContainerEngine(engineType string, containerConfig ContainerConfig) (ContainerEngine, error) { + switch engineType { + // case "docker": + // return NewDockerEngine(containerConfig) + case "exadocker": + return NewExadockerEngine(containerConfig) + default: + return nil, fmt.Errorf("unsupported container engine type: %s", engineType) + } +} + +// func NewDockerEngine(config ContainerConfig) (ContainerEngine, error) { +// return Docker{ +// ContainerConfig: config, +// }, nil +// } + +func NewExadockerEngine(config ContainerConfig) (ContainerEngine, error) { + return Exadocker{ + ContainerConfig: config, + }, nil } diff --git a/worker/exadocker.go b/worker/exadocker.go index eb071bc3d..267ea7866 100644 --- a/worker/exadocker.go +++ b/worker/exadocker.go @@ -8,22 +8,10 @@ import ( "os/exec" "strings" "time" - - "github.com/ohsu-comp-bio/funnel/events" ) type Exadocker struct { - Image string - Command []string - Volumes []Volume - Workdir string - ContainerName string - RemoveContainer bool - Env map[string]string - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer - Event *events.ExecutorWriter + ContainerConfig } type ExadockerInfo struct { @@ -32,79 +20,101 @@ type ExadockerInfo struct { Name string } -var _ ContainerEngine = Exadocker{} - // Run runs the Docker command and blocks until done. -func (exa Exadocker) Run(ctx context.Context) error { +func (exadocker Exadocker) Run(ctx context.Context) error { // Sync docker API version info. err := SyncDockerAPIVersion() if err != nil { - exa.Event.Error("failed to sync docker client API version", err) + exadocker.Event.Error("failed to sync docker client API version", err) } - pullcmd := exec.Command("docker", "pull", exa.Image) + commandArgs := append(exadocker.Driver[1:], "pull", exadocker.Image) + pullcmd := exec.Command(exadocker.Driver[0], commandArgs...) + + // Run the command and check for errors. err = pullcmd.Run() if err != nil { - exa.Event.Error("failed to pull docker image", err) + exadocker.Event.Error("failed to pull docker image", err) } - args := []string{"run", "-i", "--read-only"} + args := exadocker.Driver[1:] + args = append(args, "run", "-i", "--read-only") - if exa.RemoveContainer { + if exadocker.RemoveContainer { args = append(args, "--rm") } - if exa.Env != nil { - for k, v := range exa.Env { + if exadocker.Env != nil { + for k, v := range exadocker.Env { args = append(args, "-e", fmt.Sprintf("%s=%s", k, v)) } } - if exa.ContainerName != "" { - args = append(args, "--name", exa.ContainerName) + if exadocker.ContainerName != "" { + args = append(args, "--name", exadocker.ContainerName) } - if exa.Workdir != "" { - args = append(args, "-w", exa.Workdir) + if exadocker.Workdir != "" { + args = append(args, "-w", exadocker.Workdir) } - for _, vol := range exa.Volumes { + for _, vol := range exadocker.Volumes { arg := formatVolumeArg(vol) args = append(args, "-v", arg) } - args = append(args, exa.Image) - args = append(args, exa.Command...) + args = append(args, exadocker.GetImage()) + args = append(args, exadocker.Command...) // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` - exa.Event.Info("Running command", "cmd", "docker "+strings.Join(args, " ")) - cmd := exec.Command("docker", args...) + cmd := exec.Command(exadocker.Driver[0], args...) - if exa.Stdin != nil { - cmd.Stdin = exa.Stdin + if exadocker.Stdin != nil { + cmd.Stdin = exadocker.Stdin } - if exa.Stdout != nil { - cmd.Stdout = exa.Stdout + if exadocker.Stdout != nil { + cmd.Stdout = exadocker.Stdout } - if exa.Stderr != nil { - cmd.Stderr = exa.Stderr + if exadocker.Stderr != nil { + cmd.Stderr = exadocker.Stderr } - go exa.inspectContainer(ctx) + go exadocker.inspectContainer(ctx) out := cmd.Run() - exa.Event.Info("Command %s Complete exit=%s", strings.Join(args, " "), out) + exadocker.Event.Info("Command %s Complete exit=%s", strings.Join(args, " "), out) return out } // Stop stops the container. -func (exa Exadocker) Stop() error { - exa.Event.Info("Stopping container", "container", exa.ContainerName) +func (exadocker Exadocker) Stop() error { + exadocker.Event.Info("Stopping container", "container", exadocker.ContainerName) // cmd := exec.Command("docker", "stop", exa.ContainerName) - cmd := exec.Command("docker", "rm", "-f", exa.ContainerName) //switching to this to be a bit more forceful + driverArgs := strings.Join(exadocker.Driver[1:], " ") + cmd := exec.Command(exadocker.Driver[0], driverArgs, "rm", "-f", exadocker.ContainerName) //switching to this to be a bit more forceful return cmd.Run() } -func (exa Exadocker) Inspect(ctx context.Context) (ContainerInfo, error) { - info := ContainerInfo{ +func (exadocker Exadocker) GetImage() string { + return exadocker.Image +} + +func (exadocker Exadocker) GetIO() (io.Reader, io.Writer, io.Writer) { + return exadocker.Stdin, exadocker.Stdout, exadocker.Stderr +} + +func (exadocker Exadocker) SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) { + if stdin != nil { + exadocker.Stdin = stdin + } + if stdout != nil { + exadocker.Stdout = stdout + } + if stderr != nil { + exadocker.Stderr = stderr + } +} + +func (exadocker Exadocker) Inspect(ctx context.Context) (ContainerConfig, error) { + info := ContainerConfig{ Id: "1234", Image: "image", Name: "container", @@ -113,7 +123,7 @@ func (exa Exadocker) Inspect(ctx context.Context) (ContainerInfo, error) { } // inspectContainer inspects the docker container for metadata. -func (exa *Exadocker) inspectContainer(ctx context.Context) { +func (exadocker *Exadocker) inspectContainer(ctx context.Context) { // Give the container time to start. time.Sleep(2 * time.Second) @@ -126,13 +136,14 @@ func (exa *Exadocker) inspectContainer(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - cmd := exec.CommandContext(ctx, "docker", "inspect", exa.ContainerName) + driverArgs := strings.Join(exadocker.Driver[1:], " ") + cmd := exec.CommandContext(ctx, exadocker.Command[0], driverArgs, "inspect", exadocker.ContainerName) out, err := cmd.Output() if err == nil { meta := []metadata{} err := json.Unmarshal(out, &meta) if err == nil && len(meta) == 1 { - exa.Event.Info("container metadata", + exadocker.Event.Info("container metadata", "containerID", meta[0].ID, "containerName", meta[0].Name, "containerImageHash", meta[0].Image) diff --git a/worker/step.go b/worker/step.go index 4f28c797f..165778b0d 100644 --- a/worker/step.go +++ b/worker/step.go @@ -12,7 +12,7 @@ import ( type stepWorker struct { Conf config.Worker - Command *DockerCommand + Command ContainerEngine Event *events.ExecutorWriter IP string } @@ -42,14 +42,14 @@ func (s *stepWorker) Run(ctx context.Context) error { } // Capture stdout/err to file. - if s.Command.Stdout != nil { - stdout = io.MultiWriter(s.Command.Stdout, stdout) + _, out, err := s.Command.GetIO() + if out != nil { + stdout = io.MultiWriter(out, stdout) } - if s.Command.Stderr != nil { - stderr = io.MultiWriter(s.Command.Stderr, stderr) + if err != nil { + stderr = io.MultiWriter(err, stderr) } - s.Command.Stdout = stdout - s.Command.Stderr = stderr + s.Command.SetIO(nil, stdout, stderr) go func() { done <- s.Command.Run(subctx) diff --git a/worker/worker.go b/worker/worker.go index 859ae1833..6f6426034 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -132,21 +132,28 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { // Run steps if run.ok() { ignoreError := false + f := ContainerEngineFactory{} for i, d := range task.GetExecutors() { + containerConfig := ContainerConfig{ + Image: d.Image, + Command: d.Command, + Env: d.Env, + Volumes: mapper.Volumes, + Workdir: d.Workdir, + ContainerName: fmt.Sprintf("%s-%d", task.Id, i), + // TODO make RemoveContainer configurable + RemoveContainer: true, + Event: event.NewExecutorWriter(uint32(i)), + } + containerEngine, err := f.NewContainerEngine(r.Conf.Engine, containerConfig) + if err != nil { + return err + } + s := &stepWorker{ - Conf: r.Conf, - Event: event.NewExecutorWriter(uint32(i)), - Command: &DockerCommand{ - Image: d.Image, - Command: d.Command, - Env: d.Env, - Volumes: mapper.Volumes, - Workdir: d.Workdir, - ContainerName: fmt.Sprintf("%s-%d", task.Id, i), - // TODO make RemoveContainer configurable - RemoveContainer: true, - Event: event.NewExecutorWriter(uint32(i)), - }, + Conf: r.Conf, + Event: event.NewExecutorWriter(uint32(i)), + Command: containerEngine, } // Opens stdin/out/err files and updates those fields on "cmd". @@ -195,10 +202,14 @@ func (r *DefaultWorker) Close() { // openLogs opens/creates the logs files for a step and updates those fields. func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.Executor) error { - // Find the path for task stdin + var stdin *os.File + var stdout *os.File + var stderr *os.File var err error + + // Find the path for task stdin if d.Stdin != "" { - s.Command.Stdin, err = mapper.OpenHostFile(d.Stdin) + stdin, err = mapper.OpenHostFile(d.Stdin) if err != nil { s.Event.Error("Couldn't prepare log files", err) return err @@ -207,7 +218,7 @@ func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.E // Create file for task stdout if d.Stdout != "" { - s.Command.Stdout, err = mapper.CreateHostFile(d.Stdout) + stdout, err = mapper.CreateHostFile(d.Stdout) if err != nil { s.Event.Error("Couldn't prepare log files", err) return err @@ -216,13 +227,15 @@ func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.E // Create file for task stderr if d.Stderr != "" { - s.Command.Stderr, err = mapper.CreateHostFile(d.Stderr) + stderr, err = mapper.CreateHostFile(d.Stderr) if err != nil { _ = s.Event.Error("Couldn't prepare log files", err) return err } } + s.Command.SetIO(stdin, stdout, stderr) + return nil } From 703cf4420fff8f52bf11d0464c0316951a996f21 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 16 May 2024 16:51:13 -0700 Subject: [PATCH 09/67] Rename ContainerEngine to ContainerType --- cmd/util/flags.go | 2 ++ config/config.go | 5 ++++- config/default.go | 2 +- examples/exacloud.config.yml | 4 ++-- worker/container_engine.go | 6 +++--- worker/worker.go | 7 +++++-- 6 files changed, 17 insertions(+), 9 deletions(-) diff --git a/cmd/util/flags.go b/cmd/util/flags.go index 635216cda..ac952d239 100644 --- a/cmd/util/flags.go +++ b/cmd/util/flags.go @@ -101,6 +101,8 @@ func workerFlags(flagConf *config.Config) *pflag.FlagSet { f.Var(&flagConf.Worker.PollingRate, "Worker.PollingRate", "How often to poll for cancel signals") f.StringVar(&flagConf.Worker.WorkDir, "Worker.WorkDir", flagConf.Worker.WorkDir, "Working directory") f.BoolVar(&flagConf.Worker.LeaveWorkDir, "Worker.LeaveWorkDir", flagConf.Worker.LeaveWorkDir, "Leave working directory after execution") + f.StringVar(&flagConf.Worker.ContainerType, "Worker.ContainerType", flagConf.Worker.ContainerType, "Container engine to use for executing tasks. One of ['docker', 'exadocker']") + f.StringVar(&flagConf.Worker.ContainerDriver, "Worker.ContainerDriver", flagConf.Worker.ContainerDriver, "Overrides the default command used to run containers.") return f } diff --git a/config/config.go b/config/config.go index 73d8e2bc7..19371be49 100644 --- a/config/config.go +++ b/config/config.go @@ -153,7 +153,10 @@ type Worker struct { MaxParallelTransfers int // Container engine to use for executing tasks. // Typically this is "docker". - Engine string + ContainerType string + // Command to use for the container engine. + // This can be used to override the default command used to run containers. + ContainerDriver string } // HPCBackend describes the configuration for a HPC scheduler backend such as diff --git a/config/default.go b/config/default.go index 758cfdd26..511fa1abc 100644 --- a/config/default.go +++ b/config/default.go @@ -61,7 +61,7 @@ func DefaultConfig() Config { LogUpdateRate: Duration(time.Second * 5), LogTailSize: 10000, MaxParallelTransfers: 10, - Engine: "docker", + ContainerType: "docker", }, Logger: logger.DefaultConfig(), // databases / event handlers diff --git a/examples/exacloud.config.yml b/examples/exacloud.config.yml index 91b7052f6..08badecd0 100644 --- a/examples/exacloud.config.yml +++ b/examples/exacloud.config.yml @@ -6,8 +6,8 @@ LocalStorage: Worker: LeaveWorkDir: true - ContainerEngine: exadocker - Driver: sudo /opt/acc/sbin/exadocker + ContainerType: exadocker + ContainerDriver: sudo /opt/acc/sbin/exadocker Server: HostName: exahead1 diff --git a/worker/container_engine.go b/worker/container_engine.go index d8fae6259..257cf9b38 100644 --- a/worker/container_engine.go +++ b/worker/container_engine.go @@ -41,14 +41,14 @@ type ContainerConfig struct { type ContainerEngineFactory struct{} -func (f *ContainerEngineFactory) NewContainerEngine(engineType string, containerConfig ContainerConfig) (ContainerEngine, error) { - switch engineType { +func (f *ContainerEngineFactory) NewContainerEngine(containerType string, containerConfig ContainerConfig) (ContainerEngine, error) { + switch containerType { // case "docker": // return NewDockerEngine(containerConfig) case "exadocker": return NewExadockerEngine(containerConfig) default: - return nil, fmt.Errorf("unsupported container engine type: %s", engineType) + return nil, fmt.Errorf("unsupported container type: %s", containerType) } } diff --git a/worker/worker.go b/worker/worker.go index 6f6426034..44e2b744a 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -145,9 +145,12 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { RemoveContainer: true, Event: event.NewExecutorWriter(uint32(i)), } - containerEngine, err := f.NewContainerEngine(r.Conf.Engine, containerConfig) + if r.Conf.ContainerDriver != "" { + containerConfig.Driver = []string{r.Conf.ContainerDriver} + } + containerEngine, err := f.NewContainerEngine(r.Conf.ContainerType, containerConfig) if err != nil { - return err + run.syserr = err } s := &stepWorker{ From 83d6e1c72399631d73f809661fa3c2249c646b0d Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 17 May 2024 11:00:24 -0700 Subject: [PATCH 10/67] Add initial Exadocker unit test [no ci] --- worker/exadocker_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 worker/exadocker_test.go diff --git a/worker/exadocker_test.go b/worker/exadocker_test.go new file mode 100644 index 000000000..6a25d8f82 --- /dev/null +++ b/worker/exadocker_test.go @@ -0,0 +1,29 @@ +package worker + +import ( + "bytes" + "context" + "testing" + + "github.com/ohsu-comp-bio/funnel/events" + "github.com/stretchr/testify/assert" +) + +func TestExadockerRun(t *testing.T) { + var buf bytes.Buffer + + containerConfig := ContainerConfig{ + Image: "ubuntu", + Driver: []string{"docker"}, + Command: []string{"echo", "hello"}, + Stdout: &buf, + Event: events.NewExecutorWriter("task-1", 0, 0, events.Noop{}), + } + + f := ContainerEngineFactory{} + exadocker, err := f.NewContainerEngine("exadocker", containerConfig) + + err = exadocker.Run(context.Background()) + assert.NoError(t, err) + assert.Contains(t, buf.String(), "hello") +} From 39584adf3f53336f61d5c50b2d10b2c73acbb8b8 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 17 May 2024 12:52:56 -0700 Subject: [PATCH 11/67] Update TaskService to pass config to Funnel worker [no ci] --- cmd/server/run.go | 2 +- compute/hpc_backend.go | 4 +++- server/tes.go | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/server/run.go b/cmd/server/run.go index daf5dd45b..58a6b19bf 100644 --- a/cmd/server/run.go +++ b/cmd/server/run.go @@ -271,7 +271,7 @@ func NewServer(ctx context.Context, conf config.Config, log *logger.Logger) (*Se Compute: compute, Read: reader, Log: log, - HostName: conf.Server.HostName, + Config: conf, }, Events: &events.Service{Writer: writer}, Nodes: nodes, diff --git a/compute/hpc_backend.go b/compute/hpc_backend.go index 272b8a21c..d4c519214 100644 --- a/compute/hpc_backend.go +++ b/compute/hpc_backend.go @@ -262,7 +262,9 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(ctx context.Context, task *tes.Task zone = zones[0] } - args := fmt.Sprintf("--Server.HostName %v", ctx.Value("HostName")) + conf := ctx.Value("Config").(config.Config) + configFile := config.ToYamlFile(conf, filepath.Join(workdir, "config.yaml")) + args := fmt.Sprintf("--config", configFile) err = submitTpl.Execute(f, map[string]interface{}{ "TaskId": task.Id, "WorkDir": workdir, diff --git a/server/tes.go b/server/tes.go index 399c4143a..b1ef8d62e 100644 --- a/server/tes.go +++ b/server/tes.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/ohsu-comp-bio/funnel/config" "github.com/ohsu-comp-bio/funnel/events" "github.com/ohsu-comp-bio/funnel/logger" "github.com/ohsu-comp-bio/funnel/tes" @@ -29,7 +30,7 @@ type TaskService struct { Compute events.Computer Read tes.ReadOnlyServer Log *logger.Logger - HostName string + Config config.Config } // CreateTask provides an HTTP/gRPC endpoint for creating a task. @@ -45,7 +46,7 @@ func (ts *TaskService) CreateTask(ctx context.Context, task *tes.Task) (*tes.Cre return nil, fmt.Errorf("error from backend: %s", err) } - ctx = context.WithValue(ctx, "HostName", ts.HostName) + ctx = context.WithValue(ctx, "Config", ts.Config) if err := ts.Event.WriteEvent(ctx, events.NewTaskCreated(task)); err != nil { return nil, fmt.Errorf("error creating task: %s", err) } From 1d93375577dd4f359f054c6812c1fa1ec4c1affa Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 17 May 2024 12:58:26 -0700 Subject: [PATCH 12/67] Add small fix in HPC Template config [no ci] --- compute/hpc_backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/hpc_backend.go b/compute/hpc_backend.go index d4c519214..4e121faf7 100644 --- a/compute/hpc_backend.go +++ b/compute/hpc_backend.go @@ -264,7 +264,7 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(ctx context.Context, task *tes.Task conf := ctx.Value("Config").(config.Config) configFile := config.ToYamlFile(conf, filepath.Join(workdir, "config.yaml")) - args := fmt.Sprintf("--config", configFile) + args := fmt.Sprintf("--config %v", configFile) err = submitTpl.Execute(f, map[string]interface{}{ "TaskId": task.Id, "WorkDir": workdir, From 1c4221d083a0b348a7e1ce00e9f5e97e32b88c9f Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 17 May 2024 13:02:50 -0700 Subject: [PATCH 13/67] Update worker config path in HPC Template [no ci] --- compute/hpc_backend.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compute/hpc_backend.go b/compute/hpc_backend.go index 4e121faf7..4c8b6c81d 100644 --- a/compute/hpc_backend.go +++ b/compute/hpc_backend.go @@ -263,7 +263,8 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(ctx context.Context, task *tes.Task } conf := ctx.Value("Config").(config.Config) - configFile := config.ToYamlFile(conf, filepath.Join(workdir, "config.yaml")) + configFile := filepath.Join(workdir, "config.yaml") + err = config.ToYamlFile(conf, configFile) args := fmt.Sprintf("--config %v", configFile) err = submitTpl.Execute(f, map[string]interface{}{ "TaskId": task.Id, From 53bb0cb6edbcdfb770e5422f5074ffa94bcc12f1 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 20 May 2024 12:45:52 -0700 Subject: [PATCH 14/67] Add debugging from Exacloud testing --- examples/exacloud.config.yml | 1 + worker/exadocker.go | 29 +++++++++++++++++++++++------ worker/step.go | 2 ++ worker/worker.go | 6 +++++- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/examples/exacloud.config.yml b/examples/exacloud.config.yml index 08badecd0..7cf235201 100644 --- a/examples/exacloud.config.yml +++ b/examples/exacloud.config.yml @@ -36,6 +36,7 @@ Slurm: SCRATCH_PATH="/mnt/scratch/${SLURM_JOB_ID}" cd $SCRATCH_PATH srun bash -c 'funnel worker run --taskID {{.TaskId}} {{.Args}}' --temp-dir $SCRATCH_PATH + sleep 3 cd - srun /usr/local/bin/rmdir-scratch.sh diff --git a/worker/exadocker.go b/worker/exadocker.go index 267ea7866..3dc6dd0a3 100644 --- a/worker/exadocker.go +++ b/worker/exadocker.go @@ -23,16 +23,18 @@ type ExadockerInfo struct { // Run runs the Docker command and blocks until done. func (exadocker Exadocker) Run(ctx context.Context) error { // Sync docker API version info. - err := SyncDockerAPIVersion() - if err != nil { - exadocker.Event.Error("failed to sync docker client API version", err) - } + // err := SyncDockerAPIVersion() + // if err != nil { + // exadocker.Event.Error("failed to sync docker client API version", err) + // } commandArgs := append(exadocker.Driver[1:], "pull", exadocker.Image) pullcmd := exec.Command(exadocker.Driver[0], commandArgs...) + // fmt.Println("DEBUG: exadocker.Driver:", exadocker.Driver) + // fmt.Println("DEBUG: pullcmd:", pullcmd) // Run the command and check for errors. - err = pullcmd.Run() + err := pullcmd.Run() if err != nil { exadocker.Event.Error("failed to pull docker image", err) } @@ -59,7 +61,7 @@ func (exadocker Exadocker) Run(ctx context.Context) error { } for _, vol := range exadocker.Volumes { - arg := formatVolumeArg(vol) + arg := formatExaVolumeArg(vol) args = append(args, "-v", arg) } @@ -68,12 +70,15 @@ func (exadocker Exadocker) Run(ctx context.Context) error { // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` cmd := exec.Command(exadocker.Driver[0], args...) + // fmt.Println("DEBUG: cmd:", cmd) if exadocker.Stdin != nil { cmd.Stdin = exadocker.Stdin } if exadocker.Stdout != nil { + fmt.Println("DEBUG: exadocker.Stdout:", exadocker.Stdout) cmd.Stdout = exadocker.Stdout + fmt.Println("DEBUG: cmd.Stdout:", cmd.Stdout) } if exadocker.Stderr != nil { cmd.Stderr = exadocker.Stderr @@ -93,6 +98,18 @@ func (exadocker Exadocker) Stop() error { return cmd.Run() } +func formatExaVolumeArg(v Volume) string { + // `o` is structed as "HostPath:ContainerPath:Mode". + mode := "rw" + if v.Readonly { + mode = "ro" + } + // fmt.Println("DEBUG: v.HostPath a:", v.HostPath) + v.HostPath = "/mnt/scratch/${SLURM_JOB_ID}" + v.HostPath + // fmt.Println("DEBUG: v.HostPath b:", v.HostPath) + return fmt.Sprintf("%s:%s:%s", v.HostPath, v.ContainerPath, mode) +} + func (exadocker Exadocker) GetImage() string { return exadocker.Image } diff --git a/worker/step.go b/worker/step.go index 165778b0d..2ee4b0bff 100644 --- a/worker/step.go +++ b/worker/step.go @@ -2,6 +2,7 @@ package worker import ( "context" + "fmt" "io" "sync" "time" @@ -43,6 +44,7 @@ func (s *stepWorker) Run(ctx context.Context) error { // Capture stdout/err to file. _, out, err := s.Command.GetIO() + fmt.Println("DEBUG: out:", out) if out != nil { stdout = io.MultiWriter(out, stdout) } diff --git a/worker/worker.go b/worker/worker.go index 44e2b744a..ac02b4eb7 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/ohsu-comp-bio/funnel/config" @@ -134,6 +135,8 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { ignoreError := false f := ContainerEngineFactory{} for i, d := range task.GetExecutors() { + fmt.Println("DEBUG: d:", d) + fmt.Println("DEBUG: d.Stdout:", d.Stdout) containerConfig := ContainerConfig{ Image: d.Image, Command: d.Command, @@ -146,7 +149,7 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { Event: event.NewExecutorWriter(uint32(i)), } if r.Conf.ContainerDriver != "" { - containerConfig.Driver = []string{r.Conf.ContainerDriver} + containerConfig.Driver = strings.Fields(r.Conf.ContainerDriver) } containerEngine, err := f.NewContainerEngine(r.Conf.ContainerType, containerConfig) if err != nil { @@ -237,6 +240,7 @@ func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.E } } + fmt.Println("DEBUG: stdout:", stdout) s.Command.SetIO(stdin, stdout, stderr) return nil From a422b82e2e88fa85d90495ca06568c3292b262cc Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 24 May 2024 14:52:49 -0700 Subject: [PATCH 15/67] Update input/output working example (md5sum) - Test command: ```sh funnel task create examples/md5sum.json ``` [no ci] --- cmd/util/flags.go | 1 + config/config.go | 2 + examples/exacloud.config.yml | 10 +-- worker/container_engine.go | 2 +- worker/exadocker.go | 24 +++++- worker/file_mapper.go | 145 +++++++++++++++++++++++++++++++++-- worker/step.go | 3 +- worker/worker.go | 36 ++++++++- 8 files changed, 202 insertions(+), 21 deletions(-) diff --git a/cmd/util/flags.go b/cmd/util/flags.go index ac952d239..6efca3ffd 100644 --- a/cmd/util/flags.go +++ b/cmd/util/flags.go @@ -100,6 +100,7 @@ func workerFlags(flagConf *config.Config) *pflag.FlagSet { f.Var(&flagConf.Worker.LogUpdateRate, "Worker.LogUpdateRate", "How often to send stdout/stderr log updates") f.Var(&flagConf.Worker.PollingRate, "Worker.PollingRate", "How often to poll for cancel signals") f.StringVar(&flagConf.Worker.WorkDir, "Worker.WorkDir", flagConf.Worker.WorkDir, "Working directory") + f.StringVar(&flagConf.Worker.ScratchPath, "Worker.ScratchPath", flagConf.Worker.ScratchPath, "Scratch directory") f.BoolVar(&flagConf.Worker.LeaveWorkDir, "Worker.LeaveWorkDir", flagConf.Worker.LeaveWorkDir, "Leave working directory after execution") f.StringVar(&flagConf.Worker.ContainerType, "Worker.ContainerType", flagConf.Worker.ContainerType, "Container engine to use for executing tasks. One of ['docker', 'exadocker']") f.StringVar(&flagConf.Worker.ContainerDriver, "Worker.ContainerDriver", flagConf.Worker.ContainerDriver, "Overrides the default command used to run containers.") diff --git a/config/config.go b/config/config.go index 19371be49..66c183581 100644 --- a/config/config.go +++ b/config/config.go @@ -137,6 +137,8 @@ type Node struct { type Worker struct { // Directory to write task files to WorkDir string + // Additional directory to symlink to the working directory. + ScratchPath string // How often the worker should poll for cancel signals PollingRate Duration // How often to update stdout/stderr log fields. diff --git a/examples/exacloud.config.yml b/examples/exacloud.config.yml index 7cf235201..019e21033 100644 --- a/examples/exacloud.config.yml +++ b/examples/exacloud.config.yml @@ -32,11 +32,11 @@ Slurm: {{printf "#SBATCH --tmp %.0fGB" .DiskGb}} {{- end}} - srun /usr/local/bin/mkdir-scratch.sh + srun /usr/local/bin/mkdir-scratch.sh > /dev/null SCRATCH_PATH="/mnt/scratch/${SLURM_JOB_ID}" + cd $SCRATCH_PATH - srun bash -c 'funnel worker run --taskID {{.TaskId}} {{.Args}}' --temp-dir $SCRATCH_PATH - sleep 3 - cd - - srun /usr/local/bin/rmdir-scratch.sh + srun bash -c 'funnel worker run --taskID {{.TaskId}} {{.Args}} --Worker.ScratchPath .' --temp-dir $SCRATCH_PATH + cd - > /dev/null + srun /usr/local/bin/rmdir-scratch.sh > /dev/null diff --git a/worker/container_engine.go b/worker/container_engine.go index 257cf9b38..ddd36d303 100644 --- a/worker/container_engine.go +++ b/worker/container_engine.go @@ -59,7 +59,7 @@ func (f *ContainerEngineFactory) NewContainerEngine(containerType string, contai // } func NewExadockerEngine(config ContainerConfig) (ContainerEngine, error) { - return Exadocker{ + return &Exadocker{ ContainerConfig: config, }, nil } diff --git a/worker/exadocker.go b/worker/exadocker.go index 3dc6dd0a3..843ee23b8 100644 --- a/worker/exadocker.go +++ b/worker/exadocker.go @@ -60,7 +60,9 @@ func (exadocker Exadocker) Run(ctx context.Context) error { args = append(args, "-w", exadocker.Workdir) } + fmt.Println("DEBUG: exadocker.Volumes:", exadocker.Volumes) for _, vol := range exadocker.Volumes { + fmt.Println("DEBUG: vol:", vol) arg := formatExaVolumeArg(vol) args = append(args, "-v", arg) } @@ -69,23 +71,30 @@ func (exadocker Exadocker) Run(ctx context.Context) error { args = append(args, exadocker.Command...) // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` + fmt.Println("DEBUG: exadocker:", exadocker) cmd := exec.Command(exadocker.Driver[0], args...) - // fmt.Println("DEBUG: cmd:", cmd) + fmt.Println("DEBUG: cmd:", cmd) if exadocker.Stdin != nil { cmd.Stdin = exadocker.Stdin } + fmt.Println("DEBUG: exadocker.Stdout:", exadocker.Stdout) if exadocker.Stdout != nil { - fmt.Println("DEBUG: exadocker.Stdout:", exadocker.Stdout) cmd.Stdout = exadocker.Stdout + // cmd.Stdout = os.Stdout fmt.Println("DEBUG: cmd.Stdout:", cmd.Stdout) } + fmt.Println("DEBUG: exadocker.Stderr:", exadocker.Stderr) if exadocker.Stderr != nil { cmd.Stderr = exadocker.Stderr + fmt.Println("DEBUG: cmd.Stderr:", cmd.Stderr) } go exadocker.inspectContainer(ctx) + + // fmt.Println("DEBUG: exadocker.Stdout:", exadocker.Stdout) out := cmd.Run() exadocker.Event.Info("Command %s Complete exit=%s", strings.Join(args, " "), out) + fmt.Println("DEBUG: out:", out) return out } @@ -105,7 +114,7 @@ func formatExaVolumeArg(v Volume) string { mode = "ro" } // fmt.Println("DEBUG: v.HostPath a:", v.HostPath) - v.HostPath = "/mnt/scratch/${SLURM_JOB_ID}" + v.HostPath + // v.HostPath = "/mnt/scratch/${SLURM_JOB_ID}" + v.HostPath // fmt.Println("DEBUG: v.HostPath b:", v.HostPath) return fmt.Sprintf("%s:%s:%s", v.HostPath, v.ContainerPath, mode) } @@ -118,7 +127,11 @@ func (exadocker Exadocker) GetIO() (io.Reader, io.Writer, io.Writer) { return exadocker.Stdin, exadocker.Stdout, exadocker.Stderr } -func (exadocker Exadocker) SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) { +func (exadocker *Exadocker) SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) { + fmt.Println("DEBUG: In SetIO...") + fmt.Println("DEBUG: exadocker.go stdin:", stdin) + fmt.Println("DEBUG: exadocker.go stdout:", stdout) + fmt.Println("DEBUG: exadocker.go stderr:", stderr) if stdin != nil { exadocker.Stdin = stdin } @@ -128,6 +141,9 @@ func (exadocker Exadocker) SetIO(stdin io.Reader, stdout io.Writer, stderr io.Wr if stderr != nil { exadocker.Stderr = stderr } + fmt.Println("DEBUG: exadocker.Stdin:", exadocker.Stdin) + fmt.Println("DEBUG: exadocker.Stdout:", exadocker.Stdout) + fmt.Println("DEBUG: exadocker.Stderr:", exadocker.Stderr) } func (exadocker Exadocker) Inspect(ctx context.Context) (ContainerConfig, error) { diff --git a/worker/file_mapper.go b/worker/file_mapper.go index 4219c724d..3f3417ecd 100644 --- a/worker/file_mapper.go +++ b/worker/file_mapper.go @@ -2,6 +2,7 @@ package worker import ( "fmt" + "io" "io/ioutil" "os" "path" @@ -21,10 +22,11 @@ import ( // outputs, uploads, stdin/out/err, etc. FileMapper helps the worker engine // manage all these paths. type FileMapper struct { - Volumes []Volume - Inputs []*tes.Input - Outputs []*tes.Output - WorkDir string + Volumes []Volume + Inputs []*tes.Input + Outputs []*tes.Output + WorkDir string + ScratchDir string } // Volume represents a volume mounted into a docker container. @@ -96,6 +98,124 @@ func (mapper *FileMapper) MapTask(task *tes.Task) error { return nil } +func (mapper *FileMapper) CopyInputsToScratch(scratchDir string) error { + scratchAbsDir, err := filepath.Abs(scratchDir) + if err != nil { + return err + } + + // Copy the input file or directory to the scratch target + for _, input := range mapper.Inputs { + scratchTarget := filepath.Join(scratchAbsDir, input.Path) + + info, err := os.Stat(input.Path) + if err != nil { + fmt.Println(err) + } + if info.IsDir() { + // Ensure the scratch target directory exists + if err := os.MkdirAll(scratchTarget, 0755); err != nil { + return fmt.Errorf("failed to create scratch directory: %w", err) + } + copyDir(input.Path, scratchTarget) + } else { + // Ensure the scratch target directory exists + if err := os.MkdirAll(path.Dir(scratchTarget), 0755); err != nil { + return fmt.Errorf("failed to create scratch directory: %w", err) + } + err = copyFile(input.Path, scratchTarget) + dat, err := os.ReadFile(scratchTarget) + } + } + + return nil +} + +func (mapper *FileMapper) CopyOutputsToWorkDir(scratchDir string) error { + scratchAbsDir, err := filepath.Abs(scratchDir) + if err != nil { + return err + } + err = filepath.Walk(scratchAbsDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + return nil + }) + if err != nil { + fmt.Println(err) + } + + // Copy the input file or directory to the scratch target + for _, output := range mapper.Outputs { + scratchTarget := filepath.Join(scratchAbsDir, output.Path) + + info, err := os.Stat(scratchTarget) + if err != nil { + fmt.Println(err) + } + if info.IsDir() { + // Ensure the scratch target directory exists + if err := os.MkdirAll(output.Path, 0755); err != nil { + return fmt.Errorf("failed to create output path: %w", err) + } + copyDir(scratchTarget, output.Path) + } else { + // Ensure the scratch target directory exists + if err := os.MkdirAll(path.Dir(output.Path), 0755); err != nil { + return fmt.Errorf("failed to create output path: %w", err) + } + err = copyFile(scratchTarget, output.Path) + } + } + + return nil +} + +func copyDir(src string, dst string) error { + entries, err := os.ReadDir(src) + if err != nil { + return err + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + if err := os.MkdirAll(dstPath, 0755); err != nil { + return err + } + if err := copyDir(srcPath, dstPath); err != nil { + return err + } + } else { + if err := copyFile(srcPath, dstPath); err != nil { + return err + } + } + } + return nil +} + +func copyFile(src string, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return err + } + defer sourceFile.Close() + + destFile, err := os.Create(dst) + if err != nil { + return err + } + defer destFile.Close() + + _, err = io.Copy(destFile, sourceFile) + return err +} + // AddVolume adds a mapped volume to the mapper. A corresponding Volume record // is added to mapper.Volumes. // @@ -148,6 +268,15 @@ func (mapper *FileMapper) HostPath(src string) (string, error) { return p, nil } +func (mapper *FileMapper) HostScratchPath(src string) (string, error) { + p := path.Join(mapper.ScratchDir, src) + p = path.Clean(p) + if !mapper.IsSubpath(p, mapper.ScratchDir) { + return "", fmt.Errorf("Invalid path: %s is not a valid subpath of %s", p, mapper.ScratchDir) + } + return p, nil +} + // OpenHostFile opens a file on the host file system at a mapped path. // "src" is an unmapped path. This function will handle mapping the path. // @@ -173,7 +302,13 @@ func (mapper *FileMapper) OpenHostFile(src string) (*os.File, error) { // // If the path can't be mapped or the file can't be created, an error is returned. func (mapper *FileMapper) CreateHostFile(src string) (*os.File, error) { - p, perr := mapper.HostPath(src) + var p string + var perr error + if mapper.ScratchDir != "" { + p, perr = mapper.HostScratchPath(src) + } else { + p, perr = mapper.HostPath(src) + } if perr != nil { return nil, perr } diff --git a/worker/step.go b/worker/step.go index 2ee4b0bff..3b6be7562 100644 --- a/worker/step.go +++ b/worker/step.go @@ -2,7 +2,6 @@ package worker import ( "context" - "fmt" "io" "sync" "time" @@ -44,7 +43,6 @@ func (s *stepWorker) Run(ctx context.Context) error { // Capture stdout/err to file. _, out, err := s.Command.GetIO() - fmt.Println("DEBUG: out:", out) if out != nil { stdout = io.MultiWriter(out, stdout) } @@ -52,6 +50,7 @@ func (s *stepWorker) Run(ctx context.Context) error { stderr = io.MultiWriter(err, stderr) } s.Command.SetIO(nil, stdout, stderr) + _, out, err = s.Command.GetIO() go func() { done <- s.Command.Run(subctx) diff --git a/worker/worker.go b/worker/worker.go index ac02b4eb7..3b0717f71 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -60,6 +60,13 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { // set up task specific utilities event = events.NewTaskWriter(task.GetId(), 0, r.EventWriter) mapper = NewFileMapper(filepath.Join(r.Conf.WorkDir, task.GetId())) + if r.Conf.ScratchPath != "" { + scratchAbsDir, err := filepath.Abs(r.Conf.ScratchPath) + if err != nil { + return err + } + mapper.ScratchDir = filepath.Join(scratchAbsDir, filepath.Join(r.Conf.WorkDir, task.GetId())) + } event.Info("Version", version.LogFields()...) event.State(tes.State_INITIALIZING) @@ -130,13 +137,30 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { event.State(tes.State_RUNNING) } + // Create symlinks between the working directory and the Scratch Directory + if run.ok() && r.Conf.ScratchPath != "" { + err := mapper.CopyInputsToScratch(r.Conf.ScratchPath) + if err != nil { + return err + } + // Get the absolute path of the scratch directory + scratchAbsDir, err := filepath.Abs(r.Conf.ScratchPath) + if err != nil { + return err + } + // For every volume in mapper, add the Scratch Path prefix to the host path + for idx, v := range mapper.Volumes { + v.HostPath = filepath.Join(scratchAbsDir, v.HostPath) + mapper.Volumes[idx] = v + } + + } + // Run steps if run.ok() { ignoreError := false f := ContainerEngineFactory{} for i, d := range task.GetExecutors() { - fmt.Println("DEBUG: d:", d) - fmt.Println("DEBUG: d.Stdout:", d.Stdout) containerConfig := ContainerConfig{ Image: d.Image, Command: d.Command, @@ -182,6 +206,11 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { } } + if run.ok() && r.Conf.ScratchPath != "" { + fmt.Println("Copying Outputs to WorkDir...") + mapper.CopyOutputsToWorkDir(r.Conf.ScratchPath) + } + // Upload outputs var outputLog []*tes.OutputFileLog if run.ok() { @@ -215,7 +244,7 @@ func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.E // Find the path for task stdin if d.Stdin != "" { - stdin, err = mapper.OpenHostFile(d.Stdin) + stdin, err = mapper.CreateHostFile(d.Stdin) if err != nil { s.Event.Error("Couldn't prepare log files", err) return err @@ -240,7 +269,6 @@ func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.E } } - fmt.Println("DEBUG: stdout:", stdout) s.Command.SetIO(stdin, stdout, stderr) return nil From 021eeef222161aa6c60233cdd4a8a0a8dde9d8e7 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 30 May 2024 11:28:04 -0700 Subject: [PATCH 16/67] Update docker.go to implement ContainerEngine interface - Can be used in HPC environment with unique docker setups (e.g. exadocker) as well as future command templating efforts. [no ci] --- config/default.go | 1 + storage/local.go | 2 +- worker/container_engine.go | 30 ++++++--- worker/docker.go | 130 +++++++++++++++++++------------------ worker/exadocker.go | 39 ++++++++--- worker/file_mapper.go | 2 +- worker/step.go | 5 +- 7 files changed, 127 insertions(+), 82 deletions(-) diff --git a/config/default.go b/config/default.go index 511fa1abc..97ff07364 100644 --- a/config/default.go +++ b/config/default.go @@ -62,6 +62,7 @@ func DefaultConfig() Config { LogTailSize: 10000, MaxParallelTransfers: 10, ContainerType: "docker", + ContainerDriver: "docker", }, Logger: logger.DefaultConfig(), // databases / event handlers diff --git a/storage/local.go b/storage/local.go index 75f6401bc..0f56d079d 100644 --- a/storage/local.go +++ b/storage/local.go @@ -112,7 +112,7 @@ func (local *Local) UnsupportedOperations(url string) UnsupportedOperations { path := getPath(url) if !isAllowed(path, local.allowedDirs) { err := fmt.Errorf( - "localStorage: can't access file, path is not in allowed directories: %s", url) + "localStorage: can't access file, path is not in allowed directories: %s, %s", url, local.allowedDirs) return AllUnsupported(err) } return AllSupported() diff --git a/worker/container_engine.go b/worker/container_engine.go index ddd36d303..a2a6b483b 100644 --- a/worker/container_engine.go +++ b/worker/container_engine.go @@ -9,17 +9,26 @@ import ( ) type ContainerEngine interface { + // Run runs the container. Run(ctx context.Context) error + // Stop stops the container. Stop() error - Inspect(ctx context.Context) (ContainerConfig, error) + // Inspect returns the container configuration. + InspectContainer(ctx context.Context) ContainerConfig + // GetImage returns the image name for the container. GetImage() string + // GetIO returns the stdin, stdout, and stderr for the container. GetIO() (io.Reader, io.Writer, io.Writer) + // SetIO sets the stdin, stdout, and stderr for the container. SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) + + // SyncAPIVersions ensures that the client uses the same API version as the server. + SyncAPIVersion() error } type ContainerConfig struct { @@ -39,12 +48,17 @@ type ContainerConfig struct { Event *events.ExecutorWriter } +type ContainerVersion struct { + Client string + Server string +} + type ContainerEngineFactory struct{} func (f *ContainerEngineFactory) NewContainerEngine(containerType string, containerConfig ContainerConfig) (ContainerEngine, error) { switch containerType { - // case "docker": - // return NewDockerEngine(containerConfig) + case "docker": + return NewDockerEngine(containerConfig) case "exadocker": return NewExadockerEngine(containerConfig) default: @@ -52,11 +66,11 @@ func (f *ContainerEngineFactory) NewContainerEngine(containerType string, contai } } -// func NewDockerEngine(config ContainerConfig) (ContainerEngine, error) { -// return Docker{ -// ContainerConfig: config, -// }, nil -// } +func NewDockerEngine(config ContainerConfig) (ContainerEngine, error) { + return &Docker{ + ContainerConfig: config, + }, nil +} func NewExadockerEngine(config ContainerConfig) (ContainerEngine, error) { return &Exadocker{ diff --git a/worker/docker.go b/worker/docker.go index 1dd928e6b..2368824e4 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -9,91 +9,86 @@ import ( "os/exec" "strings" "time" - - "github.com/ohsu-comp-bio/funnel/events" ) -// DockerCommand is responsible for configuring and running a docker container. -type DockerCommand struct { - Image string - Command []string - Volumes []Volume - Workdir string - ContainerName string - RemoveContainer bool - Env map[string]string - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer - Event *events.ExecutorWriter +// Docker is responsible for configuring and running a docker container. +type Docker struct { + ContainerConfig } // Run runs the Docker command and blocks until done. -func (dcmd DockerCommand) Run(ctx context.Context) error { +func (docker Docker) Run(ctx context.Context) error { // Sync docker API version info. - err := SyncDockerAPIVersion() + err := docker.SyncAPIVersion() if err != nil { - dcmd.Event.Error("failed to sync docker client API version", err) + docker.Event.Error("failed to sync docker client API version", err) } - pullcmd := exec.Command("docker", "pull", dcmd.Image) + pullcmd := exec.Command("docker", "pull", docker.Image) err = pullcmd.Run() if err != nil { - dcmd.Event.Error("failed to pull docker image", err) + docker.Event.Error("failed to pull docker image", err) + } + + var args []string + + if len(docker.ContainerConfig.Driver) > 1 { + // Merge driver parts and command parts + args = append(args, docker.ContainerConfig.Driver[1:]...) } - args := []string{"run", "-i", "--read-only"} + args = append(args, "run", "-i", "--read-only") - if dcmd.RemoveContainer { + if docker.RemoveContainer { args = append(args, "--rm") } - if dcmd.Env != nil { - for k, v := range dcmd.Env { + if docker.Env != nil { + for k, v := range docker.Env { args = append(args, "-e", fmt.Sprintf("%s=%s", k, v)) } } - if dcmd.ContainerName != "" { - args = append(args, "--name", dcmd.ContainerName) + if docker.ContainerName != "" { + args = append(args, "--name", docker.ContainerName) } - if dcmd.Workdir != "" { - args = append(args, "-w", dcmd.Workdir) + if docker.Workdir != "" { + args = append(args, "-w", docker.Workdir) } - for _, vol := range dcmd.Volumes { + for _, vol := range docker.Volumes { arg := formatVolumeArg(vol) args = append(args, "-v", arg) } - args = append(args, dcmd.Image) - args = append(args, dcmd.Command...) + args = append(args, docker.Image) + args = append(args, docker.Command...) // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` - dcmd.Event.Info("Running command", "cmd", "docker "+strings.Join(args, " ")) - cmd := exec.Command("docker", args...) + docker.Event.Info("Running command", "cmd", docker.ContainerConfig.Driver[0]+" "+strings.Join(args, " ")) + cmd := exec.Command(docker.ContainerConfig.Driver[0], args...) - if dcmd.Stdin != nil { - cmd.Stdin = dcmd.Stdin + if docker.Stdin != nil { + cmd.Stdin = docker.Stdin } - if dcmd.Stdout != nil { - cmd.Stdout = dcmd.Stdout + if docker.Stdout != nil { + cmd.Stdout = docker.Stdout } - if dcmd.Stderr != nil { - cmd.Stderr = dcmd.Stderr + if docker.Stderr != nil { + cmd.Stderr = docker.Stderr } - go dcmd.inspectContainer(ctx) + go docker.InspectContainer(ctx) out := cmd.Run() - dcmd.Event.Info("Command %s Complete exit=%s", strings.Join(args, " "), out) + docker.Event.Info("Command %s Complete exit=%s", strings.Join(args, " "), out) return out } // Stop stops the container. -func (dcmd DockerCommand) Stop() error { - dcmd.Event.Info("Stopping container", "container", dcmd.ContainerName) - // cmd := exec.Command("docker", "stop", dcmd.ContainerName) - cmd := exec.Command("docker", "rm", "-f", dcmd.ContainerName) //switching to this to be a bit more forceful +func (docker Docker) Stop() error { + docker.Event.Info("Stopping container", "container", docker.ContainerName) + // cmd := exec.Command("docker", "stop", docker.ContainerName) + cmd := exec.Command("docker", "rm", "-f", docker.ContainerName) //switching to this to be a bit more forceful return cmd.Run() } @@ -106,14 +101,28 @@ func formatVolumeArg(v Volume) string { return fmt.Sprintf("%s:%s:%s", v.HostPath, v.ContainerPath, mode) } -type metadata struct { - ID string - Name string - Image string +func (docker Docker) GetImage() string { + return docker.Image +} + +func (docker Docker) GetIO() (io.Reader, io.Writer, io.Writer) { + return docker.Stdin, docker.Stdout, docker.Stderr +} + +func (docker *Docker) SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) { + if stdin != nil && stdin != (*os.File)(nil) { + docker.Stdin = stdin + } + if stdout != nil && stdout != (*os.File)(nil) { + docker.Stdout = stdout + } + if stderr != nil && stderr != (*os.File)(nil) { + docker.Stderr = stderr + } } // inspectContainer inspects the docker container for metadata. -func (dcmd *DockerCommand) inspectContainer(ctx context.Context) { +func (docker *Docker) InspectContainer(ctx context.Context) ContainerConfig { // Give the container time to start. time.Sleep(2 * time.Second) @@ -124,40 +133,37 @@ func (dcmd *DockerCommand) inspectContainer(ctx context.Context) { for i := 0; i < 5; i++ { select { case <-ctx.Done(): - return + return ContainerConfig{} case <-ticker.C: - cmd := exec.CommandContext(ctx, "docker", "inspect", dcmd.ContainerName) + cmd := exec.CommandContext(ctx, "docker", "inspect", docker.ContainerName) out, err := cmd.Output() if err == nil { - meta := []metadata{} + meta := []ContainerConfig{} err := json.Unmarshal(out, &meta) if err == nil && len(meta) == 1 { - dcmd.Event.Info("container metadata", - "containerID", meta[0].ID, + docker.Event.Info("container metadata", + "containerID", meta[0].Id, "containerName", meta[0].Name, "containerImageHash", meta[0].Image) - return + return meta[0] } } } } -} -type dockerVersion struct { - Client string - Server string + return ContainerConfig{} } // SyncDockerAPIVersion ensures that the client uses the same API version as // the server. -func SyncDockerAPIVersion() error { +func (docker *Docker) SyncAPIVersion() error { if os.Getenv("DOCKER_API_VERSION") == "" { cmd := exec.Command("docker", "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) out, err := cmd.Output() if err != nil { return fmt.Errorf("docker version command failed: %v", err) } - version := &dockerVersion{} + version := &ContainerVersion{} err = json.Unmarshal(out, version) if err != nil { return fmt.Errorf("failed to unmarshal docker version: %v", err) diff --git a/worker/exadocker.go b/worker/exadocker.go index 843ee23b8..c806201a0 100644 --- a/worker/exadocker.go +++ b/worker/exadocker.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "os" "os/exec" "strings" "time" @@ -89,7 +90,7 @@ func (exadocker Exadocker) Run(ctx context.Context) error { cmd.Stderr = exadocker.Stderr fmt.Println("DEBUG: cmd.Stderr:", cmd.Stderr) } - go exadocker.inspectContainer(ctx) + go exadocker.InspectContainer(ctx) // fmt.Println("DEBUG: exadocker.Stdout:", exadocker.Stdout) out := cmd.Run() @@ -156,7 +157,7 @@ func (exadocker Exadocker) Inspect(ctx context.Context) (ContainerConfig, error) } // inspectContainer inspects the docker container for metadata. -func (exadocker *Exadocker) inspectContainer(ctx context.Context) { +func (exadocker *Exadocker) InspectContainer(ctx context.Context) ContainerConfig { // Give the container time to start. time.Sleep(2 * time.Second) @@ -167,22 +168,44 @@ func (exadocker *Exadocker) inspectContainer(ctx context.Context) { for i := 0; i < 5; i++ { select { case <-ctx.Done(): - return + return ContainerConfig{} case <-ticker.C: - driverArgs := strings.Join(exadocker.Driver[1:], " ") - cmd := exec.CommandContext(ctx, exadocker.Command[0], driverArgs, "inspect", exadocker.ContainerName) + cmd := exec.CommandContext(ctx, "docker", "inspect", exadocker.ContainerName) out, err := cmd.Output() if err == nil { - meta := []metadata{} + meta := []ContainerConfig{} err := json.Unmarshal(out, &meta) if err == nil && len(meta) == 1 { exadocker.Event.Info("container metadata", - "containerID", meta[0].ID, + "containerID", meta[0].Id, "containerName", meta[0].Name, "containerImageHash", meta[0].Image) - return + return meta[0] } } } } + + return ContainerConfig{} +} + +// SyncDockerAPIVersion ensures that the client uses the same API version as +// the server. +func (exadocker *Exadocker) SyncAPIVersion() error { + if os.Getenv("DOCKER_API_VERSION") == "" { + cmd := exec.Command("docker", "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("docker version command failed: %v", err) + } + version := &ContainerVersion{} + err = json.Unmarshal(out, version) + if err != nil { + return fmt.Errorf("failed to unmarshal docker version: %v", err) + } + if version.Client != version.Server { + os.Setenv("DOCKER_API_VERSION", version.Server) + } + } + return nil } diff --git a/worker/file_mapper.go b/worker/file_mapper.go index 3f3417ecd..8b2dfd33b 100644 --- a/worker/file_mapper.go +++ b/worker/file_mapper.go @@ -124,7 +124,7 @@ func (mapper *FileMapper) CopyInputsToScratch(scratchDir string) error { return fmt.Errorf("failed to create scratch directory: %w", err) } err = copyFile(input.Path, scratchTarget) - dat, err := os.ReadFile(scratchTarget) + _, _ = os.ReadFile(scratchTarget) } } diff --git a/worker/step.go b/worker/step.go index 3b6be7562..42e1c8ba2 100644 --- a/worker/step.go +++ b/worker/step.go @@ -3,6 +3,7 @@ package worker import ( "context" "io" + "os" "sync" "time" @@ -43,10 +44,10 @@ func (s *stepWorker) Run(ctx context.Context) error { // Capture stdout/err to file. _, out, err := s.Command.GetIO() - if out != nil { + if out != nil && out != (*os.File)(nil) { stdout = io.MultiWriter(out, stdout) } - if err != nil { + if err != nil && err != (*os.File)(nil) { stderr = io.MultiWriter(err, stderr) } s.Command.SetIO(nil, stdout, stderr) From 8e2474c57e59a7c1101bb1c99b7c0566672a2701 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 30 May 2024 16:50:41 -0700 Subject: [PATCH 17/67] Remove exadocker implementation in favor of docker --- compute/hpc_backend.go | 3 + examples/exacloud.config.yml | 42 ------- worker/container_engine.go | 8 -- worker/exadocker.go | 211 ----------------------------------- worker/exadocker_test.go | 29 ----- worker/worker.go | 1 - 6 files changed, 3 insertions(+), 291 deletions(-) delete mode 100644 examples/exacloud.config.yml delete mode 100644 worker/exadocker.go delete mode 100644 worker/exadocker_test.go diff --git a/compute/hpc_backend.go b/compute/hpc_backend.go index 4c8b6c81d..309a1b44e 100644 --- a/compute/hpc_backend.go +++ b/compute/hpc_backend.go @@ -265,6 +265,9 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(ctx context.Context, task *tes.Task conf := ctx.Value("Config").(config.Config) configFile := filepath.Join(workdir, "config.yaml") err = config.ToYamlFile(conf, configFile) + if err != nil { + return "", err + } args := fmt.Sprintf("--config %v", configFile) err = submitTpl.Execute(f, map[string]interface{}{ "TaskId": task.Id, diff --git a/examples/exacloud.config.yml b/examples/exacloud.config.yml deleted file mode 100644 index 019e21033..000000000 --- a/examples/exacloud.config.yml +++ /dev/null @@ -1,42 +0,0 @@ -LocalStorage: - # Whitelist of local directory paths which Funnel is allowed to access. - AllowedDirs: - - ./ - - /tmp - -Worker: - LeaveWorkDir: true - ContainerType: exadocker - ContainerDriver: sudo /opt/acc/sbin/exadocker - -Server: - HostName: exahead1 - -Compute: slurm - -Slurm: - Template: | - #!/bin/bash - #SBATCH --job-name {{.TaskId}} - #SBATCH --ntasks 1 - #SBATCH --error {{.WorkDir}}/funnel-stderr - #SBATCH --output {{.WorkDir}}/funnel-stdout - #SBATCH --gres disk:1 - {{if ne .Cpus 0 -}} - {{printf "#SBATCH --cpus-per-task %d" .Cpus}} - {{- end}} - {{if ne .RamGb 0.0 -}} - {{printf "#SBATCH --mem %.0fGB" .RamGb}} - {{- end}} - {{if ne .DiskGb 0.0 -}} - {{printf "#SBATCH --tmp %.0fGB" .DiskGb}} - {{- end}} - - srun /usr/local/bin/mkdir-scratch.sh > /dev/null - SCRATCH_PATH="/mnt/scratch/${SLURM_JOB_ID}" - - cd $SCRATCH_PATH - srun bash -c 'funnel worker run --taskID {{.TaskId}} {{.Args}} --Worker.ScratchPath .' --temp-dir $SCRATCH_PATH - - cd - > /dev/null - srun /usr/local/bin/rmdir-scratch.sh > /dev/null diff --git a/worker/container_engine.go b/worker/container_engine.go index a2a6b483b..e5c79f962 100644 --- a/worker/container_engine.go +++ b/worker/container_engine.go @@ -59,8 +59,6 @@ func (f *ContainerEngineFactory) NewContainerEngine(containerType string, contai switch containerType { case "docker": return NewDockerEngine(containerConfig) - case "exadocker": - return NewExadockerEngine(containerConfig) default: return nil, fmt.Errorf("unsupported container type: %s", containerType) } @@ -71,9 +69,3 @@ func NewDockerEngine(config ContainerConfig) (ContainerEngine, error) { ContainerConfig: config, }, nil } - -func NewExadockerEngine(config ContainerConfig) (ContainerEngine, error) { - return &Exadocker{ - ContainerConfig: config, - }, nil -} diff --git a/worker/exadocker.go b/worker/exadocker.go deleted file mode 100644 index c806201a0..000000000 --- a/worker/exadocker.go +++ /dev/null @@ -1,211 +0,0 @@ -package worker - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - "os/exec" - "strings" - "time" -) - -type Exadocker struct { - ContainerConfig -} - -type ExadockerInfo struct { - Id string - Image string - Name string -} - -// Run runs the Docker command and blocks until done. -func (exadocker Exadocker) Run(ctx context.Context) error { - // Sync docker API version info. - // err := SyncDockerAPIVersion() - // if err != nil { - // exadocker.Event.Error("failed to sync docker client API version", err) - // } - - commandArgs := append(exadocker.Driver[1:], "pull", exadocker.Image) - pullcmd := exec.Command(exadocker.Driver[0], commandArgs...) - // fmt.Println("DEBUG: exadocker.Driver:", exadocker.Driver) - // fmt.Println("DEBUG: pullcmd:", pullcmd) - - // Run the command and check for errors. - err := pullcmd.Run() - if err != nil { - exadocker.Event.Error("failed to pull docker image", err) - } - - args := exadocker.Driver[1:] - args = append(args, "run", "-i", "--read-only") - - if exadocker.RemoveContainer { - args = append(args, "--rm") - } - - if exadocker.Env != nil { - for k, v := range exadocker.Env { - args = append(args, "-e", fmt.Sprintf("%s=%s", k, v)) - } - } - - if exadocker.ContainerName != "" { - args = append(args, "--name", exadocker.ContainerName) - } - - if exadocker.Workdir != "" { - args = append(args, "-w", exadocker.Workdir) - } - - fmt.Println("DEBUG: exadocker.Volumes:", exadocker.Volumes) - for _, vol := range exadocker.Volumes { - fmt.Println("DEBUG: vol:", vol) - arg := formatExaVolumeArg(vol) - args = append(args, "-v", arg) - } - - args = append(args, exadocker.GetImage()) - args = append(args, exadocker.Command...) - - // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` - fmt.Println("DEBUG: exadocker:", exadocker) - cmd := exec.Command(exadocker.Driver[0], args...) - fmt.Println("DEBUG: cmd:", cmd) - - if exadocker.Stdin != nil { - cmd.Stdin = exadocker.Stdin - } - fmt.Println("DEBUG: exadocker.Stdout:", exadocker.Stdout) - if exadocker.Stdout != nil { - cmd.Stdout = exadocker.Stdout - // cmd.Stdout = os.Stdout - fmt.Println("DEBUG: cmd.Stdout:", cmd.Stdout) - } - fmt.Println("DEBUG: exadocker.Stderr:", exadocker.Stderr) - if exadocker.Stderr != nil { - cmd.Stderr = exadocker.Stderr - fmt.Println("DEBUG: cmd.Stderr:", cmd.Stderr) - } - go exadocker.InspectContainer(ctx) - - // fmt.Println("DEBUG: exadocker.Stdout:", exadocker.Stdout) - out := cmd.Run() - exadocker.Event.Info("Command %s Complete exit=%s", strings.Join(args, " "), out) - fmt.Println("DEBUG: out:", out) - return out -} - -// Stop stops the container. -func (exadocker Exadocker) Stop() error { - exadocker.Event.Info("Stopping container", "container", exadocker.ContainerName) - // cmd := exec.Command("docker", "stop", exa.ContainerName) - driverArgs := strings.Join(exadocker.Driver[1:], " ") - cmd := exec.Command(exadocker.Driver[0], driverArgs, "rm", "-f", exadocker.ContainerName) //switching to this to be a bit more forceful - return cmd.Run() -} - -func formatExaVolumeArg(v Volume) string { - // `o` is structed as "HostPath:ContainerPath:Mode". - mode := "rw" - if v.Readonly { - mode = "ro" - } - // fmt.Println("DEBUG: v.HostPath a:", v.HostPath) - // v.HostPath = "/mnt/scratch/${SLURM_JOB_ID}" + v.HostPath - // fmt.Println("DEBUG: v.HostPath b:", v.HostPath) - return fmt.Sprintf("%s:%s:%s", v.HostPath, v.ContainerPath, mode) -} - -func (exadocker Exadocker) GetImage() string { - return exadocker.Image -} - -func (exadocker Exadocker) GetIO() (io.Reader, io.Writer, io.Writer) { - return exadocker.Stdin, exadocker.Stdout, exadocker.Stderr -} - -func (exadocker *Exadocker) SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) { - fmt.Println("DEBUG: In SetIO...") - fmt.Println("DEBUG: exadocker.go stdin:", stdin) - fmt.Println("DEBUG: exadocker.go stdout:", stdout) - fmt.Println("DEBUG: exadocker.go stderr:", stderr) - if stdin != nil { - exadocker.Stdin = stdin - } - if stdout != nil { - exadocker.Stdout = stdout - } - if stderr != nil { - exadocker.Stderr = stderr - } - fmt.Println("DEBUG: exadocker.Stdin:", exadocker.Stdin) - fmt.Println("DEBUG: exadocker.Stdout:", exadocker.Stdout) - fmt.Println("DEBUG: exadocker.Stderr:", exadocker.Stderr) -} - -func (exadocker Exadocker) Inspect(ctx context.Context) (ContainerConfig, error) { - info := ContainerConfig{ - Id: "1234", - Image: "image", - Name: "container", - } - return info, nil -} - -// inspectContainer inspects the docker container for metadata. -func (exadocker *Exadocker) InspectContainer(ctx context.Context) ContainerConfig { - // Give the container time to start. - time.Sleep(2 * time.Second) - - // Inspect the container for metadata - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - - for i := 0; i < 5; i++ { - select { - case <-ctx.Done(): - return ContainerConfig{} - case <-ticker.C: - cmd := exec.CommandContext(ctx, "docker", "inspect", exadocker.ContainerName) - out, err := cmd.Output() - if err == nil { - meta := []ContainerConfig{} - err := json.Unmarshal(out, &meta) - if err == nil && len(meta) == 1 { - exadocker.Event.Info("container metadata", - "containerID", meta[0].Id, - "containerName", meta[0].Name, - "containerImageHash", meta[0].Image) - return meta[0] - } - } - } - } - - return ContainerConfig{} -} - -// SyncDockerAPIVersion ensures that the client uses the same API version as -// the server. -func (exadocker *Exadocker) SyncAPIVersion() error { - if os.Getenv("DOCKER_API_VERSION") == "" { - cmd := exec.Command("docker", "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) - out, err := cmd.Output() - if err != nil { - return fmt.Errorf("docker version command failed: %v", err) - } - version := &ContainerVersion{} - err = json.Unmarshal(out, version) - if err != nil { - return fmt.Errorf("failed to unmarshal docker version: %v", err) - } - if version.Client != version.Server { - os.Setenv("DOCKER_API_VERSION", version.Server) - } - } - return nil -} diff --git a/worker/exadocker_test.go b/worker/exadocker_test.go deleted file mode 100644 index 6a25d8f82..000000000 --- a/worker/exadocker_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package worker - -import ( - "bytes" - "context" - "testing" - - "github.com/ohsu-comp-bio/funnel/events" - "github.com/stretchr/testify/assert" -) - -func TestExadockerRun(t *testing.T) { - var buf bytes.Buffer - - containerConfig := ContainerConfig{ - Image: "ubuntu", - Driver: []string{"docker"}, - Command: []string{"echo", "hello"}, - Stdout: &buf, - Event: events.NewExecutorWriter("task-1", 0, 0, events.Noop{}), - } - - f := ContainerEngineFactory{} - exadocker, err := f.NewContainerEngine("exadocker", containerConfig) - - err = exadocker.Run(context.Background()) - assert.NoError(t, err) - assert.Contains(t, buf.String(), "hello") -} diff --git a/worker/worker.go b/worker/worker.go index 3b0717f71..3b1201b37 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -207,7 +207,6 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { } if run.ok() && r.Conf.ScratchPath != "" { - fmt.Println("Copying Outputs to WorkDir...") mapper.CopyOutputsToWorkDir(r.Conf.ScratchPath) } From f5acbabe85d28f45f2f86db959c4d7bc929334a4 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 30 May 2024 16:51:29 -0700 Subject: [PATCH 18/67] Add initial HPC documentation and example --- website/config.yaml | 5 + website/content/docs/interop/exacloud.md | 15 --- website/content/docs/interop/hpc.md | 117 +++++++++++++++++++++++ website/layouts/_default/single.html | 2 +- 4 files changed, 123 insertions(+), 16 deletions(-) delete mode 100644 website/content/docs/interop/exacloud.md create mode 100644 website/content/docs/interop/hpc.md diff --git a/website/config.yaml b/website/config.yaml index 6dd8c8858..3393f0df9 100644 --- a/website/config.yaml +++ b/website/config.yaml @@ -13,3 +13,8 @@ security: funcs: getenv: - ^FUNNEL_ + +markup: + goldmark: + renderer: + unsafe: true diff --git a/website/content/docs/interop/exacloud.md b/website/content/docs/interop/exacloud.md deleted file mode 100644 index 8c9cd4c75..000000000 --- a/website/content/docs/interop/exacloud.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Exacloud -menu: - main: - parent: Interop ---- - -> ⚠️ Exacloud support is currently in development and requires a few additional steps to run which are included below. - -# Exacloud - -## Getting Started - -## Additional Resources - diff --git a/website/content/docs/interop/hpc.md b/website/content/docs/interop/hpc.md new file mode 100644 index 000000000..5f19d7827 --- /dev/null +++ b/website/content/docs/interop/hpc.md @@ -0,0 +1,117 @@ +--- +title: HPC +menu: + main: + parent: Interop +--- + +> ⚠️ HPC support is in active development and commands may change. + +# HPC + +## Getting Started + +Funnel supports submitting tasks to HPC (High-Performance Computing) backends via job schedulers like Slurm. + +To get started with using HPC with Funnel, you will need to configure Funnel to use a suitable container engine and job scheduler. This guide will walk you through setting up the necessary configurations and running tasks on an HPC system using Slurm. + +## Example + +Below is an example configuration for setting up Funnel to run tasks on an HPC system using Slurm as the job scheduler and exadocker as the container engine. This configuration includes settings for local storage, worker behavior, server details, and Slurm job submission templates. + +
+ config.yml + +```yaml +LocalStorage: + # Whitelist of local directory paths which Funnel is allowed to access. + AllowedDirs: + - ./ + - /home/users/example + +Worker: + LeaveWorkDir: true + ContainerDriver: sudo /opt/acc/sbin/exadocker + +Server: + HostName: exahead1 + +Compute: slurm + +# Custom template for Slurm +Slurm: + Template: | + #!/bin/bash + #SBATCH --job-name {{.TaskId}} + #SBATCH --ntasks 1 + #SBATCH --error {{.WorkDir}}/funnel-stderr + #SBATCH --output {{.WorkDir}}/funnel-stdout + #SBATCH --gres disk:1 + {{if ne .Cpus 0 -}} + {{printf "#SBATCH --cpus-per-task %d" .Cpus}} + {{- end}} + {{if ne .RamGb 0.0 -}} + {{printf "#SBATCH --mem %.0fGB" .RamGb}} + {{- end}} + {{if ne .DiskGb 0.0 -}} + {{printf "#SBATCH --tmp %.0fGB" .DiskGb}} + {{- end}} + + srun /usr/local/bin/mkdir-scratch.sh > /dev/null + SCRATCH_PATH="/mnt/scratch/${SLURM_JOB_ID}" + + cd $SCRATCH_PATH + srun bash -c 'funnel worker run --taskID {{.TaskId}} {{.Args}} --Worker.ScratchPath .' --temp-dir $SCRATCH_PATH + + cd - > /dev/null + srun /usr/local/bin/rmdir-scratch.sh > /dev/null +``` +
+ +
+ md5sum.json + +```json +{ + "name": "md5sum example", + "description": "Demonstrates input and output files using a simple md5sum command.", + "inputs": [ + { + "name": "md5sum input", + "description": "Input to md5sum. /tmp/md5sum_input must exist on the host system.", + "url": "file:///tmp/md5sum_input", + "type": "FILE", + "path": "/tmp/in" + } + ], + "outputs": [ + { + "name": "md5sum stdout", + "description": "Stdout of md5sum is captured and copied to /tmp/md5_output on the host system.", + "url": "file:///tmp/md5sum_output", + "type": "FILE", + "path": "/tmp/out" + } + ], + "executors": [ + { + "image": "alpine", + "command": ["md5sum", "/tmp/in"], + "stdout": "/tmp/out" + } + ] +} +``` +
+ +## Start Funnel Server + +```sh +funnel server run --config config.yml +``` + +## Submit Task + +```sh +funnel task create md5sum.json +``` \ No newline at end of file diff --git a/website/layouts/_default/single.html b/website/layouts/_default/single.html index 9bcb97c5e..11c37ed28 100644 --- a/website/layouts/_default/single.html +++ b/website/layouts/_default/single.html @@ -31,7 +31,7 @@
- {{ .Content }} + {{ .Content | safeHTML }}
From 545036649e1a93d756384d2d784e2d695102ddc8 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 31 May 2024 13:00:08 -0700 Subject: [PATCH 19/67] Update docker and kin-openapi to compatible versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - github.com/docker/docker versions above v24.0.9 resulted in `panic: Can't connect docker client`: ``` ➜ go test tests/auth/auth_test.go --- FAIL: TestBasicAuthFail (0.01s) panic: Can't connect docker client [recovered] panic: Can't connect docker client FAIL command-line-arguments 0.673s FAIL ``` - This error appears to be originating from [util/dockerutil/docker.go](https://github.com/ohsu-comp-bio/funnel/blob/f5acbabe85d28f45f2f86db959c4d7bc929334a4/util/dockerutil/docker.go#L28). - github.com/getkin/kin-openapi versions above v0.112.0 resulted in type errors: v0.113.0: ``` ➜ go test util/openapi2proto/main.go util/openapi2proto/main.go:54:38: invalid operation: p.Value.AdditionalProperties != nil (mismatched types openapi3.AdditionalProperties and untyped nil) util/openapi2proto/main.go:55:24: cannot use p.Value.AdditionalProperties (variable of type openapi3.AdditionalProperties) as *openapi3.SchemaRef value in argument to getType FAIL command-line-arguments [build failed] FAIL ``` v0.123.0 (latest): ``` ➜ go test util/openapi2proto/main.go util/openapi2proto/main.go:43:7: cannot convert "integer" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:45:7: cannot convert "boolean" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:47:7: cannot convert "number" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:49:7: cannot convert "object" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:54:38: invalid operation: p.Value.AdditionalProperties != nil (mismatched types openapi3.AdditionalProperties and untyped nil) util/openapi2proto/main.go:55:24: cannot use p.Value.AdditionalProperties (variable of type openapi3.AdditionalProperties) as *openapi3.SchemaRef value in argument to getType util/openapi2proto/main.go:60:7: cannot convert "array" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:72:17: cannot use p.Value.Type (variable of type *openapi3.Types) as string value in return statement util/openapi2proto/main.go:84:32: invalid operation: param.Schema.Value.Type != "" (mismatched types *openapi3.Types and untyped string) util/openapi2proto/main.go:175:17: resp.Get undefined (type *openapi3.Responses has no field or method Get) util/openapi2proto/main.go:175:17: too many errors FAIL command-line-arguments [build failed] FAIL ``` --- go.mod | 27 +++++++---------- go.sum | 64 +++++++++++++++------------------------ util/dockerutil/docker.go | 2 +- 3 files changed, 36 insertions(+), 57 deletions(-) diff --git a/go.mod b/go.mod index a8fc56bb8..16994fe07 100644 --- a/go.mod +++ b/go.mod @@ -17,12 +17,12 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect - github.com/docker/docker v26.1.1+incompatible + github.com/docker/docker v24.0.9+incompatible github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elazarl/go-bindata-assetfs v1.0.1 github.com/gammazero/workerpool v1.1.3 - github.com/getkin/kin-openapi v0.124.0 + github.com/getkin/kin-openapi v0.12.0 github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a github.com/ghodss/yaml v1.0.0 github.com/gizak/termui v2.3.0+incompatible @@ -58,7 +58,7 @@ require ( golang.org/x/time v0.5.0 google.golang.org/api v0.177.0 google.golang.org/genproto v0.0.0-20240506185236-b8a5c65736ae // indirect - google.golang.org/grpc v1.63.2 + google.golang.org/grpc v1.64.0 gopkg.in/olivere/elastic.v5 v5.0.86 k8s.io/api v0.31.0-alpha.0 k8s.io/apimachinery v0.31.0-alpha.0 @@ -67,9 +67,9 @@ require ( require ( github.com/gogo/protobuf v1.3.2 - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 github.com/hashicorp/go-multierror v1.1.1 - google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae + google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 google.golang.org/protobuf v1.34.1 ) @@ -84,11 +84,11 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect @@ -113,7 +113,6 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/yaml v0.3.1 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -128,16 +127,13 @@ require ( github.com/maruel/panicparse v1.6.2 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nsf/termbox-go v1.1.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -159,17 +155,16 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect - go.opentelemetry.io/otel v1.26.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect - go.opentelemetry.io/otel/metric v1.26.0 // indirect - go.opentelemetry.io/otel/sdk v1.26.0 // indirect - go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.opentelemetry.io/otel v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/otel/sdk v1.27.0 // indirect + go.opentelemetry.io/otel/trace v1.27.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.21.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 828c4bca4..c01f858f9 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,6 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -56,8 +54,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -80,8 +76,10 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v26.1.1+incompatible h1:oI+4kkAgIwwb54b9OC7Xc3hSgu1RlJA/Lln/DF72djQ= -github.com/docker/docker v26.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= +github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -114,8 +112,8 @@ github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZ github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M= -github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= +github.com/getkin/kin-openapi v0.12.0 h1:xuxoUDhL5maK2aoXZHfyqp+hmWLyW2NizAKocDCwUTI= +github.com/getkin/kin-openapi v0.12.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a h1:yU/FENpkHYISWsQrbr3pcZOBj0EuRjPzNc1+dTCLu44= github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a/go.mod h1:AEugkNu3BjBxyz958nJ5holD9PRjta6iprcoUauDbU4= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -212,8 +210,8 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -230,8 +228,6 @@ github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cO github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= -github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -297,8 +293,6 @@ github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQ github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -306,8 +300,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -332,8 +324,6 @@ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2sz github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= -github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -402,6 +392,7 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -413,8 +404,6 @@ github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYN github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -440,20 +429,14 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.5 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= -go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= -go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= -go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= -go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= -go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= -go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= @@ -580,10 +563,10 @@ google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20240506185236-b8a5c65736ae h1:HjgkYCl6cWQEKSHkpUp4Q8VB74swzyBwTz1wtTzahm0= google.golang.org/genproto v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:i4np6Wrjp8EujFAUn0CM0SH+iZhY1EbrfzEIJbFkHFM= -google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae h1:AH34z6WAGVNkllnKs5raNq3yRq93VnjBG6rpfub/jYk= -google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:FfiGhwUm6CJviekPrc0oJ+7h29e+DmWU6UtjX0ZvI7Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae h1:c55+MER4zkBS14uJhSZMGGmya0yJx5iHV4x/fpOSNRk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= +google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -592,8 +575,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -617,6 +600,7 @@ gopkg.in/olivere/elastic.v5 v5.0.86/go.mod h1:M3WNlsF+WhYn7api4D87NIflwTV/c0iVs8 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/util/dockerutil/docker.go b/util/dockerutil/docker.go index 6f25c49d1..21a46c455 100644 --- a/util/dockerutil/docker.go +++ b/util/dockerutil/docker.go @@ -13,7 +13,7 @@ import ( // NewDockerClient returns a new docker client. This util handles // working around some client/server API version mismatch issues. func NewDockerClient() (*client.Client, error) { - dclient, err := client.NewEnvClient() + dclient, err := client.NewClientWithOpts(client.FromEnv) if err != nil { return nil, err } From 2269fd1dfc51872057677839ce099dbeb89b5b0b Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 31 May 2024 13:45:22 -0700 Subject: [PATCH 20/67] Update util/openapi2proto/main.go with latest kin-openapi version --- go.mod | 37 ++++++++-------- go.sum | 89 +++++++++++++++++++++----------------- util/openapi2proto/main.go | 43 +++++++++--------- 3 files changed, 91 insertions(+), 78 deletions(-) diff --git a/go.mod b/go.mod index 16994fe07..44941b690 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,14 @@ go 1.22.0 toolchain go1.22.2 require ( - cloud.google.com/go/datastore v1.16.0 - cloud.google.com/go/pubsub v1.37.0 + cloud.google.com/go/datastore v1.17.0 + cloud.google.com/go/pubsub v1.38.0 github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Shopify/sarama v1.38.1 github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 - github.com/aws/aws-sdk-go v1.52.3 + github.com/aws/aws-sdk-go v1.53.14 github.com/boltdb/bolt v1.3.1 github.com/cenkalti/backoff v2.2.1+incompatible github.com/dgraph-io/badger/v2 v2.2007.4 @@ -22,7 +22,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/elazarl/go-bindata-assetfs v1.0.1 github.com/gammazero/workerpool v1.1.3 - github.com/getkin/kin-openapi v0.12.0 + github.com/getkin/kin-openapi v0.124.0 github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a github.com/ghodss/yaml v1.0.0 github.com/gizak/termui v2.3.0+incompatible @@ -43,7 +43,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/ncw/swift v1.0.53 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1 github.com/prometheus/common v0.53.0 github.com/rs/xid v1.5.0 github.com/shirou/gopsutil v3.21.11+incompatible @@ -56,8 +56,8 @@ require ( golang.org/x/net v0.25.0 golang.org/x/oauth2 v0.20.0 golang.org/x/time v0.5.0 - google.golang.org/api v0.177.0 - google.golang.org/genproto v0.0.0-20240506185236-b8a5c65736ae // indirect + google.golang.org/api v0.182.0 + google.golang.org/genproto v0.0.0-20240528184218-531527333157 // indirect google.golang.org/grpc v1.64.0 gopkg.in/olivere/elastic.v5 v5.0.86 k8s.io/api v0.31.0-alpha.0 @@ -69,13 +69,13 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 github.com/hashicorp/go-multierror v1.1.1 - google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 google.golang.org/protobuf v1.34.1 ) require ( - cloud.google.com/go v0.112.2 // indirect - cloud.google.com/go/auth v0.3.0 // indirect + cloud.google.com/go v0.114.0 // indirect + cloud.google.com/go/auth v0.5.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.8 // indirect @@ -93,10 +93,10 @@ require ( github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/emicklei/go-restful/v3 v3.12.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gammazero/deque v0.2.1 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -113,6 +113,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/yaml v0.2.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -129,16 +130,18 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nsf/termbox-go v1.1.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/procfs v0.14.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect @@ -153,8 +156,8 @@ require ( github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect go.opentelemetry.io/otel v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.27.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect @@ -164,13 +167,13 @@ require ( golang.org/x/term v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.21.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.4.0 // indirect k8s.io/klog/v2 v2.120.1 // indirect - k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f // indirect + k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index c01f858f9..bba254fd2 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,23 @@ cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= -cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms= -cloud.google.com/go/auth v0.3.0 h1:PRyzEpGfx/Z9e8+lHsbkoUVXD0gnu4MNmm7Gp8TQNIs= -cloud.google.com/go/auth v0.3.0/go.mod h1:lBv6NKTWp8E3LPzmO1TbiiRKc4drLOfHsgmlH9ogv5w= +cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= +cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= +cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw= +cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/datastore v1.16.0 h1:LrZmu9l/qjoX/ilR+ECSMyO6tDYpijp3RR5LBM0HjpU= -cloud.google.com/go/datastore v1.16.0/go.mod h1:WIGbYyZE4GUJC+RLuVgpl6myNMKZGzlfbtN3Tch4R+8= +cloud.google.com/go/datastore v1.17.0 h1:UEmzuUdyDE58HV2jcb0BoqwCAwsJS2mtHapCsMmhVh0= +cloud.google.com/go/datastore v1.17.0/go.mod h1:RiRZU0G6VVlIVlv1HRo3vSAPFHULV0ddBNsXO+Sony4= cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= -cloud.google.com/go/kms v1.15.9 h1:ouZjTxCqDNEdxWfaAAbRzG22s/2iewRw6JPARQL+0vc= -cloud.google.com/go/kms v1.15.9/go.mod h1:5v/R/RRuBUVO+eJioGcqENr3syh8ZqNn1y1Wc9DjM+4= -cloud.google.com/go/pubsub v1.37.0 h1:0uEEfaB1VIJzabPpwpZf44zWAKAme3zwKKxHk7vJQxQ= -cloud.google.com/go/pubsub v1.37.0/go.mod h1:YQOQr1uiUM092EXwKs56OPT650nwnawc+8/IjoUeGzQ= +cloud.google.com/go/kms v1.17.1 h1:5k0wXqkxL+YcXd4viQzTqCgzzVKKxzgrK+rCZJytEQs= +cloud.google.com/go/kms v1.17.1/go.mod h1:DCMnCF/apA6fZk5Cj4XsD979OyHAqFasPuA5Sd0kGlQ= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= +cloud.google.com/go/pubsub v1.38.0 h1:J1OT7h51ifATIedjqk/uBNPh+1hkvUaH4VKbz4UuAsc= +cloud.google.com/go/pubsub v1.38.0/go.mod h1:IPMJSWSus/cu57UyR01Jqa/bNOQA+XnPF6Z4dKW4fAA= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -36,8 +38,8 @@ github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloD github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= -github.com/aws/aws-sdk-go v1.52.3 h1:BNPJmHOXNoM/iBWJKrvaQvJOweRcp3KLpzdb65CfQwU= -github.com/aws/aws-sdk-go v1.52.3/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.53.14 h1:SzhkC2Pzag0iRW8WBb80RzKdGXDydJR9LAMs2GyKJ2M= +github.com/aws/aws-sdk-go v1.53.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -95,8 +97,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= -github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= -github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -112,8 +114,8 @@ github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZ github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/getkin/kin-openapi v0.12.0 h1:xuxoUDhL5maK2aoXZHfyqp+hmWLyW2NizAKocDCwUTI= -github.com/getkin/kin-openapi v0.12.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= +github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M= +github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a h1:yU/FENpkHYISWsQrbr3pcZOBj0EuRjPzNc1+dTCLu44= github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a/go.mod h1:AEugkNu3BjBxyz958nJ5holD9PRjta6iprcoUauDbU4= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -125,8 +127,8 @@ github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -228,6 +230,8 @@ github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cO github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= +github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -300,6 +304,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -324,6 +330,8 @@ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2sz github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= +github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -333,15 +341,15 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= -github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s= -github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -392,7 +400,6 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -404,6 +411,8 @@ github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYN github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -418,17 +427,17 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8= -go.einride.tech/aip v0.66.0/go.mod h1:qAhMsfT7plxBX+Oy7Huol6YUvZ0ZzdUz26yZsQwfl1M= +go.einride.tech/aip v0.67.1 h1:d/4TW92OxXBngkSOwWS2CH5rez869KpKMaN44mdxkFI= +go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI= go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 h1:A3SayB3rNyt+1S6qpI9mHPkeHTZbD7XILEqWnYZb2l0= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 h1:vS1Ao/R55RNV4O7TA2Qopok8yN+X0LIP6RVWLFkprck= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0/go.mod h1:BMsdeOxN04K0L5FNUBfjFdvwWGNe/rkmSwH4Aelu/X0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= @@ -550,8 +559,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20170921000349-586095a6e407/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.177.0 h1:8a0p/BbPa65GlqGWtUKxot4p0TV8OGOfyTjtmkXNXmk= -google.golang.org/api v0.177.0/go.mod h1:srbhue4MLjkjbkux5p3dw/ocYOSZTaIEvf7bCOnFQDw= +google.golang.org/api v0.182.0 h1:if5fPvudRQ78GeRx3RayIoiuV7modtErPIZC/T2bIvE= +google.golang.org/api v0.182.0/go.mod h1:cGhjy4caqA5yXRzEhkHI8Y9mfyC2VLTlER2l08xaqtM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= @@ -561,12 +570,12 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240506185236-b8a5c65736ae h1:HjgkYCl6cWQEKSHkpUp4Q8VB74swzyBwTz1wtTzahm0= -google.golang.org/genproto v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:i4np6Wrjp8EujFAUn0CM0SH+iZhY1EbrfzEIJbFkHFM= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto v0.0.0-20240528184218-531527333157 h1:u7WMYrIrVvs0TF5yaKwKNbcJyySYf+HAIFXxWltJOXE= +google.golang.org/genproto v0.0.0-20240528184218-531527333157/go.mod h1:ubQlAQnzejB8uZzszhrTCU2Fyp6Vi7ZE5nn0c3W8+qQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -600,11 +609,11 @@ gopkg.in/olivere/elastic.v5 v5.0.86/go.mod h1:M3WNlsF+WhYn7api4D87NIflwTV/c0iVs8 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= @@ -619,8 +628,8 @@ k8s.io/client-go v0.31.0-alpha.0 h1:VVxcVKbqni0wAWd3rNuXcNcpE7gT6jvVLBe/YlWKu8s= k8s.io/client-go v0.31.0-alpha.0/go.mod h1:XM6Bsdnz08pV+PfIUNDOgblfG6PVLI9ll6xHMk1Ifso= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f h1:0LQagt0gDpKqvIkAMPaRGcXawNMouPECM1+F9BVxEaM= -k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f/go.mod h1:S9tOR0FxgyusSNR+MboCuiDpVWkAifZvaYI1Q2ubgro= +k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a h1:zD1uj3Jf+mD4zmA7W+goE5TxDkI7OGJjBNBzq5fJtLA= +k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/util/openapi2proto/main.go b/util/openapi2proto/main.go index 5177a2a10..4d4010991 100644 --- a/util/openapi2proto/main.go +++ b/util/openapi2proto/main.go @@ -39,38 +39,39 @@ type ServicePath struct { } func getType(p *openapi3.SchemaRef) (bool, string) { - switch p.Value.Type { - case "integer": + if p.Value.Type.Includes("integer") { return false, "int32" - case "boolean": + } + if p.Value.Type.Includes("boolean") { return false, "bool" - case "number": + } + if p.Value.Type.Includes("number") { return false, "double" - case "object": + } + if p.Value.Type.Includes("object") { if p.Ref != "" { t := strings.Split(p.Ref, "/") return false, t[len(t)-1] } - if p.Value.AdditionalProperties != nil { - _, aType := getType(p.Value.AdditionalProperties) + if p.Value.AdditionalProperties.Has != nil && *p.Value.AdditionalProperties.Has != false { + _, aType := getType(p.Value.AdditionalProperties.Schema) return false, fmt.Sprintf("map", aType) } return false, "map" - //return fmt.Sprintf("%#v", p.Value) - case "array": + } + if p.Value.Type.Includes("array") { if p.Value.Items.Ref != "" { t := strings.Split(p.Value.Items.Ref, "/") return true, t[len(t)-1] } _, aType := getType(p.Value.Items) return true, aType - default: - if p.Ref != "" { - t := strings.Split(p.Ref, "/") - return false, t[len(t)-1] - } - return false, p.Value.Type } + if p.Ref != "" { + t := strings.Split(p.Ref, "/") + return false, t[len(t)-1] + } + return false, p.Value.Type.Slice()[0] } func getParamType(param *openapi3.Parameter) (bool, string) { @@ -81,7 +82,7 @@ func getParamType(param *openapi3.Parameter) (bool, string) { return false, t[len(t)-1] } - if param.Schema.Value.Type != "" { + if param.Schema.Value.Type != nil && param.Schema.Value.Type.Includes("") == false { return getType(param.Schema) } return false, "" @@ -172,7 +173,7 @@ func cleanSchema(messages []Message, enums []Enum, services []ServicePath) { } func getResponseMessage(resp *openapi3.Responses) string { - schema := resp.Get(200).Value.Content.Get("application/json").Schema + schema := resp.Status(200).Value.Content.Get("application/json").Schema s := strings.Split(schema.Ref, "/") return s[len(s)-1] } @@ -218,7 +219,7 @@ func main() { } } - for path, req := range doc.Paths { + for path, req := range doc.Paths.Map() { if req.Get != nil { //log.Printf("Get: %s %s\n", path, req.Get.OperationID) reqFields := []Field{} @@ -252,14 +253,14 @@ func main() { } } - for path, req := range doc.Paths { + for path, req := range doc.Paths.Map() { p := ServicePath{} if req.Get != nil { p.Name = req.Get.OperationID p.Path = path p.Mode = "get" p.InputType = req.Get.OperationID + "Request" - p.OutputType = getResponseMessage(&req.Get.Responses) + p.OutputType = getResponseMessage(req.Get.Responses) service = append(service, p) } if req.Post != nil { @@ -274,7 +275,7 @@ func main() { } else { p.InputType = req.Post.OperationID + "Request" } - p.OutputType = getResponseMessage(&req.Post.Responses) + p.OutputType = getResponseMessage(req.Post.Responses) service = append(service, p) } } From 8648580642bef958e33f00cf3a1de69e56aba599 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 31 May 2024 13:52:24 -0700 Subject: [PATCH 21/67] Update github.com/docker/docker to latest version --- go.mod | 6 ++++-- go.sum | 18 ++++++++++++++---- tests/funnel_utils.go | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 44941b690..4886d653d 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect - github.com/docker/docker v24.0.9+incompatible + github.com/docker/docker v26.1.3+incompatible github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elazarl/go-bindata-assetfs v1.0.1 @@ -84,11 +84,11 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/distribution v2.8.3+incompatible // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect @@ -128,6 +128,7 @@ require ( github.com/maruel/panicparse v1.6.2 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect @@ -159,6 +160,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect go.opentelemetry.io/otel v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.27.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.opentelemetry.io/otel/trace v1.27.0 // indirect diff --git a/go.sum b/go.sum index bba254fd2..d2825c394 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -56,6 +58,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -78,10 +82,8 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= -github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= -github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo= +github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -297,6 +299,8 @@ github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQ github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -440,12 +444,18 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= diff --git a/tests/funnel_utils.go b/tests/funnel_utils.go index a418bfb1f..90cdb7761 100644 --- a/tests/funnel_utils.go +++ b/tests/funnel_utils.go @@ -411,7 +411,7 @@ func (f *Funnel) StartServerInDocker(containerName, imageName string, extraArgs func (f *Funnel) findTestServerContainers() []string { res := []string{} - containers, err := f.Docker.ContainerList(context.Background(), dockerTypes.ContainerListOptions{}) + containers, err := f.Docker.ContainerList(context.Background(), container.ListOptions{}) if err != nil { panic(err) } From 75647ceb858dc61887a47fde1b742e5b9cf6ef39 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 31 May 2024 14:01:47 -0700 Subject: [PATCH 22/67] Update Docker client creation in tests --- .github/workflows/nextflow.yaml | 2 +- tes/utils_test.go | 3 ++- util/dockerutil/docker.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index 57685e720..58a7c3438 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -44,7 +44,7 @@ jobs: - name: Install Nextflow run: | cd .. - git clone https://github.com/nextflow-io/nextflow/ -b tes-update-1.1 + git clone https://github.com/nextflow-io/nextflow cd nextflow make compile diff --git a/tes/utils_test.go b/tes/utils_test.go index b89a6c1b2..9f2977a4b 100644 --- a/tes/utils_test.go +++ b/tes/utils_test.go @@ -16,7 +16,8 @@ func TestBase64Encode(t *testing.T) { }, } - expected := "ewogICJleGVjdXRvcnMiOiBbCiAgICB7CiAgICAgICJjb21tYW5kIjogWwogICAgICAgICJlY2hvIiwKICAgICAgICAiaGVsbG8gd29ybGQiCiAgICAgIF0sCiAgICAgICJpbWFnZSI6ICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAidGFzazEiCn0=" + // expected := "ewogICJleGVjdXRvcnMiOiBbCiAgICB7CiAgICAgICJjb21tYW5kIjogWwogICAgICAgICJlY2hvIiwKICAgICAgICAiaGVsbG8gd29ybGQiCiAgICAgIF0sCiAgICAgICJpbWFnZSI6ICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAidGFzazEiCn0=" + expected := "ewogICJleGVjdXRvcnMiOiAgWwogICAgewogICAgICAiY29tbWFuZCI6ICBbCiAgICAgICAgImVjaG8iLAogICAgICAgICJoZWxsbyB3b3JsZCIKICAgICAgXSwKICAgICAgImltYWdlIjogICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAgInRhc2sxIgp9" encoded, err := Base64Encode(task) if err != nil { diff --git a/util/dockerutil/docker.go b/util/dockerutil/docker.go index 21a46c455..fa0bc7e69 100644 --- a/util/dockerutil/docker.go +++ b/util/dockerutil/docker.go @@ -13,7 +13,7 @@ import ( // NewDockerClient returns a new docker client. This util handles // working around some client/server API version mismatch issues. func NewDockerClient() (*client.Client, error) { - dclient, err := client.NewClientWithOpts(client.FromEnv) + dclient, err := client.NewClientWithOpts() if err != nil { return nil, err } From 997d3c2e42aa0e6c9ba1d1e14e15f4817583a810 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 31 May 2024 14:06:03 -0700 Subject: [PATCH 23/67] Add `WithAPIVersionNegotiation` to docker client creation --- util/dockerutil/docker.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/util/dockerutil/docker.go b/util/dockerutil/docker.go index fa0bc7e69..8dd0f0358 100644 --- a/util/dockerutil/docker.go +++ b/util/dockerutil/docker.go @@ -13,7 +13,10 @@ import ( // NewDockerClient returns a new docker client. This util handles // working around some client/server API version mismatch issues. func NewDockerClient() (*client.Client, error) { - dclient, err := client.NewClientWithOpts() + dclient, err := client.NewClientWithOpts( + client.FromEnv, + client.WithAPIVersionNegotiation(), + ) if err != nil { return nil, err } From 56e74c2dcebc48e1f0054991178bd06f52ca73ee Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 31 May 2024 16:04:25 -0700 Subject: [PATCH 24/67] Rename 'Driver' to 'DriverCommand' in ContainerConfig --- worker/container_engine.go | 3 +-- worker/docker.go | 24 +++++++++++++----------- worker/worker.go | 14 +++++++------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/worker/container_engine.go b/worker/container_engine.go index e5c79f962..6178f8e3d 100644 --- a/worker/container_engine.go +++ b/worker/container_engine.go @@ -35,11 +35,10 @@ type ContainerConfig struct { Id string Image string Name string - Driver []string + DriverCommand []string Command []string Volumes []Volume Workdir string - ContainerName string RemoveContainer bool Env map[string]string Stdin io.Reader diff --git a/worker/docker.go b/worker/docker.go index 2368824e4..1a2f1be92 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -32,9 +32,9 @@ func (docker Docker) Run(ctx context.Context) error { var args []string - if len(docker.ContainerConfig.Driver) > 1 { + if len(docker.ContainerConfig.DriverCommand) > 1 { // Merge driver parts and command parts - args = append(args, docker.ContainerConfig.Driver[1:]...) + args = append(args, docker.ContainerConfig.DriverCommand[1:]...) } args = append(args, "run", "-i", "--read-only") @@ -49,8 +49,8 @@ func (docker Docker) Run(ctx context.Context) error { } } - if docker.ContainerName != "" { - args = append(args, "--name", docker.ContainerName) + if docker.Name != "" { + args = append(args, "--name", docker.Name) } if docker.Workdir != "" { @@ -66,8 +66,8 @@ func (docker Docker) Run(ctx context.Context) error { args = append(args, docker.Command...) // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` - docker.Event.Info("Running command", "cmd", docker.ContainerConfig.Driver[0]+" "+strings.Join(args, " ")) - cmd := exec.Command(docker.ContainerConfig.Driver[0], args...) + docker.Event.Info("Running command", "cmd", docker.ContainerConfig.DriverCommand[0]+" "+strings.Join(args, " ")) + cmd := exec.Command(docker.ContainerConfig.DriverCommand[0], args...) if docker.Stdin != nil { cmd.Stdin = docker.Stdin @@ -86,9 +86,9 @@ func (docker Docker) Run(ctx context.Context) error { // Stop stops the container. func (docker Docker) Stop() error { - docker.Event.Info("Stopping container", "container", docker.ContainerName) - // cmd := exec.Command("docker", "stop", docker.ContainerName) - cmd := exec.Command("docker", "rm", "-f", docker.ContainerName) //switching to this to be a bit more forceful + docker.Event.Info("Stopping container", "container", docker.Name) + // cmd := exec.Command("docker", "stop", docker.Name) + cmd := exec.Command("docker", "rm", "-f", docker.Name) //switching to this to be a bit more forceful return cmd.Run() } @@ -135,11 +135,13 @@ func (docker *Docker) InspectContainer(ctx context.Context) ContainerConfig { case <-ctx.Done(): return ContainerConfig{} case <-ticker.C: - cmd := exec.CommandContext(ctx, "docker", "inspect", docker.ContainerName) + cmd := exec.CommandContext(ctx, "docker", "inspect", docker.Name) out, err := cmd.Output() if err == nil { + fmt.Println("DEBUG: string(out):", string(out)) + meta := []ContainerConfig{} - err := json.Unmarshal(out, &meta) + err = json.Unmarshal(out, &meta) if err == nil && len(meta) == 1 { docker.Event.Info("container metadata", "containerID", meta[0].Id, diff --git a/worker/worker.go b/worker/worker.go index 3b1201b37..06cd98f12 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -162,18 +162,18 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { f := ContainerEngineFactory{} for i, d := range task.GetExecutors() { containerConfig := ContainerConfig{ - Image: d.Image, - Command: d.Command, - Env: d.Env, - Volumes: mapper.Volumes, - Workdir: d.Workdir, - ContainerName: fmt.Sprintf("%s-%d", task.Id, i), + Image: d.Image, + Command: d.Command, + Env: d.Env, + Volumes: mapper.Volumes, + Workdir: d.Workdir, + Name: fmt.Sprintf("%s-%d", task.Id, i), // TODO make RemoveContainer configurable RemoveContainer: true, Event: event.NewExecutorWriter(uint32(i)), } if r.Conf.ContainerDriver != "" { - containerConfig.Driver = strings.Fields(r.Conf.ContainerDriver) + containerConfig.DriverCommand = strings.Fields(r.Conf.ContainerDriver) } containerEngine, err := f.NewContainerEngine(r.Conf.ContainerType, containerConfig) if err != nil { From a9411bb0df2e35186813ef2dbd9794c2c47aaa48 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 31 May 2024 17:14:00 -0700 Subject: [PATCH 25/67] Fix minor linting errors --- cmd/node/run.go | 4 +- compute/hpc_backend.go | 15 ++-- compute/hpc_backend_test.go | 3 +- database/boltdb/events.go | 3 + database/mongodb/scheduler.go | 4 + storage/generic_s3.go | 6 ++ storage/local.go | 160 +++++++++++++++++----------------- tests/funnel_utils.go | 3 + worker/file_mapper.go | 10 ++- worker/step.go | 1 - worker/taskreader_test.go | 6 ++ 11 files changed, 124 insertions(+), 91 deletions(-) diff --git a/cmd/node/run.go b/cmd/node/run.go index c0492273a..c2dc791bd 100644 --- a/cmd/node/run.go +++ b/cmd/node/run.go @@ -34,8 +34,8 @@ func Run(ctx context.Context, conf config.Config, log *logger.Logger) error { return err } - runctx, cancel := context.WithCancel(context.Background()) - runctx = util.SignalContext(ctx, time.Nanosecond, syscall.SIGINT, syscall.SIGTERM) + _, cancel := context.WithCancel(context.Background()) + runctx := util.SignalContext(ctx, time.Nanosecond, syscall.SIGINT, syscall.SIGTERM) defer cancel() hupsig := make(chan os.Signal, 1) diff --git a/compute/hpc_backend.go b/compute/hpc_backend.go index 309a1b44e..d5e160496 100644 --- a/compute/hpc_backend.go +++ b/compute/hpc_backend.go @@ -262,13 +262,16 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(ctx context.Context, task *tes.Task zone = zones[0] } - conf := ctx.Value("Config").(config.Config) - configFile := filepath.Join(workdir, "config.yaml") - err = config.ToYamlFile(conf, configFile) - if err != nil { - return "", err + var args string + if ctx.Value("Config") == nil { + conf := ctx.Value("Config").(config.Config) + configFile := filepath.Join(workdir, "config.yaml") + err = config.ToYamlFile(conf, configFile) + if err != nil { + return "", err + } + args = fmt.Sprintf("--config %v", configFile) } - args := fmt.Sprintf("--config %v", configFile) err = submitTpl.Execute(f, map[string]interface{}{ "TaskId": task.Id, "WorkDir": workdir, diff --git a/compute/hpc_backend_test.go b/compute/hpc_backend_test.go index 5b91abb3e..2c2add627 100644 --- a/compute/hpc_backend_test.go +++ b/compute/hpc_backend_test.go @@ -1,6 +1,7 @@ package compute import ( + "context" "fmt" "io/ioutil" "testing" @@ -58,7 +59,7 @@ funnel worker run --taskID {{.TaskId}} Conf: conf, } - sf, err := b.setupTemplatedHPCSubmit(task) + sf, err := b.setupTemplatedHPCSubmit(context.Background(), task) if err != nil { t.Fatal(err) } diff --git a/database/boltdb/events.go b/database/boltdb/events.go index d06cccd77..e5f649a71 100644 --- a/database/boltdb/events.go +++ b/database/boltdb/events.go @@ -146,6 +146,9 @@ func (taskBolt *BoltDB) WriteEvent(ctx context.Context, req *events.Event) error tx.Bucket(SysLogs).Put(idBytes, logbytes) return nil }) + if err != nil { + return err + } } return err diff --git a/database/mongodb/scheduler.go b/database/mongodb/scheduler.go index 33de34765..96435a8a4 100644 --- a/database/mongodb/scheduler.go +++ b/database/mongodb/scheduler.go @@ -19,6 +19,10 @@ func (db *MongoDB) ReadQueue(n int) []*tes.Task { var tasks []*tes.Task opts := options.Find().SetSort(bson.M{"creationtime": 1}).SetLimit(int64(n)) cursor, err := db.tasks(db.client).Find(context.TODO(), bson.M{"state": tes.State_QUEUED}, opts) + if err != nil { + fmt.Println(err) + return nil + } err = cursor.All(context.TODO(), &tasks) if err != nil { diff --git a/storage/generic_s3.go b/storage/generic_s3.go index bd37c5a27..ea4ea66d1 100644 --- a/storage/generic_s3.go +++ b/storage/generic_s3.go @@ -73,6 +73,9 @@ func (s3 *GenericS3) Stat(ctx context.Context, url string) (*Object, error) { } isDir, err := isDir(s3.client, u.bucket, u.path) + if err != nil { + return nil, fmt.Errorf("genericS3: stat object: %s", err) + } if isDir { return &Object{ URL: url, @@ -135,6 +138,9 @@ func (s3 *GenericS3) Get(ctx context.Context, url, path string) (*Object, error) } isDir, err := isDir(s3.client, u.bucket, u.path) + if err != nil { + return nil, fmt.Errorf("genericS3: getting object %s: %v", url, err) + } if isDir { objects, err := s3.List(ctx, url) if err != nil { diff --git a/storage/local.go b/storage/local.go index 0f56d079d..d2d24719d 100644 --- a/storage/local.go +++ b/storage/local.go @@ -168,100 +168,102 @@ func copyFile(ctx context.Context, source string, dest string) (err error) { // Hard links file source to destination dest. func linkFile(ctx context.Context, source string, dest string) error { - // If source has a glob or wildcard, get the filepath using the filepath.Glob function - if strings.Contains(source, "*") { - globs, err := filepath.Glob(source) - if err != nil { - return fmt.Errorf("failed to get filepath using Glob: %v", err) - } - for _, glob := range globs { - // Correctly calculate the destination for each file - destFile := filepath.Join(dest, filepath.Base(glob)) - err := processItem(ctx, glob, destFile) - if err != nil { - return err - } - } - return nil - } else { - return processItem(ctx, source, dest) - } + // If source has a glob or wildcard, get the filepath using the filepath.Glob function + if strings.Contains(source, "*") { + globs, err := filepath.Glob(source) + if err != nil { + return fmt.Errorf("failed to get filepath using Glob: %v", err) + } + for _, glob := range globs { + // Correctly calculate the destination for each file + destFile := filepath.Join(dest, filepath.Base(glob)) + err := processItem(ctx, glob, destFile) + if err != nil { + return err + } + } + return nil + } else { + return processItem(ctx, source, dest) + } } - // Process a single item (file or directory) func processItem(ctx context.Context, source, dest string) error { - fileInfo, err := os.Stat(source) - if err != nil { - return err - } - - if fileInfo.IsDir() { - return processDirectory(ctx, source, dest) - } else { - return processFile(ctx, source, dest) - } + fileInfo, err := os.Stat(source) + if err != nil { + return err + } + + if fileInfo.IsDir() { + return processDirectory(ctx, source, dest) + } else { + return processFile(ctx, source, dest) + } } // Process a directory func processDirectory(ctx context.Context, source, dest string) error { - // Create destination directory - err := os.MkdirAll(dest, 0755) // Adjust permissions as needed - if err != nil { - return err - } - - entries, err := os.ReadDir(source) - if err != nil { - return err - } - - for _, entry := range entries { - srcPath := filepath.Join(source, entry.Name()) - destPath := filepath.Join(dest, entry.Name()) - - if entry.IsDir() { - err = processDirectory(ctx, srcPath, destPath) - } else { - err = processFile(ctx, srcPath, destPath) - } - - if err != nil { - return err - } - } - return nil + // Create destination directory + err := os.MkdirAll(dest, 0755) // Adjust permissions as needed + if err != nil { + return err + } + + entries, err := os.ReadDir(source) + if err != nil { + return err + } + + for _, entry := range entries { + srcPath := filepath.Join(source, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) + + if entry.IsDir() { + err = processDirectory(ctx, srcPath, destPath) + } else { + err = processFile(ctx, srcPath, destPath) + } + + if err != nil { + return err + } + } + return nil } // Process a single file func processFile(ctx context.Context, source, dest string) error { - // without this resulting link could be a symlink - parent, err := filepath.EvalSymlinks(source) - - same, err := sameFile(parent, dest) - if err != nil { - return err - } - if same { - return nil - } - - err = os.Link(parent, dest) - if err != nil { - return copyFile(ctx, parent, dest) - } - return nil + // without this resulting link could be a symlink + parent, err := filepath.EvalSymlinks(source) + if err != nil { + return err + } + + same, err := sameFile(parent, dest) + if err != nil { + return err + } + if same { + return nil + } + + err = os.Link(parent, dest) + if err != nil { + return copyFile(ctx, parent, dest) + } + return nil } func FilePathWalkDir(root string) ([]string, error) { - var files []string - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { - files = append(files, path) - } - return nil - }) - return files, err + var files []string + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + files = append(files, path) + } + return nil + }) + return files, err } func sameFile(source string, dest string) (bool, error) { diff --git a/tests/funnel_utils.go b/tests/funnel_utils.go index 90cdb7761..32f027932 100644 --- a/tests/funnel_utils.go +++ b/tests/funnel_utils.go @@ -332,6 +332,9 @@ func (f *Funnel) StartServerInDocker(containerName, imageName string, extraArgs if gopath == "" { funnelBinary, err = filepath.Abs("../../funnel") + if err != nil { + log.Error("Error finding funnel binary", err) + } } else { if runtime.GOOS == "linux" { funnelBinary, err = filepath.Abs(filepath.Join( diff --git a/worker/file_mapper.go b/worker/file_mapper.go index 8b2dfd33b..a3c5c2b02 100644 --- a/worker/file_mapper.go +++ b/worker/file_mapper.go @@ -124,6 +124,9 @@ func (mapper *FileMapper) CopyInputsToScratch(scratchDir string) error { return fmt.Errorf("failed to create scratch directory: %w", err) } err = copyFile(input.Path, scratchTarget) + if err != nil { + return fmt.Errorf("failed to copy input file to scratch directory: %w", err) + } _, _ = os.ReadFile(scratchTarget) } } @@ -157,16 +160,19 @@ func (mapper *FileMapper) CopyOutputsToWorkDir(scratchDir string) error { } if info.IsDir() { // Ensure the scratch target directory exists - if err := os.MkdirAll(output.Path, 0755); err != nil { + if err = os.MkdirAll(output.Path, 0755); err != nil { return fmt.Errorf("failed to create output path: %w", err) } copyDir(scratchTarget, output.Path) } else { // Ensure the scratch target directory exists - if err := os.MkdirAll(path.Dir(output.Path), 0755); err != nil { + if err = os.MkdirAll(path.Dir(output.Path), 0755); err != nil { return fmt.Errorf("failed to create output path: %w", err) } err = copyFile(scratchTarget, output.Path) + if err != nil { + return fmt.Errorf("failed to copy output file to output directory: %w", err) + } } } diff --git a/worker/step.go b/worker/step.go index 42e1c8ba2..bc6ef113c 100644 --- a/worker/step.go +++ b/worker/step.go @@ -51,7 +51,6 @@ func (s *stepWorker) Run(ctx context.Context) error { stderr = io.MultiWriter(err, stderr) } s.Command.SetIO(nil, stdout, stderr) - _, out, err = s.Command.GetIO() go func() { done <- s.Command.Run(subctx) diff --git a/worker/taskreader_test.go b/worker/taskreader_test.go index 994d13c28..534b6bb1d 100644 --- a/worker/taskreader_test.go +++ b/worker/taskreader_test.go @@ -13,6 +13,9 @@ func TestFileTaskReader(t *testing.T) { ctx := context.Background() task, err := r.Task(ctx) + if err != nil { + t.Fatal(err) + } if task.Name != "Hello world" { t.Error("unexpected task content") } @@ -30,6 +33,9 @@ func TestBase64TaskReader(t *testing.T) { ctx := context.Background() task, err := r.Task(ctx) + if err != nil { + t.Fatal(err) + } if task.Name != "Hello world" { t.Error("unexpected task content") } From ee25bd5307a02116279a3ff7443499543d24d8c1 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 31 May 2024 17:19:31 -0700 Subject: [PATCH 26/67] Update GitHub Actions workflow (linting) --- .github/workflows/tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9f8654b21..699ddaf03 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -16,7 +16,8 @@ jobs: uses: golangci/golangci-lint-action@v3 with: version: latest - args: --timeout 3m --verbose -D unused -D errcheck -D staticcheck -D govet -D gosimple -D ineffassign + # TODO: Re-enable all linters + args: --timeout 3m --verbose -D unused -D errcheck -D staticcheck -D govet -D gosimple build: runs-on: ubuntu-latest From 449e14484c8e8b8dd6efd9c371763123d8cfab25 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 31 May 2024 17:31:19 -0700 Subject: [PATCH 27/67] fix: updated config check in hpc_backend.go --- compute/hpc_backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/hpc_backend.go b/compute/hpc_backend.go index d5e160496..022e7e395 100644 --- a/compute/hpc_backend.go +++ b/compute/hpc_backend.go @@ -263,7 +263,7 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(ctx context.Context, task *tes.Task } var args string - if ctx.Value("Config") == nil { + if ctx.Value("Config") != nil { conf := ctx.Value("Config").(config.Config) configFile := filepath.Join(workdir, "config.yaml") err = config.ToYamlFile(conf, configFile) From f5df70f41bc87892565752bf0c97185c0304a793 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 31 May 2024 17:56:58 -0700 Subject: [PATCH 28/67] Revert tabs/space change in storage/local.go --- storage/local.go | 165 ++++++++++++++++++++++++----------------------- 1 file changed, 83 insertions(+), 82 deletions(-) diff --git a/storage/local.go b/storage/local.go index d2d24719d..28caa2052 100644 --- a/storage/local.go +++ b/storage/local.go @@ -112,7 +112,7 @@ func (local *Local) UnsupportedOperations(url string) UnsupportedOperations { path := getPath(url) if !isAllowed(path, local.allowedDirs) { err := fmt.Errorf( - "localStorage: can't access file, path is not in allowed directories: %s, %s", url, local.allowedDirs) + "localStorage: can't access file, path is not in allowed directories: %s", url) return AllUnsupported(err) } return AllSupported() @@ -168,102 +168,103 @@ func copyFile(ctx context.Context, source string, dest string) (err error) { // Hard links file source to destination dest. func linkFile(ctx context.Context, source string, dest string) error { - // If source has a glob or wildcard, get the filepath using the filepath.Glob function - if strings.Contains(source, "*") { - globs, err := filepath.Glob(source) - if err != nil { - return fmt.Errorf("failed to get filepath using Glob: %v", err) - } - for _, glob := range globs { - // Correctly calculate the destination for each file - destFile := filepath.Join(dest, filepath.Base(glob)) - err := processItem(ctx, glob, destFile) - if err != nil { - return err - } - } - return nil - } else { - return processItem(ctx, source, dest) - } + // If source has a glob or wildcard, get the filepath using the filepath.Glob function + if strings.Contains(source, "*") { + globs, err := filepath.Glob(source) + if err != nil { + return fmt.Errorf("failed to get filepath using Glob: %v", err) + } + for _, glob := range globs { + // Correctly calculate the destination for each file + destFile := filepath.Join(dest, filepath.Base(glob)) + err := processItem(ctx, glob, destFile) + if err != nil { + return err + } + } + return nil + } else { + return processItem(ctx, source, dest) + } } + // Process a single item (file or directory) func processItem(ctx context.Context, source, dest string) error { - fileInfo, err := os.Stat(source) - if err != nil { - return err - } - - if fileInfo.IsDir() { - return processDirectory(ctx, source, dest) - } else { - return processFile(ctx, source, dest) - } + fileInfo, err := os.Stat(source) + if err != nil { + return err + } + + if fileInfo.IsDir() { + return processDirectory(ctx, source, dest) + } else { + return processFile(ctx, source, dest) + } } // Process a directory func processDirectory(ctx context.Context, source, dest string) error { - // Create destination directory - err := os.MkdirAll(dest, 0755) // Adjust permissions as needed - if err != nil { - return err - } - - entries, err := os.ReadDir(source) - if err != nil { - return err - } - - for _, entry := range entries { - srcPath := filepath.Join(source, entry.Name()) - destPath := filepath.Join(dest, entry.Name()) - - if entry.IsDir() { - err = processDirectory(ctx, srcPath, destPath) - } else { - err = processFile(ctx, srcPath, destPath) - } - - if err != nil { - return err - } - } - return nil + // Create destination directory + err := os.MkdirAll(dest, 0755) // Adjust permissions as needed + if err != nil { + return err + } + + entries, err := os.ReadDir(source) + if err != nil { + return err + } + + for _, entry := range entries { + srcPath := filepath.Join(source, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) + + if entry.IsDir() { + err = processDirectory(ctx, srcPath, destPath) + } else { + err = processFile(ctx, srcPath, destPath) + } + + if err != nil { + return err + } + } + return nil } // Process a single file func processFile(ctx context.Context, source, dest string) error { - // without this resulting link could be a symlink - parent, err := filepath.EvalSymlinks(source) - if err != nil { - return err - } - - same, err := sameFile(parent, dest) - if err != nil { - return err - } - if same { - return nil - } - - err = os.Link(parent, dest) - if err != nil { - return copyFile(ctx, parent, dest) - } - return nil + // without this resulting link could be a symlink + parent, err := filepath.EvalSymlinks(source) + if err != nil { + return err + } + + same, err := sameFile(parent, dest) + if err != nil { + return err + } + if same { + return nil + } + + err = os.Link(parent, dest) + if err != nil { + return copyFile(ctx, parent, dest) + } + return nil } func FilePathWalkDir(root string) ([]string, error) { - var files []string - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { - files = append(files, path) - } - return nil - }) - return files, err + var files []string + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + files = append(files, path) + } + return nil + }) + return files, err } func sameFile(source string, dest string) (bool, error) { From 6374d2f758bf8e36860ffafa4eebbf8e5ff57515 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 18 Jun 2024 15:57:26 -0700 Subject: [PATCH 29/67] Update output copying in scratch directory --- cmd/server/run.go | 6 +++++- events/data.json | 14 +++++++++++++ server/server.go | 7 ++++++- storage/local.go | 5 +++-- worker/docker.go | 22 ++++++++++++++++---- worker/file_mapper.go | 48 ++++++++++++++++++++++++++++--------------- 6 files changed, 78 insertions(+), 24 deletions(-) create mode 100644 events/data.json diff --git a/cmd/server/run.go b/cmd/server/run.go index 58a6b19bf..79e808c2e 100644 --- a/cmd/server/run.go +++ b/cmd/server/run.go @@ -147,7 +147,11 @@ func NewServer(ctx context.Context, conf config.Config, log *logger.Logger) (*Se case strings.ToLower(conf.Database): continue case "log": - continue + if conf.Compute == "slurm" { + writer = &events.Logger{Log: log} + } else { + continue + } case "boltdb": writer, err = boltdb.NewBoltDB(conf.BoltDB) case "badger": diff --git a/events/data.json b/events/data.json new file mode 100644 index 000000000..ac0e930f9 --- /dev/null +++ b/events/data.json @@ -0,0 +1,14 @@ +{ + "id": "event123", + "timestamp": "2024-05-20T12:34:56Z", + "type": "TASK_CREATED", + "task": { + "id": "task123", + "executors": [ + { + "image": "alpine", + "command": ["echo", "hello world"] + } + ] + } +} diff --git a/server/server.go b/server/server.go index 871b382e3..02b6aab6b 100644 --- a/server/server.go +++ b/server/server.go @@ -53,6 +53,7 @@ func newDebugInterceptor(log *logger.Logger) grpc.UnaryServerInterceptor { "resp", resp, "err", err, ) + return resp, err } } @@ -84,7 +85,11 @@ func customErrorHandler(ctx context.Context, mux *runtime.ServeMux, marshaler ru w.WriteHeader(http.StatusNotFound) // 404 } default: - w.WriteHeader(http.StatusInternalServerError) // 500 + if (strings.Contains(st.Message(), "backend parameters not supported")) { + w.WriteHeader(http.StatusBadRequest) // 400 + } else { + w.WriteHeader(http.StatusInternalServerError) // 500 + } } // Write the error message diff --git a/storage/local.go b/storage/local.go index 28caa2052..a1016e632 100644 --- a/storage/local.go +++ b/storage/local.go @@ -168,6 +168,7 @@ func copyFile(ctx context.Context, source string, dest string) (err error) { // Hard links file source to destination dest. func linkFile(ctx context.Context, source string, dest string) error { + // If source has a glob or wildcard, get the filepath using the filepath.Glob function if strings.Contains(source, "*") { globs, err := filepath.Glob(source) @@ -176,8 +177,7 @@ func linkFile(ctx context.Context, source string, dest string) error { } for _, glob := range globs { // Correctly calculate the destination for each file - destFile := filepath.Join(dest, filepath.Base(glob)) - err := processItem(ctx, glob, destFile) + err := processItem(ctx, glob, dest) if err != nil { return err } @@ -196,6 +196,7 @@ func processItem(ctx context.Context, source, dest string) error { return err } + if fileInfo.IsDir() { return processDirectory(ctx, source, dest) } else { diff --git a/worker/docker.go b/worker/docker.go index 1a2f1be92..fd8e9793a 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -24,7 +24,15 @@ func (docker Docker) Run(ctx context.Context) error { docker.Event.Error("failed to sync docker client API version", err) } - pullcmd := exec.Command("docker", "pull", docker.Image) + var pullArgs []string + + if len(docker.ContainerConfig.DriverCommand) > 1 { + // Merge driver parts and command parts + pullArgs = append(pullArgs, docker.ContainerConfig.DriverCommand[1:]...) + } + + pullArgs = append(pullArgs, "pull", docker.Image) + pullcmd := exec.Command(docker.ContainerConfig.DriverCommand[0], pullArgs...) err = pullcmd.Run() if err != nil { docker.Event.Error("failed to pull docker image", err) @@ -66,7 +74,6 @@ func (docker Docker) Run(ctx context.Context) error { args = append(args, docker.Command...) // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` - docker.Event.Info("Running command", "cmd", docker.ContainerConfig.DriverCommand[0]+" "+strings.Join(args, " ")) cmd := exec.Command(docker.ContainerConfig.DriverCommand[0], args...) if docker.Stdin != nil { @@ -138,7 +145,6 @@ func (docker *Docker) InspectContainer(ctx context.Context) ContainerConfig { cmd := exec.CommandContext(ctx, "docker", "inspect", docker.Name) out, err := cmd.Output() if err == nil { - fmt.Println("DEBUG: string(out):", string(out)) meta := []ContainerConfig{} err = json.Unmarshal(out, &meta) @@ -160,7 +166,15 @@ func (docker *Docker) InspectContainer(ctx context.Context) ContainerConfig { // the server. func (docker *Docker) SyncAPIVersion() error { if os.Getenv("DOCKER_API_VERSION") == "" { - cmd := exec.Command("docker", "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) + var args []string + + if len(docker.ContainerConfig.DriverCommand) > 1 { + // Merge driver parts and command parts + args = append(args, docker.ContainerConfig.DriverCommand[1:]...) + } + + args = append(args, "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) + cmd := exec.Command(docker.ContainerConfig.DriverCommand[0], args...) out, err := cmd.Output() if err != nil { return fmt.Errorf("docker version command failed: %v", err) diff --git a/worker/file_mapper.go b/worker/file_mapper.go index a3c5c2b02..acd7ba0da 100644 --- a/worker/file_mapper.go +++ b/worker/file_mapper.go @@ -154,24 +154,40 @@ func (mapper *FileMapper) CopyOutputsToWorkDir(scratchDir string) error { for _, output := range mapper.Outputs { scratchTarget := filepath.Join(scratchAbsDir, output.Path) - info, err := os.Stat(scratchTarget) + matches, err := filepath.Glob(scratchTarget) if err != nil { - fmt.Println(err) - } - if info.IsDir() { - // Ensure the scratch target directory exists - if err = os.MkdirAll(output.Path, 0755); err != nil { - return fmt.Errorf("failed to create output path: %w", err) - } - copyDir(scratchTarget, output.Path) - } else { - // Ensure the scratch target directory exists - if err = os.MkdirAll(path.Dir(output.Path), 0755); err != nil { - return fmt.Errorf("failed to create output path: %w", err) - } - err = copyFile(scratchTarget, output.Path) + return fmt.Errorf("invalid pattern %s: %w", scratchTarget, err) + } + if len(matches) == 0 { + return fmt.Errorf("no files matched the pattern: %s", scratchTarget) + } + + parentDir := filepath.Dir(output.Path) + for _, src := range matches { + info, err := os.Stat(src) if err != nil { - return fmt.Errorf("failed to copy output file to output directory: %w", err) + return fmt.Errorf("failed to stat output path: %w", err) + } + // If output is a directory + if info.IsDir() { + // Ensure the scratch target directory exists + if err = os.MkdirAll(output.Path, 0755); err != nil { + return fmt.Errorf("failed to create output path: %w", err) + } + copyDir(src, output.Path) + // If output is a file + } else { + // Ensure the scratch target directory exists + if err = os.MkdirAll(path.Dir(parentDir), 0755); err != nil { + return fmt.Errorf("failed to create output path: %w", err) + } + + workDirPath := filepath.Join(parentDir, info.Name()) + + err = copyFile(src, workDirPath) + if err != nil { + return fmt.Errorf("failed to copy output file to output directory: %w", err) + } } } } From f27f302195829a4ac55386eeca4dedc6a54d7ddb Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 18 Jun 2024 16:15:57 -0700 Subject: [PATCH 30/67] Update compliance-test.yaml --- .github/workflows/compliance-test.yaml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/workflows/compliance-test.yaml b/.github/workflows/compliance-test.yaml index 45990039c..7cf1817e6 100644 --- a/.github/workflows/compliance-test.yaml +++ b/.github/workflows/compliance-test.yaml @@ -1,11 +1,11 @@ # Workflow for running the TES compliance suite against Funnel - +# # This includes the following steps: # 1. Build Funnel and store the resulting binary artifact # 2. Install tes-compliance-suite and run against every version of TES simultaneously # 3. start-report-deployment: Send a dispatch to the funnel-compliance repository to generate and publish # the tes-compliance-suite report to https://ohsu-comp-bio.github.io/funnel-compliance/ - +# # Optionally debug via SSH # Ref: https://fleetdm.com/engineering/tips-for-github-actions-usability # @@ -90,11 +90,7 @@ jobs: - name: Run OpenAPI Test Runner run: | - # Clone the 'upstream' OpenAPI Test Runner when PR #65 is merged - # https://github.com/elixir-cloud-aai/openapi-test-runner/pull/65 - # git clone https://github.com/elixir-cloud-aai/openapi-test-runner - git clone https://github.com/ohsu-comp-bio/openapi-test-runner -b fix/create-and-filter-task - + git clone https://github.com/elixir-cloud-aai/openapi-test-runner cd openapi-test-runner python3 -m venv venv source venv/bin/activate @@ -102,20 +98,15 @@ jobs: python setup.py install openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/" - - name: Install TES compliance suite + - name: Install Legacy TES compliance suite (Report Generation) run: | git clone https://github.com/lbeckman314/tes-compliance-suite -b feature/tesv1.1 cd tes-compliance-suite - /root/.pyenv/shims/python3 -m venv venv + python3 -m venv venv source venv/bin/activate pip install -r requirements.txt python setup.py install mkdir reports - - - name: Test compliance (Report Publishing) - run: | - cd tes-compliance-suite - source venv/bin/activate tes-compliance-suite report --version "${{ matrix.version }}" --server "http://localhost:8000/" start-report-deployment: From 0f36f1115289abc65163592c6c057e4322451d49 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Wed, 19 Jun 2024 19:13:41 -0700 Subject: [PATCH 31/67] Debug: add tmate step to Nextflow test --- .github/workflows/nextflow.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index 58a7c3438..deaaa88e2 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -41,12 +41,13 @@ jobs: chmod +x ./funnel ./funnel server --LocalStorage.AllowedDirs $HOME run & + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Install Nextflow run: | - cd .. - git clone https://github.com/nextflow-io/nextflow - cd nextflow - make compile + curl -s https://get.nextflow.io | bash + chmod +x nextflow - name: Install nf-canary and GA4GH-TES plugin run: | From c0cc2a804513deb95cf946d76ca3ecf26c9bfa92 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Wed, 19 Jun 2024 19:24:13 -0700 Subject: [PATCH 32/67] Update Nextflow tests --- .github/workflows/nextflow.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index deaaa88e2..8d5206321 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -41,17 +41,14 @@ jobs: chmod +x ./funnel ./funnel server --LocalStorage.AllowedDirs $HOME run & - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: Install Nextflow run: | + cd .. curl -s https://get.nextflow.io | bash chmod +x nextflow - name: Install nf-canary and GA4GH-TES plugin run: | - cd .. git clone https://github.com/seqeralabs/nf-canary cd nf-canary cat <> nextflow.config @@ -61,8 +58,10 @@ jobs: process.executor = 'tes' tes.endpoint = 'http://localhost:8000' EOF + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 - name: Run nf-canary tests run: | - cd ../nf-canary - ../nextflow/launch.sh run main.nf + ../nextflow run main.nf From 69bddbe2b86a4d1c3fa9ff8b9b2a70e62bd795c0 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Wed, 19 Jun 2024 20:00:08 -0700 Subject: [PATCH 33/67] Update file copying behavior for Nextflow tests --- .github/workflows/nextflow.yaml | 5 +- storage/local.go | 169 +++++++++++++++++--------------- 2 files changed, 89 insertions(+), 85 deletions(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index 8d5206321..5f6c20ac1 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -43,7 +43,6 @@ jobs: - name: Install Nextflow run: | - cd .. curl -s https://get.nextflow.io | bash chmod +x nextflow @@ -59,9 +58,7 @@ jobs: tes.endpoint = 'http://localhost:8000' EOF - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: Run nf-canary tests run: | + cd nf-canary ../nextflow run main.nf diff --git a/storage/local.go b/storage/local.go index a1016e632..db0d39fac 100644 --- a/storage/local.go +++ b/storage/local.go @@ -148,6 +148,15 @@ func copyFile(ctx context.Context, source string, dest string) (err error) { } defer sf.Close() + // If dest is a directory, create a file in that directory with the same name as source + fileInfo, err := os.Stat(dest) + if err != nil { + return fmt.Errorf("failed to get file info: %v", err) + } + if fileInfo.IsDir() { + dest = filepath.Join(dest, filepath.Base(source)) + } + // Create and open dest file for writing df, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY, 0775) if err != nil { @@ -169,103 +178,101 @@ func copyFile(ctx context.Context, source string, dest string) (err error) { // Hard links file source to destination dest. func linkFile(ctx context.Context, source string, dest string) error { - // If source has a glob or wildcard, get the filepath using the filepath.Glob function - if strings.Contains(source, "*") { - globs, err := filepath.Glob(source) - if err != nil { - return fmt.Errorf("failed to get filepath using Glob: %v", err) - } - for _, glob := range globs { - // Correctly calculate the destination for each file - err := processItem(ctx, glob, dest) - if err != nil { - return err - } - } - return nil - } else { - return processItem(ctx, source, dest) - } + // If source has a glob or wildcard, get the filepath using the filepath.Glob function + if strings.Contains(source, "*") { + globs, err := filepath.Glob(source) + if err != nil { + return fmt.Errorf("failed to get filepath using Glob: %v", err) + } + for _, glob := range globs { + // Correctly calculate the destination for each file + err := processItem(ctx, glob, dest) + if err != nil { + return err + } + } + return nil + } else { + return processItem(ctx, source, dest) + } } - // Process a single item (file or directory) func processItem(ctx context.Context, source, dest string) error { - fileInfo, err := os.Stat(source) - if err != nil { - return err - } - + fileInfo, err := os.Stat(source) + if err != nil { + return err + } - if fileInfo.IsDir() { - return processDirectory(ctx, source, dest) - } else { - return processFile(ctx, source, dest) - } + if fileInfo.IsDir() { + return processDirectory(ctx, source, dest) + } else { + return processFile(ctx, source, dest) + } } // Process a directory func processDirectory(ctx context.Context, source, dest string) error { - // Create destination directory - err := os.MkdirAll(dest, 0755) // Adjust permissions as needed - if err != nil { - return err - } - - entries, err := os.ReadDir(source) - if err != nil { - return err - } - - for _, entry := range entries { - srcPath := filepath.Join(source, entry.Name()) - destPath := filepath.Join(dest, entry.Name()) - - if entry.IsDir() { - err = processDirectory(ctx, srcPath, destPath) - } else { - err = processFile(ctx, srcPath, destPath) - } - - if err != nil { - return err - } - } - return nil + // Create destination directory + err := os.MkdirAll(dest, 0755) // Adjust permissions as needed + if err != nil { + return err + } + + entries, err := os.ReadDir(source) + if err != nil { + return err + } + + for _, entry := range entries { + srcPath := filepath.Join(source, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) + + if entry.IsDir() { + err = processDirectory(ctx, srcPath, destPath) + } else { + err = processFile(ctx, srcPath, destPath) + } + + if err != nil { + return err + } + } + return nil } // Process a single file func processFile(ctx context.Context, source, dest string) error { - // without this resulting link could be a symlink - parent, err := filepath.EvalSymlinks(source) - if err != nil { - return err - } - - same, err := sameFile(parent, dest) - if err != nil { - return err - } - if same { - return nil - } - - err = os.Link(parent, dest) - if err != nil { - return copyFile(ctx, parent, dest) - } - return nil + // without this resulting link could be a symlink + parent, err := filepath.EvalSymlinks(source) + if err != nil { + return err + } + + same, err := sameFile(parent, dest) + if err != nil { + return err + } + if same { + return nil + } + + err = os.Link(parent, dest) + if err != nil { + return copyFile(ctx, parent, dest) + } + return nil } func FilePathWalkDir(root string) ([]string, error) { - var files []string - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { - files = append(files, path) - } - return nil - }) - return files, err + var files []string + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + files = append(files, path) + } + return nil + }) + return files, err } func sameFile(source string, dest string) (bool, error) { From 2af9c4d308f057214011a202e4f1aeea9666aa45 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 20 Jun 2024 14:20:45 -0700 Subject: [PATCH 34/67] Update local file copying behavior to support directory dests --- .github/workflows/nextflow.yaml | 3 +++ server/server.go | 40 ++++++++++++++++----------------- storage/local.go | 8 +++---- storage/local_test.go | 36 +++++++++++++++++++++++++++++ tes/utils_test.go | 3 +-- 5 files changed, 64 insertions(+), 26 deletions(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index 5f6c20ac1..bfb22038e 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -58,6 +58,9 @@ jobs: tes.endpoint = 'http://localhost:8000' EOF + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Run nf-canary tests run: | cd nf-canary diff --git a/server/server.go b/server/server.go index 02b6aab6b..4c3a7ecb9 100644 --- a/server/server.go +++ b/server/server.go @@ -62,16 +62,16 @@ func newDebugInterceptor(log *logger.Logger) grpc.UnaryServerInterceptor { // Returns '400' for invalid backend parameters and '500' for all other errors // Required for TES Compliance Tests func customErrorHandler(ctx context.Context, mux *runtime.ServeMux, marshaler runtime.Marshaler, w http.ResponseWriter, r *http.Request, err error) { - const fallback = `{"error": "failed to process the request"}` + const fallback = `{"error": "failed to process the request"}` - st, ok := status.FromError(err) - if !ok { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fallback)) - return - } + st, ok := status.FromError(err) + if !ok { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fallback)) + return + } - // Map specific gRPC error codes to HTTP status codes + // Map specific gRPC error codes to HTTP status codes switch st.Code() { case codes.Unauthenticated: w.WriteHeader(http.StatusUnauthorized) // 401 @@ -79,31 +79,31 @@ func customErrorHandler(ctx context.Context, mux *runtime.ServeMux, marshaler ru w.WriteHeader(http.StatusForbidden) // 403 case codes.NotFound: // Special case for missing tasks (TES Compliance Suite) - if (strings.Contains(st.Message(), "task not found")) { + if strings.Contains(st.Message(), "task not found") { w.WriteHeader(http.StatusInternalServerError) // 500 } else { w.WriteHeader(http.StatusNotFound) // 404 } default: - if (strings.Contains(st.Message(), "backend parameters not supported")) { + if strings.Contains(st.Message(), "backend parameters not supported") { w.WriteHeader(http.StatusBadRequest) // 400 } else { w.WriteHeader(http.StatusInternalServerError) // 500 } } - // Write the error message - jErr := JSONError{Error: st.Message()} - jErrBytes, mErr := marshaler.Marshal(jErr) - if mErr != nil { - w.Write([]byte(fallback)) - return - } - w.Write(jErrBytes) + // Write the error message + jErr := JSONError{Error: st.Message()} + jErrBytes, mErr := marshaler.Marshal(jErr) + if mErr != nil { + w.Write([]byte(fallback)) + return + } + w.Write(jErrBytes) } type JSONError struct { - Error string `json:"error"` + Error string `json:"error"` } // Serve starts the server and does not block. This will open TCP ports @@ -138,7 +138,7 @@ func (s *Server) Serve(pctx context.Context) error { marsh := NewMarshaler() grpcMux := runtime.NewServeMux( runtime.WithMarshalerOption(runtime.MIMEWildcard, marsh), runtime.WithErrorHandler(customErrorHandler)) - + // m := protojson.MarshalOptions{ // Indent: " ", // EmitUnpopulated: true, diff --git a/storage/local.go b/storage/local.go index db0d39fac..1c74bff36 100644 --- a/storage/local.go +++ b/storage/local.go @@ -148,7 +148,7 @@ func copyFile(ctx context.Context, source string, dest string) (err error) { } defer sf.Close() - // If dest is a directory, create a file in that directory with the same name as source + // If dest is a directory, set destination to be to be a file in that directory fileInfo, err := os.Stat(dest) if err != nil { return fmt.Errorf("failed to get file info: %v", err) @@ -198,7 +198,7 @@ func linkFile(ctx context.Context, source string, dest string) error { } // Process a single item (file or directory) -func processItem(ctx context.Context, source, dest string) error { +func processItem(ctx context.Context, source string, dest string) error { fileInfo, err := os.Stat(source) if err != nil { return err @@ -212,7 +212,7 @@ func processItem(ctx context.Context, source, dest string) error { } // Process a directory -func processDirectory(ctx context.Context, source, dest string) error { +func processDirectory(ctx context.Context, source string, dest string) error { // Create destination directory err := os.MkdirAll(dest, 0755) // Adjust permissions as needed if err != nil { @@ -242,7 +242,7 @@ func processDirectory(ctx context.Context, source, dest string) error { } // Process a single file -func processFile(ctx context.Context, source, dest string) error { +func processFile(ctx context.Context, source string, dest string) error { // without this resulting link could be a symlink parent, err := filepath.EvalSymlinks(source) if err != nil { diff --git a/storage/local_test.go b/storage/local_test.go index 587b6443e..c87ab8339 100644 --- a/storage/local_test.go +++ b/storage/local_test.go @@ -234,6 +234,42 @@ func TestLocalPutPath(t *testing.T) { } } +func TestProcessFile(t *testing.T) { + ctx := context.Background() + tmp, err := os.MkdirTemp("", "funnel-test-local-storage") + if err != nil { + t.Fatal(err) + } + + cp := path.Join(tmp, "input.txt") + op := path.Join(tmp, "output.txt") + os.WriteFile(cp, []byte("foo"), os.ModePerm) + + err = processFile(ctx, cp, op) + if err != nil { + t.Fatal(err) + } + + b, err := os.ReadFile(op) + if err != nil { + t.Fatal(err) + } + if string(b) != "foo" { + t.Fatal("Unexpected content") + } + + dir := path.Join(tmp, "dir") + err = os.Mkdir(dir, os.ModePerm) + if err != nil { + t.Fatal(err) + } + + err = processFile(ctx, cp, dir) + if err != nil { + t.Fatal(err) + } +} + // Tests Put when source and dest reference the same file (inode) // Since Local storage hard-links files when possible we need to protect // against the case where the same path is 'Put' twice diff --git a/tes/utils_test.go b/tes/utils_test.go index 9f2977a4b..b89a6c1b2 100644 --- a/tes/utils_test.go +++ b/tes/utils_test.go @@ -16,8 +16,7 @@ func TestBase64Encode(t *testing.T) { }, } - // expected := "ewogICJleGVjdXRvcnMiOiBbCiAgICB7CiAgICAgICJjb21tYW5kIjogWwogICAgICAgICJlY2hvIiwKICAgICAgICAiaGVsbG8gd29ybGQiCiAgICAgIF0sCiAgICAgICJpbWFnZSI6ICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAidGFzazEiCn0=" - expected := "ewogICJleGVjdXRvcnMiOiAgWwogICAgewogICAgICAiY29tbWFuZCI6ICBbCiAgICAgICAgImVjaG8iLAogICAgICAgICJoZWxsbyB3b3JsZCIKICAgICAgXSwKICAgICAgImltYWdlIjogICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAgInRhc2sxIgp9" + expected := "ewogICJleGVjdXRvcnMiOiBbCiAgICB7CiAgICAgICJjb21tYW5kIjogWwogICAgICAgICJlY2hvIiwKICAgICAgICAiaGVsbG8gd29ybGQiCiAgICAgIF0sCiAgICAgICJpbWFnZSI6ICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAidGFzazEiCn0=" encoded, err := Base64Encode(task) if err != nil { From 0f14338410142b8e0b65a5f1f209ed0bc28a9148 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 20 Jun 2024 15:37:12 -0700 Subject: [PATCH 35/67] Update local dest for files with glob patterns --- storage/local.go | 10 +--------- storage/local_test.go | 11 ----------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/storage/local.go b/storage/local.go index 1c74bff36..9b034332b 100644 --- a/storage/local.go +++ b/storage/local.go @@ -148,15 +148,6 @@ func copyFile(ctx context.Context, source string, dest string) (err error) { } defer sf.Close() - // If dest is a directory, set destination to be to be a file in that directory - fileInfo, err := os.Stat(dest) - if err != nil { - return fmt.Errorf("failed to get file info: %v", err) - } - if fileInfo.IsDir() { - dest = filepath.Join(dest, filepath.Base(source)) - } - // Create and open dest file for writing df, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY, 0775) if err != nil { @@ -186,6 +177,7 @@ func linkFile(ctx context.Context, source string, dest string) error { } for _, glob := range globs { // Correctly calculate the destination for each file + dest = filepath.Join(dest, filepath.Base(glob)) err := processItem(ctx, glob, dest) if err != nil { return err diff --git a/storage/local_test.go b/storage/local_test.go index c87ab8339..9631cd83c 100644 --- a/storage/local_test.go +++ b/storage/local_test.go @@ -257,17 +257,6 @@ func TestProcessFile(t *testing.T) { if string(b) != "foo" { t.Fatal("Unexpected content") } - - dir := path.Join(tmp, "dir") - err = os.Mkdir(dir, os.ModePerm) - if err != nil { - t.Fatal(err) - } - - err = processFile(ctx, cp, dir) - if err != nil { - t.Fatal(err) - } } // Tests Put when source and dest reference the same file (inode) From 85ce5debad541d2a507c773e4290c37cac3ded53 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 20 Jun 2024 15:43:22 -0700 Subject: [PATCH 36/67] Update Base64Encode test to match Github Actions test - TODO: investigate why this is necessary... --- .github/workflows/nextflow.yaml | 3 --- tes/utils_test.go | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index bfb22038e..5f6c20ac1 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -58,9 +58,6 @@ jobs: tes.endpoint = 'http://localhost:8000' EOF - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: Run nf-canary tests run: | cd nf-canary diff --git a/tes/utils_test.go b/tes/utils_test.go index b89a6c1b2..af33afc3b 100644 --- a/tes/utils_test.go +++ b/tes/utils_test.go @@ -16,6 +16,8 @@ func TestBase64Encode(t *testing.T) { }, } + // TODO: Investigate strange whitespace behavior in Github Actions + // expected := "ewogICJleGVjdXRvcnMiOiBbCiAgICB7CiAgICAgICJjb21tYW5kIjogWwogICAgICAgICJlY2hvIiwKICAgICAgICAiaGVsbG8gd29ybGQiCiAgICAgIF0sCiAgICAgICJpbWFnZSI6ICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAidGFzazEiCn0=" expected := "ewogICJleGVjdXRvcnMiOiBbCiAgICB7CiAgICAgICJjb21tYW5kIjogWwogICAgICAgICJlY2hvIiwKICAgICAgICAiaGVsbG8gd29ybGQiCiAgICAgIF0sCiAgICAgICJpbWFnZSI6ICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAidGFzazEiCn0=" encoded, err := Base64Encode(task) From 56ec776c41d6edcef8e8073a0216744743776239 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 20 Jun 2024 15:55:08 -0700 Subject: [PATCH 37/67] Fix Base64Encoding test value --- tes/utils_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes/utils_test.go b/tes/utils_test.go index af33afc3b..b903286e6 100644 --- a/tes/utils_test.go +++ b/tes/utils_test.go @@ -18,7 +18,7 @@ func TestBase64Encode(t *testing.T) { // TODO: Investigate strange whitespace behavior in Github Actions // expected := "ewogICJleGVjdXRvcnMiOiBbCiAgICB7CiAgICAgICJjb21tYW5kIjogWwogICAgICAgICJlY2hvIiwKICAgICAgICAiaGVsbG8gd29ybGQiCiAgICAgIF0sCiAgICAgICJpbWFnZSI6ICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAidGFzazEiCn0=" - expected := "ewogICJleGVjdXRvcnMiOiBbCiAgICB7CiAgICAgICJjb21tYW5kIjogWwogICAgICAgICJlY2hvIiwKICAgICAgICAiaGVsbG8gd29ybGQiCiAgICAgIF0sCiAgICAgICJpbWFnZSI6ICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAidGFzazEiCn0=" + expected := "ewogICJleGVjdXRvcnMiOiAgWwogICAgewogICAgICAiY29tbWFuZCI6ICBbCiAgICAgICAgImVjaG8iLAogICAgICAgICJoZWxsbyB3b3JsZCIKICAgICAgXSwKICAgICAgImltYWdlIjogICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAgInRhc2sxIgp9" encoded, err := Base64Encode(task) if err != nil { From 4bbec3f4e43d4fa8ada46d2225b488fefedf1e30 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 20 Jun 2024 17:37:06 -0700 Subject: [PATCH 38/67] Revert glob output change based on TesOutput spec - Wildcards in outputs will be uploaded to a directory, with no special handling for the filename. ```yaml tesOutput: properties: url: description: |- URL at which the TES server makes the output accessible after the task is complete. When tesOutput.path contains wildcards, it must be a directory; see `tesOutput.path_prefix` for details on how output URLs are constructed in this case. ``` --- storage/local.go | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/local.go b/storage/local.go index 9b034332b..f57cc9ff5 100644 --- a/storage/local.go +++ b/storage/local.go @@ -177,7 +177,6 @@ func linkFile(ctx context.Context, source string, dest string) error { } for _, glob := range globs { // Correctly calculate the destination for each file - dest = filepath.Join(dest, filepath.Base(glob)) err := processItem(ctx, glob, dest) if err != nil { return err From ece1266849e1db072d428ff8a4c9867d79094859 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 20 Jun 2024 17:49:58 -0700 Subject: [PATCH 39/67] Debug: Add tmate step to Nextflow tests --- .github/workflows/nextflow.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index 5f6c20ac1..ee494f1ec 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -58,6 +58,9 @@ jobs: tes.endpoint = 'http://localhost:8000' EOF + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Run nf-canary tests run: | cd nf-canary From c81d0a7b454299bfebaf41a65709d48d47681e66 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 20 Jun 2024 18:22:42 -0700 Subject: [PATCH 40/67] Update wildcard file output destination --- .github/workflows/nextflow.yaml | 3 --- storage/local.go | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index ee494f1ec..5f6c20ac1 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -58,9 +58,6 @@ jobs: tes.endpoint = 'http://localhost:8000' EOF - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: Run nf-canary tests run: | cd nf-canary diff --git a/storage/local.go b/storage/local.go index f57cc9ff5..8ee1881c0 100644 --- a/storage/local.go +++ b/storage/local.go @@ -177,6 +177,8 @@ func linkFile(ctx context.Context, source string, dest string) error { } for _, glob := range globs { // Correctly calculate the destination for each file + // TODO: Verify that path_prefix is being removed from the dest filepath + dest = filepath.Join(dest, filepath.Base(glob)) err := processItem(ctx, glob, dest) if err != nil { return err From f7905e558ccc62faa6787695f3d1c7f0cb383756 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 22 Jul 2024 18:11:57 -0700 Subject: [PATCH 41/67] Add initial command templating for container engines --- cmd/util/flags.go | 3 +- config/config.go | 23 ++++++++-- config/default.go | 14 +++++- worker/container_engine.go | 17 +++----- worker/docker.go | 88 +++++++++++++++++--------------------- worker/worker.go | 11 ++--- 6 files changed, 86 insertions(+), 70 deletions(-) diff --git a/cmd/util/flags.go b/cmd/util/flags.go index 6efca3ffd..8bac7cfe4 100644 --- a/cmd/util/flags.go +++ b/cmd/util/flags.go @@ -102,8 +102,7 @@ func workerFlags(flagConf *config.Config) *pflag.FlagSet { f.StringVar(&flagConf.Worker.WorkDir, "Worker.WorkDir", flagConf.Worker.WorkDir, "Working directory") f.StringVar(&flagConf.Worker.ScratchPath, "Worker.ScratchPath", flagConf.Worker.ScratchPath, "Scratch directory") f.BoolVar(&flagConf.Worker.LeaveWorkDir, "Worker.LeaveWorkDir", flagConf.Worker.LeaveWorkDir, "Leave working directory after execution") - f.StringVar(&flagConf.Worker.ContainerType, "Worker.ContainerType", flagConf.Worker.ContainerType, "Container engine to use for executing tasks. One of ['docker', 'exadocker']") - f.StringVar(&flagConf.Worker.ContainerDriver, "Worker.ContainerDriver", flagConf.Worker.ContainerDriver, "Overrides the default command used to run containers.") + f.StringVar(&flagConf.Worker.DriverCommand, "Worker.DriverCommand", flagConf.Worker.DriverCommand, "Overrides the default command used to run containers.") return f } diff --git a/config/config.go b/config/config.go index 66c183581..24cb517b4 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( + "io" "os" "github.com/ohsu-comp-bio/funnel/logger" @@ -154,11 +155,27 @@ type Worker struct { // Limit the number of concurrent downloads/uploads MaxParallelTransfers int // Container engine to use for executing tasks. - // Typically this is "docker". - ContainerType string + Container ContainerConfig // Command to use for the container engine. // This can be used to override the default command used to run containers. - ContainerDriver string + DriverCommand string +} + +type ContainerConfig struct { + Id string + Image string + Name string + Command []string + Workdir string + RemoveContainer bool + Env map[string]string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + DriverCommand string + RunCommand string // template string + PullCommand string // template string + StopCommand string // template string } // HPCBackend describes the configuration for a HPC scheduler backend such as diff --git a/config/default.go b/config/default.go index 97ff07364..1da4e3220 100644 --- a/config/default.go +++ b/config/default.go @@ -61,8 +61,18 @@ func DefaultConfig() Config { LogUpdateRate: Duration(time.Second * 5), LogTailSize: 10000, MaxParallelTransfers: 10, - ContainerType: "docker", - ContainerDriver: "docker", + Container: ContainerConfig{ + DriverCommand: "docker", + RunCommand: `run -i --read-only +{{if .RemoveContainer}}--rm{{end}} +{{range $k, $v := .Env}}-e {{$k}}={{$v}} {{end}} +{{if .Name}}--name {{.Name}}{{end}} +{{if .Workdir}}-w {{.Workdir}}{{end}} +{{range .Volumes}}-v {{.HostPath}}:{{.ContainerPath}}:{{if .Readonly}}ro{{else}}rw{{end}}{{end}} +{{.Image}} {{.Command}}`, + PullCommand: "pull {{.Image}}", + StopCommand: "rm -f {{.Name}}", + }, }, Logger: logger.DefaultConfig(), // databases / event handlers diff --git a/worker/container_engine.go b/worker/container_engine.go index 6178f8e3d..2d07a20ac 100644 --- a/worker/container_engine.go +++ b/worker/container_engine.go @@ -2,7 +2,6 @@ package worker import ( "context" - "fmt" "io" "github.com/ohsu-comp-bio/funnel/events" @@ -35,8 +34,7 @@ type ContainerConfig struct { Id string Image string Name string - DriverCommand []string - Command []string + Command string Volumes []Volume Workdir string RemoveContainer bool @@ -45,6 +43,10 @@ type ContainerConfig struct { Stdout io.Writer Stderr io.Writer Event *events.ExecutorWriter + DriverCommand string + RunCommand string // template string + PullCommand string // template string + StopCommand string // template string } type ContainerVersion struct { @@ -54,13 +56,8 @@ type ContainerVersion struct { type ContainerEngineFactory struct{} -func (f *ContainerEngineFactory) NewContainerEngine(containerType string, containerConfig ContainerConfig) (ContainerEngine, error) { - switch containerType { - case "docker": - return NewDockerEngine(containerConfig) - default: - return nil, fmt.Errorf("unsupported container type: %s", containerType) - } +func (f *ContainerEngineFactory) NewContainerEngine(containerConfig ContainerConfig) (ContainerEngine, error) { + return NewDockerEngine(containerConfig) } func NewDockerEngine(config ContainerConfig) (ContainerEngine, error) { diff --git a/worker/docker.go b/worker/docker.go index fd8e9793a..c5c88819d 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -1,6 +1,7 @@ package worker import ( + "bytes" "context" "encoding/json" "fmt" @@ -8,6 +9,7 @@ import ( "os" "os/exec" "strings" + "text/template" "time" ) @@ -24,58 +26,57 @@ func (docker Docker) Run(ctx context.Context) error { docker.Event.Error("failed to sync docker client API version", err) } - var pullArgs []string - - if len(docker.ContainerConfig.DriverCommand) > 1 { - // Merge driver parts and command parts - pullArgs = append(pullArgs, docker.ContainerConfig.DriverCommand[1:]...) - } - - pullArgs = append(pullArgs, "pull", docker.Image) - pullcmd := exec.Command(docker.ContainerConfig.DriverCommand[0], pullArgs...) - err = pullcmd.Run() + err = docker.executeCommand(ctx, docker.PullCommand) if err != nil { docker.Event.Error("failed to pull docker image", err) } - var args []string - - if len(docker.ContainerConfig.DriverCommand) > 1 { - // Merge driver parts and command parts - args = append(args, docker.ContainerConfig.DriverCommand[1:]...) + err = docker.executeCommand(ctx, docker.RunCommand) + if err != nil { + docker.Event.Error("failed to run docker container", err) } - args = append(args, "run", "-i", "--read-only") + go docker.InspectContainer(ctx) + return nil +} - if docker.RemoveContainer { - args = append(args, "--rm") +// Stop stops the container. +func (docker Docker) Stop() error { + docker.Event.Info("Stopping container", "container", docker.Name) + // cmd := exec.Command("docker", "stop", docker.Name) + // cmd := exec.Command("docker", "rm", "-f", docker.Name) //switching to this to be a bit more forceful + // return cmd.Run() + err := docker.executeCommand(context.Background(), docker.StopCommand) + if err != nil { + docker.Event.Error("failed to stop docker container", err) + return err } + return nil +} - if docker.Env != nil { - for k, v := range docker.Env { - args = append(args, "-e", fmt.Sprintf("%s=%s", k, v)) - } +func (docker Docker) executeCommand(ctx context.Context, commandTemplate string) error { + tmpl, err := template.New("command").Parse(commandTemplate) + if err != nil { + return fmt.Errorf("failed to parse template for command: %w", err) } - if docker.Name != "" { - args = append(args, "--name", docker.Name) + var cmdBuffer bytes.Buffer + err = tmpl.Execute(&cmdBuffer, docker.ContainerConfig) + if err != nil { + return fmt.Errorf("failed to execute template for command: %w", err) } - if docker.Workdir != "" { - args = append(args, "-w", docker.Workdir) - } + cmdParts := strings.Fields(cmdBuffer.String()) + driverCmd := strings.Fields(docker.DriverCommand) - for _, vol := range docker.Volumes { - arg := formatVolumeArg(vol) - args = append(args, "-v", arg) + var cmd *exec.Cmd + if len(driverCmd) > 1 { + cmdArgs := append(driverCmd[1:], cmdParts...) + cmd = exec.CommandContext(ctx, driverCmd[0], cmdArgs...) + } else { + cmd = exec.CommandContext(ctx, driverCmd[0], cmdParts...) } - args = append(args, docker.Image) - args = append(args, docker.Command...) - - // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` - cmd := exec.Command(docker.ContainerConfig.DriverCommand[0], args...) - if docker.Stdin != nil { cmd.Stdin = docker.Stdin } @@ -85,17 +86,7 @@ func (docker Docker) Run(ctx context.Context) error { if docker.Stderr != nil { cmd.Stderr = docker.Stderr } - go docker.InspectContainer(ctx) - out := cmd.Run() - docker.Event.Info("Command %s Complete exit=%s", strings.Join(args, " "), out) - return out -} -// Stop stops the container. -func (docker Docker) Stop() error { - docker.Event.Info("Stopping container", "container", docker.Name) - // cmd := exec.Command("docker", "stop", docker.Name) - cmd := exec.Command("docker", "rm", "-f", docker.Name) //switching to this to be a bit more forceful return cmd.Run() } @@ -167,14 +158,15 @@ func (docker *Docker) InspectContainer(ctx context.Context) ContainerConfig { func (docker *Docker) SyncAPIVersion() error { if os.Getenv("DOCKER_API_VERSION") == "" { var args []string + driverCmd := strings.Fields(docker.ContainerConfig.DriverCommand) if len(docker.ContainerConfig.DriverCommand) > 1 { // Merge driver parts and command parts - args = append(args, docker.ContainerConfig.DriverCommand[1:]...) + args = append(args, driverCmd[1:]...) } args = append(args, "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) - cmd := exec.Command(docker.ContainerConfig.DriverCommand[0], args...) + cmd := exec.Command(driverCmd[0], args...) out, err := cmd.Output() if err != nil { return fmt.Errorf("docker version command failed: %v", err) diff --git a/worker/worker.go b/worker/worker.go index 06cd98f12..5ceb3ea9a 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -163,7 +163,7 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { for i, d := range task.GetExecutors() { containerConfig := ContainerConfig{ Image: d.Image, - Command: d.Command, + Command: strings.Join(d.Command, " "), Env: d.Env, Volumes: mapper.Volumes, Workdir: d.Workdir, @@ -172,10 +172,11 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { RemoveContainer: true, Event: event.NewExecutorWriter(uint32(i)), } - if r.Conf.ContainerDriver != "" { - containerConfig.DriverCommand = strings.Fields(r.Conf.ContainerDriver) - } - containerEngine, err := f.NewContainerEngine(r.Conf.ContainerType, containerConfig) + containerConfig.DriverCommand = r.Conf.Container.DriverCommand + containerConfig.RunCommand = r.Conf.Container.RunCommand + containerConfig.PullCommand = r.Conf.Container.PullCommand + containerConfig.StopCommand = r.Conf.Container.StopCommand + containerEngine, err := f.NewContainerEngine(containerConfig) if err != nil { run.syserr = err } From edc1600bfe6b7798de7648f1a34f0dca05fac91a Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 22 Jul 2024 18:17:11 -0700 Subject: [PATCH 42/67] Add initial Docker/ContainerEngine tests --- worker/container_engine.go | 6 +- worker/docker_test.go | 127 +++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 worker/docker_test.go diff --git a/worker/container_engine.go b/worker/container_engine.go index 2d07a20ac..ea35d11e2 100644 --- a/worker/container_engine.go +++ b/worker/container_engine.go @@ -57,11 +57,7 @@ type ContainerVersion struct { type ContainerEngineFactory struct{} func (f *ContainerEngineFactory) NewContainerEngine(containerConfig ContainerConfig) (ContainerEngine, error) { - return NewDockerEngine(containerConfig) -} - -func NewDockerEngine(config ContainerConfig) (ContainerEngine, error) { return &Docker{ - ContainerConfig: config, + ContainerConfig: containerConfig, }, nil } diff --git a/worker/docker_test.go b/worker/docker_test.go new file mode 100644 index 000000000..90908bcea --- /dev/null +++ b/worker/docker_test.go @@ -0,0 +1,127 @@ +package worker + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/ohsu-comp-bio/funnel/events" + "github.com/ohsu-comp-bio/funnel/logger" +) + +var defaultConfig = ContainerConfig{ + Id: "123", + Image: "alpine", + Name: "funnel-test", + DriverCommand: "docker", + Command: "echo Hello, World!", + RunCommand: "run --name {{.Name}} {{.Image}} {{range .Command}} {{.}} {{end}}", + PullCommand: "pull {{.Image}}", + RemoveContainer: true, + Event: events.NewExecutorWriter("123", 1, 1, &events.Logger{ + Log: logger.NewLogger("test", logger.DefaultConfig()), + }), +} + +func TestDockerRun(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + err := docker.Run(context.Background()) + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } +} + +func TestDockerExecuteCommand(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + err := docker.executeCommand(context.Background(), "run --rm alpine echo Hello, World!") + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } +} + +func TestDockerStop(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + ctx, cancel := context.WithCancel(context.Background()) + + // Run the container command in a separate goroutine + go func() { + err := docker.executeCommand(ctx, "run --rm alpine sleep 30") + if err != nil && ctx.Err() == nil { + t.Errorf("Expected no error, but got: %v", err) + } + }() + + // Give the container some time to start + time.Sleep(2 * time.Second) + + // Stop the container + err := docker.Stop() + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } + + // Cancel the context to stop the goroutine if it is still running + cancel() +} + +func TestFormatVolumeArg(t *testing.T) { + volume := Volume{ + HostPath: "/path/to/source", + ContainerPath: "/path/to/destination", + Readonly: true, + } + expected := "/path/to/source:/path/to/destination:ro" + result := formatVolumeArg(volume) + if result != expected { + t.Errorf("Expected %s, but got %s", expected, result) + } +} + +func TestDockerGetImage(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + expected := "alpine" + result := docker.GetImage() + if result != expected { + t.Errorf("Expected %s, but got %s", expected, result) + } +} + +func TestDockerSetIO(t *testing.T) { + docker := Docker{} + stdin := &bytes.Buffer{} + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + docker.SetIO(stdin, stdout, stderr) + if docker.Stdin != stdin || docker.Stdout != stdout || docker.Stderr != stderr { + t.Errorf("Expected stdin, stdout, and stderr to be set correctly") + } +} + +func TestDockerInspectContainer(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + config := docker.InspectContainer(context.Background()) + if config.Id == "" { + t.Errorf("Expected non-nil container config") + } +} + +func TestDockerSyncAPIVersion(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + err := docker.SyncAPIVersion() + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } +} From b3e6066269f96039f025feb1bf5fd1f824458a30 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Wed, 24 Jul 2024 16:33:59 -0700 Subject: [PATCH 43/67] Run `go mod tidy` --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 091fd1a5c..715ad82bb 100644 --- a/go.mod +++ b/go.mod @@ -115,7 +115,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/yaml v0.2.0 // indirect + github.com/invopop/yaml v0.3.1 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect diff --git a/go.sum b/go.sum index d7d6ef2f3..b701ac2c0 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cO github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= -github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= +github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -625,7 +625,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= From 20c523880727f99cc3adbf00da24fac48283751d Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Wed, 24 Jul 2024 16:52:15 -0700 Subject: [PATCH 44/67] Update Github Actions --- .github/workflows/compliance-test.yaml | 4 +++- .github/workflows/nextflow.yaml | 4 +++- .github/workflows/tests.yaml | 12 ++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/compliance-test.yaml b/.github/workflows/compliance-test.yaml index 7cf1817e6..69d9111a4 100644 --- a/.github/workflows/compliance-test.yaml +++ b/.github/workflows/compliance-test.yaml @@ -25,6 +25,8 @@ name: Compliance Test on: push: + branches: + - main jobs: build: @@ -32,7 +34,7 @@ jobs: container: quay.io/ohsu-comp-bio/slurm steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index 47945b3f6..0aef58ef4 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -2,13 +2,15 @@ name: Nextflow Test on: push: + branches: + - main jobs: build: runs-on: ubuntu-latest steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ba170dda0..3f9e315d8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 - name: Check out code @@ -57,7 +57,7 @@ jobs: needs: build steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 @@ -81,7 +81,7 @@ jobs: needs: build steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 - name: Check out code @@ -101,7 +101,7 @@ jobs: needs: build steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 - name: Check out code @@ -122,7 +122,7 @@ jobs: needs: build steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 - name: Check out code From 80bf9a788e5e7ad1d0cd51e62ffe3c705ec244ae Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jul 2024 16:56:34 -0700 Subject: [PATCH 45/67] Update container engine IO and command execution --- config/default.go | 14 ++++++------ worker/container_engine.go | 2 +- worker/docker.go | 46 +++++++++++++++++++++++--------------- worker/docker_test.go | 22 +++++++++++++----- worker/worker.go | 3 +-- 5 files changed, 54 insertions(+), 33 deletions(-) diff --git a/config/default.go b/config/default.go index 1da4e3220..ca7ad2112 100644 --- a/config/default.go +++ b/config/default.go @@ -63,13 +63,13 @@ func DefaultConfig() Config { MaxParallelTransfers: 10, Container: ContainerConfig{ DriverCommand: "docker", - RunCommand: `run -i --read-only -{{if .RemoveContainer}}--rm{{end}} -{{range $k, $v := .Env}}-e {{$k}}={{$v}} {{end}} -{{if .Name}}--name {{.Name}}{{end}} -{{if .Workdir}}-w {{.Workdir}}{{end}} -{{range .Volumes}}-v {{.HostPath}}:{{.ContainerPath}}:{{if .Readonly}}ro{{else}}rw{{end}}{{end}} -{{.Image}} {{.Command}}`, + RunCommand: "run -i --read-only " + + "{{if .RemoveContainer}}--rm{{end}} " + + "{{range $k, $v := .Env}}-e {{$k}}={{$v}} {{end}} " + + "{{if .Name}}--name {{.Name}}{{end}} " + + "{{if .Workdir}}-w {{.Workdir}}{{end}} " + + "{{range .Volumes}}-v {{.HostPath}}:{{.ContainerPath}}:{{if .Readonly}}ro{{else}}rw{{end}} {{end}} " + + "{{.Image}} {{.Command}}", PullCommand: "pull {{.Image}}", StopCommand: "rm -f {{.Name}}", }, diff --git a/worker/container_engine.go b/worker/container_engine.go index ea35d11e2..b0a8915e8 100644 --- a/worker/container_engine.go +++ b/worker/container_engine.go @@ -34,7 +34,7 @@ type ContainerConfig struct { Id string Image string Name string - Command string + Command []string Volumes []Volume Workdir string RemoveContainer bool diff --git a/worker/docker.go b/worker/docker.go index c5c88819d..0e80285b5 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -26,27 +26,23 @@ func (docker Docker) Run(ctx context.Context) error { docker.Event.Error("failed to sync docker client API version", err) } - err = docker.executeCommand(ctx, docker.PullCommand) + err = docker.executeCommand(ctx, docker.PullCommand, false) if err != nil { docker.Event.Error("failed to pull docker image", err) } - err = docker.executeCommand(ctx, docker.RunCommand) + err = docker.executeCommand(ctx, docker.RunCommand, true) if err != nil { docker.Event.Error("failed to run docker container", err) } - go docker.InspectContainer(ctx) - return nil + return err } // Stop stops the container. func (docker Docker) Stop() error { docker.Event.Info("Stopping container", "container", docker.Name) - // cmd := exec.Command("docker", "stop", docker.Name) - // cmd := exec.Command("docker", "rm", "-f", docker.Name) //switching to this to be a bit more forceful - // return cmd.Run() - err := docker.executeCommand(context.Background(), docker.StopCommand) + err := docker.executeCommand(context.Background(), docker.StopCommand, false) if err != nil { docker.Event.Error("failed to stop docker container", err) return err @@ -54,7 +50,13 @@ func (docker Docker) Stop() error { return nil } -func (docker Docker) executeCommand(ctx context.Context, commandTemplate string) error { +func (docker Docker) executeCommand(ctx context.Context, commandTemplate string, enableIO bool) error { + var usingCommand bool = false + if strings.Contains(commandTemplate, "{{.Command}}") { + usingCommand = true + commandTemplate = strings.ReplaceAll(commandTemplate, "{{.Command}}", "") + } + tmpl, err := template.New("command").Parse(commandTemplate) if err != nil { return fmt.Errorf("failed to parse template for command: %w", err) @@ -67,8 +69,12 @@ func (docker Docker) executeCommand(ctx context.Context, commandTemplate string) } cmdParts := strings.Fields(cmdBuffer.String()) - driverCmd := strings.Fields(docker.DriverCommand) + if usingCommand { + go docker.InspectContainer(ctx) + cmdParts = append(cmdParts, docker.Command...) + } + driverCmd := strings.Fields(docker.DriverCommand) var cmd *exec.Cmd if len(driverCmd) > 1 { cmdArgs := append(driverCmd[1:], cmdParts...) @@ -77,14 +83,16 @@ func (docker Docker) executeCommand(ctx context.Context, commandTemplate string) cmd = exec.CommandContext(ctx, driverCmd[0], cmdParts...) } - if docker.Stdin != nil { - cmd.Stdin = docker.Stdin - } - if docker.Stdout != nil { - cmd.Stdout = docker.Stdout - } - if docker.Stderr != nil { - cmd.Stderr = docker.Stderr + if enableIO { + if docker.Stdin != nil { + cmd.Stdin = docker.Stdin + } + if docker.Stdout != nil { + cmd.Stdout = docker.Stdout + } + if docker.Stderr != nil { + cmd.Stderr = docker.Stderr + } } return cmd.Run() @@ -121,6 +129,7 @@ func (docker *Docker) SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) // inspectContainer inspects the docker container for metadata. func (docker *Docker) InspectContainer(ctx context.Context) ContainerConfig { + fmt.Println("DEBUG: InspectContainer()") // Give the container time to start. time.Sleep(2 * time.Second) @@ -131,6 +140,7 @@ func (docker *Docker) InspectContainer(ctx context.Context) ContainerConfig { for i := 0; i < 5; i++ { select { case <-ctx.Done(): + fmt.Println("DEBUG: ctx.Done()") return ContainerConfig{} case <-ticker.C: cmd := exec.CommandContext(ctx, "docker", "inspect", docker.Name) diff --git a/worker/docker_test.go b/worker/docker_test.go index 90908bcea..4bb87ea86 100644 --- a/worker/docker_test.go +++ b/worker/docker_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "math/rand" + "github.com/ohsu-comp-bio/funnel/events" "github.com/ohsu-comp-bio/funnel/logger" ) @@ -13,10 +15,10 @@ import ( var defaultConfig = ContainerConfig{ Id: "123", Image: "alpine", - Name: "funnel-test", + Name: "funnel-test-" + RandomString(6), DriverCommand: "docker", - Command: "echo Hello, World!", - RunCommand: "run --name {{.Name}} {{.Image}} {{range .Command}} {{.}} {{end}}", + Command: []string{"sh", "-c", "echo Hello, World!"}, + RunCommand: "run --name {{.Name}} {{.Image}} {{.Command}}", PullCommand: "pull {{.Image}}", RemoveContainer: true, Event: events.NewExecutorWriter("123", 1, 1, &events.Logger{ @@ -38,7 +40,7 @@ func TestDockerExecuteCommand(t *testing.T) { docker := Docker{ ContainerConfig: defaultConfig, } - err := docker.executeCommand(context.Background(), "run --rm alpine echo Hello, World!") + err := docker.executeCommand(context.Background(), "run --rm alpine echo Hello, World!", true) if err != nil { t.Errorf("Expected no error, but got: %v", err) } @@ -52,7 +54,7 @@ func TestDockerStop(t *testing.T) { // Run the container command in a separate goroutine go func() { - err := docker.executeCommand(ctx, "run --rm alpine sleep 30") + err := docker.executeCommand(ctx, "run --rm alpine sleep 30", true) if err != nil && ctx.Err() == nil { t.Errorf("Expected no error, but got: %v", err) } @@ -125,3 +127,13 @@ func TestDockerSyncAPIVersion(t *testing.T) { t.Errorf("Expected no error, but got: %v", err) } } + +// RandomString generates a random string of length n +func RandomString(n int) string { + var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz0123456789") + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} diff --git a/worker/worker.go b/worker/worker.go index 5ceb3ea9a..dfdff7c96 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "time" "github.com/ohsu-comp-bio/funnel/config" @@ -163,7 +162,7 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { for i, d := range task.GetExecutors() { containerConfig := ContainerConfig{ Image: d.Image, - Command: strings.Join(d.Command, " "), + Command: d.Command, Env: d.Env, Volumes: mapper.Volumes, Workdir: d.Workdir, From 0660edcc322e6a957a852171b33754e7ac5b8c75 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jul 2024 18:25:38 -0700 Subject: [PATCH 46/67] Re-enable all tests for container engines --- .github/workflows/compliance-test.yaml | 2 -- .github/workflows/nextflow.yaml | 2 -- .github/workflows/s3-test.yaml | 2 -- 3 files changed, 6 deletions(-) diff --git a/.github/workflows/compliance-test.yaml b/.github/workflows/compliance-test.yaml index 69d9111a4..a5620b6a2 100644 --- a/.github/workflows/compliance-test.yaml +++ b/.github/workflows/compliance-test.yaml @@ -25,8 +25,6 @@ name: Compliance Test on: push: - branches: - - main jobs: build: diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index 0aef58ef4..dee0e843f 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -2,8 +2,6 @@ name: Nextflow Test on: push: - branches: - - main jobs: build: diff --git a/.github/workflows/s3-test.yaml b/.github/workflows/s3-test.yaml index a6b271814..f93094eb4 100644 --- a/.github/workflows/s3-test.yaml +++ b/.github/workflows/s3-test.yaml @@ -5,8 +5,6 @@ name: S3 Integration Test on: push: - branches: - - main jobs: s3Test-integration: From 9e71330196d13f27347011b145553181c2e941af Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jul 2024 18:37:33 -0700 Subject: [PATCH 47/67] Update test suite --- .github/workflows/compliance-test.yaml | 4 +++- .github/workflows/nextflow.yaml | 1 + .github/workflows/s3-test.yaml | 4 +++- .github/workflows/tests.yaml | 28 +++++++++++++------------- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/workflows/compliance-test.yaml b/.github/workflows/compliance-test.yaml index a5620b6a2..56b7156bd 100644 --- a/.github/workflows/compliance-test.yaml +++ b/.github/workflows/compliance-test.yaml @@ -25,6 +25,8 @@ name: Compliance Test on: push: + branches: + - main jobs: build: @@ -123,4 +125,4 @@ jobs: repository: ohsu-comp-bio/funnel-compliance event: start-report github_pat: ${{ secrets.ACTIONS_TOKEN }} - + diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index dee0e843f..eb3efc703 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -49,6 +49,7 @@ jobs: - name: Install nf-canary and GA4GH-TES plugin run: | + cd .. git clone https://github.com/seqeralabs/nf-canary cd nf-canary cat <> nextflow.config diff --git a/.github/workflows/s3-test.yaml b/.github/workflows/s3-test.yaml index f93094eb4..d764c457a 100644 --- a/.github/workflows/s3-test.yaml +++ b/.github/workflows/s3-test.yaml @@ -5,6 +5,8 @@ name: S3 Integration Test on: push: + branches: + - main jobs: s3Test-integration: @@ -39,7 +41,7 @@ jobs: Secret: "minioadmin" EOF - wget https://github.com/ohsu-comp-bio/funnel/releases/download/untagged-217841a99d14ccfe289c/funnel-darwin-arm64-0.11.0.tar.gz + /bin/bash -c "$(curl -fsSL https://github.com/ohsu-comp-bio/funnel/releases/download/0.11.0/install.sh)" tar -zxvf funnel-darwin-arm64-0.11.0.tar.gz chmod +x funnel ./funnel server run --config funnel.config.yml & diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 3f9e315d8..0d749da5d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -4,20 +4,20 @@ on: push: jobs: - lint: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/setup-go@v3 - with: - go-version: 1.21 - - uses: actions/checkout@v3 - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - version: latest - # TODO: Re-enable all linters - args: --timeout 3m --verbose -D unused -D errcheck -D staticcheck -D govet -D gosimple -D ineffassign -D typecheck + # Temporarily disabling linting + # lint: + # name: lint + # runs-on: ubuntu-latest + # steps: + # - uses: actions/setup-go@v3 + # with: + # go-version: 1.21 + # - uses: actions/checkout@v3 + # - name: golangci-lint + # uses: golangci/golangci-lint-action@v3 + # with: + # version: latest + # args: --timeout 3m --verbose build: runs-on: ubuntu-latest From bd1e79b9c2803b9cbb0e17820d3f93b99484c849 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jul 2024 18:43:12 -0700 Subject: [PATCH 48/67] Nextflow test debug --- .github/workflows/nextflow.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index eb3efc703..255ebd5b3 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -60,7 +60,9 @@ jobs: tes.endpoint = 'http://localhost:8000' EOF + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Run nf-canary tests run: | - cd nf-canary ../nextflow run main.nf From 59fce8b0001e01e5ce20d7389a1b0710afb9471b Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jul 2024 18:50:12 -0700 Subject: [PATCH 49/67] Nextflow test debug --- .github/workflows/nextflow.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index 255ebd5b3..be18f10cd 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -49,7 +49,6 @@ jobs: - name: Install nf-canary and GA4GH-TES plugin run: | - cd .. git clone https://github.com/seqeralabs/nf-canary cd nf-canary cat <> nextflow.config @@ -60,9 +59,6 @@ jobs: tes.endpoint = 'http://localhost:8000' EOF - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: Run nf-canary tests run: | ../nextflow run main.nf From d0f68247ace153e638059a4f63608cc46093b31b Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jul 2024 18:55:53 -0700 Subject: [PATCH 50/67] Nextflow test debug --- .github/workflows/nextflow.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index be18f10cd..b1f169092 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -59,6 +59,10 @@ jobs: tes.endpoint = 'http://localhost:8000' EOF + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Run nf-canary tests run: | - ../nextflow run main.nf + cd nextflow + nextflow run nf-canary/main.nf From 90b69f4a88e210e62f71ef7821b0f720f506f4ff Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jul 2024 19:02:29 -0700 Subject: [PATCH 51/67] Nextflow test debug --- .github/workflows/nextflow.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index b1f169092..fd30af166 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -42,7 +42,6 @@ jobs: - name: Install Nextflow run: | - cd .. git clone https://github.com/nextflow-io/nextflow cd nextflow make compile @@ -59,10 +58,6 @@ jobs: tes.endpoint = 'http://localhost:8000' EOF - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: Run nf-canary tests run: | - cd nextflow - nextflow run nf-canary/main.nf + ./nextflow/nextflow run nf-canary/main.nf From d9fb95fa20a66167bb6074c4a8ec0ea58b2c6516 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jul 2024 19:19:14 -0700 Subject: [PATCH 52/67] Update compliance test workflow --- .github/workflows/compliance-test.yaml | 44 ++------------------------ 1 file changed, 2 insertions(+), 42 deletions(-) diff --git a/.github/workflows/compliance-test.yaml b/.github/workflows/compliance-test.yaml index 56b7156bd..b1ade0a6d 100644 --- a/.github/workflows/compliance-test.yaml +++ b/.github/workflows/compliance-test.yaml @@ -25,13 +25,10 @@ name: Compliance Test on: push: - branches: - - main jobs: build: runs-on: ubuntu-latest - container: quay.io/ohsu-comp-bio/slurm steps: - name: Set up Go 1.x uses: actions/setup-go@v5 @@ -56,12 +53,9 @@ jobs: matrix: version: [1.0.0, 1.1.0] db: ["boltdb", "mongodb"] - compute: ["local", "slurm"] + compute: ["local"] needs: build runs-on: ubuntu-latest - container: - image: quay.io/ohsu-comp-bio/slurm - options: --hostname slurmctl --cap-add sys_admin steps: # Required to access the 'tests/mongo.config.yml' file # Perhaps uploading it as an artifact would be more efficient? @@ -78,17 +72,9 @@ jobs: if [ ${{ matrix.db }} = "mongodb" ]; then make start-mongodb cat `pwd`/tests/mongo.config.yml >> config.yml - # Required for Funnel to connect MongoDB - echo "172.17.0.1 localhost" >> /etc/hosts - elif [ ${{ matrix.compute }} = "slurm" ]; then - cat `pwd`/tests/slurm.config.yml >> config.yml - cp config.yml /opt/funnel_config.yml - # Start Slurm - /usr/local/bin/docker-entrypoint.sh fi chmod +x funnel - FLAGS="--config `pwd`/config.yml" - ./funnel server run $FLAGS &> funnel.logs & + ./funnel server run --config `pwd`/config.yml &> funnel.logs & - name: Run OpenAPI Test Runner run: | @@ -100,29 +86,3 @@ jobs: python setup.py install openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/" - - name: Install Legacy TES compliance suite (Report Generation) - run: | - git clone https://github.com/lbeckman314/tes-compliance-suite -b feature/tesv1.1 - cd tes-compliance-suite - python3 -m venv venv - source venv/bin/activate - pip install -r requirements.txt - python setup.py install - mkdir reports - tes-compliance-suite report --version "${{ matrix.version }}" --server "http://localhost:8000/" - - start-report-deployment: - needs: compliance - runs-on: ubuntu-latest - steps: - # https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#create-a-repository-dispatch-event - - name: Start report generation - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_AUTHOR: "Funnel" - PAYLOAD_REVISION: "3" - with: - repository: ohsu-comp-bio/funnel-compliance - event: start-report - github_pat: ${{ secrets.ACTIONS_TOKEN }} - From 60de2480c13dd367580553aec0c129955d95616c Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jul 2024 19:26:01 -0700 Subject: [PATCH 53/67] Update S3 tests --- .github/workflows/s3-test.yaml | 70 ++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/.github/workflows/s3-test.yaml b/.github/workflows/s3-test.yaml index d764c457a..c5ffae949 100644 --- a/.github/workflows/s3-test.yaml +++ b/.github/workflows/s3-test.yaml @@ -5,15 +5,32 @@ name: S3 Integration Test on: push: - branches: - - main jobs: - s3Test-integration: + build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: 1.21 + - name: Check out code + uses: actions/checkout@v2 + + - name: Build + run: make build + + - name: Store funnel + uses: actions/upload-artifact@v2 + with: + name: funnelBin + path: funnel + + s3Test: + + runs-on: ubuntu-latest + steps: - name: Setup minio run: | docker run -d -p 9000:9000 --name minio \ @@ -23,26 +40,31 @@ jobs: -v /tmp/config:/root/.minio \ minio/minio server /data - cat < funnel.config.yml - LocalStorage: - Disabled: true - AmazonS3: - Disabled: true - GoogleStorage: - Disabled: true - HTTPStorage: - Disabled: true - FTPStorage: - Disabled: true - GenericS3: - - Disabled: false - Endpoint: "localhost:9000" - Key: "minioadmin" - Secret: "minioadmin" - EOF - /bin/bash -c "$(curl -fsSL https://github.com/ohsu-comp-bio/funnel/releases/download/0.11.0/install.sh)" - tar -zxvf funnel-darwin-arm64-0.11.0.tar.gz + + - uses: actions/download-artifact@v3 + with: + name: funnelBin + + - name: Start Funnel server + run: | + cat < config.yml + LocalStorage: + Disabled: true + AmazonS3: + Disabled: true + GoogleStorage: + Disabled: true + HTTPStorage: + Disabled: true + FTPStorage: + Disabled: true + GenericS3: + - Disabled: false + Endpoint: "localhost:9000" + Key: "minioadmin" + Secret: "minioadmin" + EOF chmod +x funnel - ./funnel server run --config funnel.config.yml & + ./funnel server run --config `pwd`/config.yml &> funnel.logs & ./funnel task run examples/s3-test.yml From 582b8347609745a83ea4573334a8400ab7778cbd Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jul 2024 19:28:02 -0700 Subject: [PATCH 54/67] Update S3 tests --- .github/workflows/s3-test.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/s3-test.yaml b/.github/workflows/s3-test.yaml index c5ffae949..e84622a23 100644 --- a/.github/workflows/s3-test.yaml +++ b/.github/workflows/s3-test.yaml @@ -28,7 +28,7 @@ jobs: path: funnel s3Test: - + needs: build runs-on: ubuntu-latest steps: - name: Setup minio @@ -40,8 +40,6 @@ jobs: -v /tmp/config:/root/.minio \ minio/minio server /data - - - uses: actions/download-artifact@v3 with: name: funnelBin From f40523140fcc1358ae2039432b1dcc9e0d4bffb8 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 26 Jul 2024 16:06:02 -0700 Subject: [PATCH 55/67] Add comments and clean up debug statements --- storage/local.go | 3 ++- worker/docker.go | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/storage/local.go b/storage/local.go index 8ee1881c0..dae397277 100644 --- a/storage/local.go +++ b/storage/local.go @@ -176,7 +176,8 @@ func linkFile(ctx context.Context, source string, dest string) error { return fmt.Errorf("failed to get filepath using Glob: %v", err) } for _, glob := range globs { - // Correctly calculate the destination for each file + // Since tesOutput.path contains a wildcard, then the dest path must be a directory + // Ref: https://github.com/ga4gh/task-execution-schemas/blob/v1.1/openapi/task_execution_service.openapi.yaml#L528 // TODO: Verify that path_prefix is being removed from the dest filepath dest = filepath.Join(dest, filepath.Base(glob)) err := processItem(ctx, glob, dest) diff --git a/worker/docker.go b/worker/docker.go index 0e80285b5..527243f23 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -129,7 +129,6 @@ func (docker *Docker) SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) // inspectContainer inspects the docker container for metadata. func (docker *Docker) InspectContainer(ctx context.Context) ContainerConfig { - fmt.Println("DEBUG: InspectContainer()") // Give the container time to start. time.Sleep(2 * time.Second) @@ -140,7 +139,6 @@ func (docker *Docker) InspectContainer(ctx context.Context) ContainerConfig { for i := 0; i < 5; i++ { select { case <-ctx.Done(): - fmt.Println("DEBUG: ctx.Done()") return ContainerConfig{} case <-ticker.C: cmd := exec.CommandContext(ctx, "docker", "inspect", docker.Name) From d891fc3246541396e1bcef107b82335ad6d1b438 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 26 Jul 2024 16:22:31 -0700 Subject: [PATCH 56/67] Clean up code for 0.11.1-rc.1 release --- Makefile | 2 +- go.mod | 230 ++++++- go.sum | 579 +++++++++++++++++- .../{interop => integrations}/nextflow.md | 2 +- .../docs/{interop => integrations}/py-tes.md | 2 +- website/content/docs/interop/hpc.md | 117 ---- 6 files changed, 806 insertions(+), 126 deletions(-) rename website/content/docs/{interop => integrations}/nextflow.md (98%) rename website/content/docs/{interop => integrations}/py-tes.md (97%) delete mode 100644 website/content/docs/interop/hpc.md diff --git a/Makefile b/Makefile index 1923e8dca..317970114 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ git_upstream := $(shell git remote get-url $(shell git config branch.$(shell git export GIT_BRANCH = $(git_branch) export GIT_UPSTREAM = $(git_upstream) -export FUNNEL_VERSION=0.11.0 +export FUNNEL_VERSION=0.11.1-rc.1 # LAST_PR_NUMBER is used by the release notes builder to generate notes # based on pull requests (PR) up until the last release. diff --git a/go.mod b/go.mod index 715ad82bb..764b154cd 100644 --- a/go.mod +++ b/go.mod @@ -79,43 +79,190 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.8 // indirect + cloud.google.com/go/kms v1.17.1 // indirect + cloud.google.com/go/longrunning v0.5.7 // indirect + cloud.google.com/go/storage v1.41.0 // indirect + code.gitea.io/sdk/gitea v0.18.0 // indirect + dario.cat/mergo v1.0.0 // indirect + github.com/AlekSi/pointer v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.29 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect + github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect + github.com/BurntSushi/toml v1.2.1 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/alessio/shellescape v1.4.1 // indirect + github.com/anchore/bubbly v0.0.0-20230518153401-87b6af8ccf22 // indirect + github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a // indirect + github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb // indirect + github.com/anchore/quill v0.4.1 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/atc0005/go-teams-notify/v2 v2.10.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.13 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.13 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.28.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.23.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/kms v1.30.0 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 // indirect + github.com/aws/smithy-go v1.20.2 // indirect + github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240514230400-03fa26f5508f // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blacktop/go-dwarf v1.0.9 // indirect + github.com/blacktop/go-macho v1.1.162 // indirect + github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb // indirect + github.com/bluesky-social/indigo v0.0.0-20240411170459-440932307e0d // indirect + github.com/buchanae/github-release-notes v0.0.0-20180827045457-200e1dacadbb // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/caarlos0/ctrlc v1.2.0 // indirect + github.com/caarlos0/env/v11 v11.0.1 // indirect + github.com/caarlos0/go-reddit/v3 v3.0.1 // indirect + github.com/caarlos0/go-shellwords v1.0.12 // indirect + github.com/caarlos0/go-version v0.1.1 // indirect + github.com/caarlos0/log v0.4.4 // indirect + github.com/carlmjohnson/versioninfo v0.22.5 // indirect + github.com/cavaliergopher/cpio v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/bubbletea v0.22.1 // indirect + github.com/charmbracelet/lipgloss v0.10.0 // indirect + github.com/charmbracelet/x/exp/ordered v0.0.0-20231010190216-1cb11efc897d // indirect + github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect + github.com/cloudflare/circl v1.3.8 // indirect + github.com/containerd/console v1.0.4 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/dghubble/go-twitter v0.0.0-20211115160449-93a8679adecb // indirect + github.com/dghubble/oauth1 v0.7.3 // indirect + github.com/dghubble/sling v1.4.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect + github.com/docker/cli v25.0.4+incompatible // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker-credential-helpers v0.8.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/elliotchance/orderedmap/v2 v2.2.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gammazero/deque v0.2.1 // indirect + github.com/github/smimesign v0.2.0 // indirect + github.com/go-fed/httpsig v1.1.0 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-git/go-git/v5 v5.12.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/runtime v0.28.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/validate v0.24.0 // indirect + github.com/go-restruct/restruct v1.2.0-alpha // indirect + github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/glog v1.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-containerregistry v0.19.1 // indirect + github.com/google/go-github v17.0.0+incompatible // indirect + github.com/google/go-github/v62 v62.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/ko v0.15.4 // indirect + github.com/google/rpmpack v0.6.1-0.20240329070804-c2247cbb881a // indirect github.com/google/s2a-go v0.1.7 // indirect + github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/google/wire v0.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.5 // indirect + github.com/goreleaser/chglog v0.6.1 // indirect + github.com/goreleaser/fileglob v1.3.0 // indirect + github.com/goreleaser/goreleaser v1.26.2 // indirect + github.com/goreleaser/nfpm/v2 v2.37.1 // indirect + github.com/gorilla/websocket v1.5.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/hcl v1.0.1-vault-5 // indirect + github.com/huandu/xstrings v1.3.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.12.0 // indirect github.com/invopop/yaml v0.3.1 // indirect + github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/go-block-format v0.2.0 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect + github.com/ipfs/go-datastore v0.6.0 // indirect + github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect + github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect + github.com/ipfs/go-ipfs-util v0.0.3 // indirect + github.com/ipfs/go-ipld-cbor v0.1.0 // indirect + github.com/ipfs/go-ipld-format v0.6.0 // indirect + github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipfs/go-log/v2 v2.5.1 // indirect + github.com/ipfs/go-metrics-interface v0.0.1 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jbenet/goprocess v0.1.4 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -124,41 +271,103 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/letsencrypt/boulder v0.0.0-20231026200631-000cd05d5491 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/maruel/panicparse v1.6.2 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-mastodon v0.0.8 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/montanaflynn/stats v0.7.1 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/mango v0.1.0 // indirect + github.com/muesli/mango-cobra v1.2.0 // indirect + github.com/muesli/mango-pflag v0.1.0 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/roff v0.1.0 // indirect + github.com/muesli/termenv v0.15.2 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nsf/termbox-go v1.1.1 // indirect + github.com/oklog/ulid v1.3.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sergi/go-diff v1.2.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect + github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/shopspring/decimal v1.2.0 // indirect + github.com/sigstore/cosign/v2 v2.2.4 // indirect + github.com/sigstore/rekor v1.3.6 // indirect + github.com/sigstore/sigstore v1.8.3 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect + github.com/slack-go/slack v0.13.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/viper v1.18.2 // indirect github.com/stretchr/objx v0.5.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect + github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect + github.com/vbatts/tar-split v0.11.5 // indirect + github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 // indirect + github.com/wagoodman/go-progress v0.0.0-20220614130704-4b1c25a33c7c // indirect + github.com/whyrusleeping/cbor-gen v0.1.1-0.20240311221002-68b9f235c302 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/xanzy/go-gitlab v0.105.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect @@ -167,19 +376,36 @@ require ( go.opentelemetry.io/otel/metric v1.27.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.opentelemetry.io/otel/trace v1.27.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/automaxprocs v1.5.3 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + gocloud.dev v0.37.0 // indirect + golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect + golang.org/x/mod v0.17.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/mail.v2 v2.3.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.4.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + lukechampine.com/blake3 v1.2.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kind v0.23.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect + software.sslmate.com/src/go-pkcs12 v0.4.0 // indirect ) diff --git a/go.sum b/go.sum index b701ac2c0..223153170 100644 --- a/go.sum +++ b/go.sum @@ -18,59 +18,242 @@ cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuA cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= cloud.google.com/go/pubsub v1.40.0 h1:0LdP+zj5XaPAGtWr2V6r88VXJlmtaB/+fde1q3TU8M0= cloud.google.com/go/pubsub v1.40.0/go.mod h1:BVJI4sI2FyXp36KFKvFwcfDRDfR8MiLT8mMhmIhdAeA= +cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0= +cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80= +code.gitea.io/sdk/gitea v0.18.0 h1:+zZrwVmujIrgobt6wVBWCqITz6bn1aBjnCUHmpZrerI= +code.gitea.io/sdk/gitea v0.18.0/go.mod h1:IG9xZJoltDNeDSW0qiF2Vqx5orMWa7OhVWrjvrd5NpI= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w= +github.com/AlekSi/pointer v1.2.0/go.mod h1:gZGfd3dpW4vEc/UlyfKKi1roIqcCgwOIvb0tSNSBle0= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 h1:n1DH8TPV4qqPTje2RcUBYwtrTWlabVp4n46+74X2pn4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0/go.mod h1:HDcZnuGbiyppErN6lB+idp4CKhjbc8gwjto6OPpyggM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0/go.mod h1:Pu5Zksi2KrU7LPbZbNINx6fuVrUp/ffvpxdDj+i8LeE= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= +github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= +github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/anchore/bubbly v0.0.0-20230518153401-87b6af8ccf22 h1:5NFK6VGgqBUOAX2SYyzFYvNdOiYDxzim8jga386FlZY= +github.com/anchore/bubbly v0.0.0-20230518153401-87b6af8ccf22/go.mod h1:Kv+Mm9CdtnV8iem48iEPIwy7/N4Wmk0hpxYNH5gTwKQ= +github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a h1:nJ2G8zWKASyVClGVgG7sfM5mwoZlZ2zYpIzN2OhjWkw= +github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a/go.mod h1:ubLFmlsv8/DFUQrZwY5syT5/8Er3ugSr4rDFwHsE3hg= +github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb h1:iDMnx6LIjtjZ46C0akqveX83WFzhpTD3eqOthawb5vU= +github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb/go.mod h1:DmTY2Mfcv38hsHbG78xMiTDdxFtkHpgYNVDPsF2TgHk= +github.com/anchore/quill v0.4.1 h1:mffDnvnER3ZgPjN5hexc3nr/4Y1dtKdDB6td5K8uInk= +github.com/anchore/quill v0.4.1/go.mod h1:t6hOPYDohN8wn2SRWQdNkJBkhmK8s3gzuHzzgcEvzQU= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/atc0005/go-teams-notify/v2 v2.10.0 h1:eQvRIkyESQgBvlUdQ/iPol/lj3QcRyrdEQM3+c/nXhM= +github.com/atc0005/go-teams-notify/v2 v2.10.0/go.mod h1:SIeE1UfCcVRYMqP5b+r1ZteHyA/2UAjzWF5COnZ8q0w= github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/aws/aws-sdk-go v1.55.1 h1:ZTNPmbRMxaK5RlTJrBullX9r/rF1MPf3yAJOLlwDiT8= github.com/aws/aws-sdk-go v1.55.1/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= +github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo= +github.com/aws/aws-sdk-go-v2/config v1.27.13 h1:WbKW8hOzrWoOA/+35S5okqO/2Ap8hkkFUzoW8Hzq24A= +github.com/aws/aws-sdk-go-v2/config v1.27.13/go.mod h1:XLiyiTMnguytjRER7u5RIkhIqS8Nyz41SwAWb4xEjxs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.13 h1:XDCJDzk/u5cN7Aple7D/MiAhx1Rjo/0nueJ0La8mRuE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.13/go.mod h1:FMNcjQrmuBYvOTZDtOLCIu0esmxjF7RuA/89iSXWzQI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9 h1:vXY/Hq1XdxHBIYgBUmug/AbMyIe1AKulPYS2/VE1X70= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9/go.mod h1:GyJJTZoHVuENM4TeJEl5Ffs4W9m19u+4wKJcDi/GZ4A= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 h1:mDnFOE2sVkyphMWtTH+stv0eW3k0OTx94K63xpxHty4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3/go.mod h1:V8MuRVcCRt5h1S+Fwu8KbC7l/gBGo3yBAyUbJM2IJOk= +github.com/aws/aws-sdk-go-v2/service/ecr v1.28.0 h1:rdPrcOZmqT2F+yzmKEImrx5XUs7Hpf4V9Rp6E8mhsxQ= +github.com/aws/aws-sdk-go-v2/service/ecr v1.28.0/go.mod h1:if7ybzzjOmDB8pat9FE35AHTY6ZxlYSy3YviSmFZv8c= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.23.5 h1:452e/nFuqPvwPg+1OD2CG/v29R9MH8egJSJKh2Qduv8= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.23.5/go.mod h1:8pvvNAklmq+hKmqyvFoMRg0bwg9sdGOvdwximmKiKP0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 h1:mbWNpfRUTT6bnacmvOTKXZjR/HycibdWzNpfbrbLDIs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5/go.mod h1:FCOPWGjsshkkICJIn9hq9xr6dLKtyaWpuUojiN3W1/8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 h1:4t+QEX7BsXz98W8W1lNvMAG+NX8qHz2CjLBxQKku40g= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3/go.mod h1:oFcjjUq5Hm09N9rpxTdeMeLeQcxS7mIkBkL8qUKng+A= +github.com/aws/aws-sdk-go-v2/service/kms v1.30.0 h1:yS0JkEdV6h9JOo8sy2JSpjX+i7vsKifU8SIeHrqiDhU= +github.com/aws/aws-sdk-go-v2/service/kms v1.30.0/go.mod h1:+I8VUUSVD4p5ISQtzpgSva4I8cJ4SQ4b1dcBcof7O+g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 h1:lW5xUzOPGAMY7HPuNF4FdyBwRc3UJ/e8KsapbesVeNU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4/go.mod h1:MGTaf3x/+z7ZGugCGvepnx2DS6+caCYYqKhzVoLNYPk= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 h1:o5cTaeunSpfXiLTIBx5xo2enQmiChtu1IBbzXnfU9Hs= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.6/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.0 h1:Qe0r0lVURDDeBQJ4yP+BOrJkvkiCo/3FH/t+wY11dmw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.0/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 h1:et3Ta53gotFR4ERLXXHIHl/Uuk1qYpP5uU7cvNql8ns= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.7/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240514230400-03fa26f5508f h1:Z0kS9pJDQgCg3u2lH6+CdYaFbyQtyukVTiUCG6re0E4= +github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240514230400-03fa26f5508f/go.mod h1:rAE739ssmE5O5fLuQ2y8uHdmOJaelE5I0Es3SxV0y1A= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blacktop/go-dwarf v1.0.9 h1:eT/L7gt0gllvvgnRXY0MFKjNB6+jtOY5DTm2ynVX2dY= +github.com/blacktop/go-dwarf v1.0.9/go.mod h1:4W2FKgSFYcZLDwnR7k+apv5i3nrau4NGl9N6VQ9DSTo= +github.com/blacktop/go-macho v1.1.162 h1:FjM3XAsJTAOGZ1eppRSX9ZBX3Bk11JMTC1amsZAOA5I= +github.com/blacktop/go-macho v1.1.162/go.mod h1:f2X4noFBob4G5bWUrzvPBKDVcFWZgDCM7rIn7ygTID0= +github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= +github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= +github.com/bluesky-social/indigo v0.0.0-20240411170459-440932307e0d h1:xxPhzCOpmOntzVe8S6tqsMdFgaB8B4NXSV54lG4B1qk= +github.com/bluesky-social/indigo v0.0.0-20240411170459-440932307e0d/go.mod h1:ysMQ0a4RYWjgyvKrl5ME352oHA6QgK900g5sB9XXgPE= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= +github.com/buchanae/github-release-notes v0.0.0-20180827045457-200e1dacadbb h1:1JIKG3zt7fIMG3Hr1sZ3LcQxEROJYJ2qwpjdvD3FCw4= +github.com/buchanae/github-release-notes v0.0.0-20180827045457-200e1dacadbb/go.mod h1:YlY7IAd5TVq6+Bv/iDwCjc122k6aynvrHRJ2E+E82pg= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/caarlos0/ctrlc v1.2.0 h1:AtbThhmbeYx1WW3WXdWrd94EHKi+0NPRGS4/4pzrjwk= +github.com/caarlos0/ctrlc v1.2.0/go.mod h1:n3gDlSjsXZ7rbD9/RprIR040b7oaLfNStikPd4gFago= +github.com/caarlos0/env/v11 v11.0.1 h1:A8dDt9Ub9ybqRSUF3fQc/TA/gTam2bKT4Pit+cwrsPs= +github.com/caarlos0/env/v11 v11.0.1/go.mod h1:2RC3HQu8BQqtEK3V4iHPxj0jOdWdbPpWJ6pOueeU1xM= +github.com/caarlos0/go-reddit/v3 v3.0.1 h1:w8ugvsrHhaE/m4ez0BO/sTBOBWI9WZTjG7VTecHnql4= +github.com/caarlos0/go-reddit/v3 v3.0.1/go.mod h1:QlwgmG5SAqxMeQvg/A2dD1x9cIZCO56BMnMdjXLoisI= +github.com/caarlos0/go-shellwords v1.0.12 h1:HWrUnu6lGbWfrDcFiHcZiwOLzHWjjrPVehULaTFgPp8= +github.com/caarlos0/go-shellwords v1.0.12/go.mod h1:bYeeX1GrTLPl5cAMYEzdm272qdsQAZiaHgeF0KTk1Gw= +github.com/caarlos0/go-version v0.1.1 h1:1bikKHkGGVIIxqCmufhSSs3hpBScgHGacrvsi8FuIfc= +github.com/caarlos0/go-version v0.1.1/go.mod h1:Ze5Qx4TsBBi5FyrSKVg1Ibc44KGV/llAaKGp86oTwZ0= +github.com/caarlos0/log v0.4.4 h1:LnvgBz/ofsJ00AupP/cEfksJSZglb1L69g4Obk/sdAc= +github.com/caarlos0/log v0.4.4/go.mod h1:+AmCI9Liv5LKXmzFmFI1htuHdTTj/0R3KuoP9DMY7Mo= +github.com/caarlos0/testfs v0.4.4/go.mod h1:bRN55zgG4XCUVVHZCeU+/Tz1Q6AxEJOEJTliBy+1DMk= +github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc= +github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8= +github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM= +github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbletea v0.22.1 h1:z66q0LWdJNOWEH9zadiAIXp2GN1AWrwNXU8obVY9X24= +github.com/charmbracelet/bubbletea v0.22.1/go.mod h1:8/7hVvbPN6ZZPkczLiB8YpLkLJ0n7DMho5Wvfd2X1C0= +github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s= +github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE= +github.com/charmbracelet/x/exp/ordered v0.0.0-20231010190216-1cb11efc897d h1:+o+e/8hf7cG0SbAzEAm/usJ8qoZPgFXhudLjop+TM0g= +github.com/charmbracelet/x/exp/ordered v0.0.0-20231010190216-1cb11efc897d/go.mod h1:aoG4bThKYIOnyB55r202eHqo6TkN7ZXV+cu4Do3eoBQ= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI= +github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= +github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= +github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/dghubble/go-twitter v0.0.0-20211115160449-93a8679adecb h1:7ENzkH+O3juL+yj2undESLTaAeRllHwCs/b8z6aWSfc= +github.com/dghubble/go-twitter v0.0.0-20211115160449-93a8679adecb/go.mod h1:qhZBgV9e4WyB1JNjHpcXVkUe3knWUwYuAPB1hITdm50= +github.com/dghubble/oauth1 v0.7.3 h1:EkEM/zMDMp3zOsX2DC/ZQ2vnEX3ELK0/l9kb+vs4ptE= +github.com/dghubble/oauth1 v0.7.3/go.mod h1:oxTe+az9NSMIucDPDCCtzJGsPhciJV33xocHfcR2sVY= +github.com/dghubble/sling v1.4.0 h1:/n8MRosVTthvMbwlNZgLx579OGVjUOy3GNEv5BIqAWY= +github.com/dghubble/sling v1.4.0/go.mod h1:0r40aNsU9EdDUVBNhfCstAtFgutjgJGYbO1oNzkMoM8= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= @@ -79,10 +262,18 @@ github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkz github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v25.0.4+incompatible h1:DatRkJ+nrFoYL2HZUzjM5Z5sAmcA5XGp+AW0oEw2+cA= +github.com/docker/cli v25.0.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqYyMHOp5vfsD1807oKo= +github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -98,20 +289,31 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk= +github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= @@ -123,8 +325,18 @@ github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a h1:yU/FENpkHYI github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a/go.mod h1:AEugkNu3BjBxyz958nJ5holD9PRjta6iprcoUauDbU4= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/github/smimesign v0.2.0 h1:Hho4YcX5N1I9XNqhq0fNx0Sts8MhLonHd+HRXVGNjvk= +github.com/github/smimesign v0.2.0/go.mod h1:iZiiwNT4HbtGRVqCQu7uJPEZCuEE5sfSSttcnePkDl4= github.com/gizak/termui v2.3.0+incompatible h1:S8wJoNumYfc/rR5UezUM4HsPEo3RJh0LKdiuDWQpjqw= github.com/gizak/termui v2.3.0+incompatible/go.mod h1:PkJoWUt/zacQKysNfQtcw1RW+eK2SxkieVBtl+4ovLA= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -137,21 +349,49 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= +github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ= +github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= +github.com/go-restruct/restruct v1.2.0-alpha h1:2Lp474S/9660+SJjpVxoKuWX09JsXHSrdV7Nv3/gkvc= +github.com/go-restruct/restruct v1.2.0-alpha/go.mod h1:KqrpKpn4M8OLznErihXTGLlsXFGeLxHUrLRRI/1YjGk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 h1:wG8n/XJQ07TmjbITcGiUaOtXxdrINDz1b0J1w0SzqDc= +github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8= +github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg= github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f/go.mod h1:ijRvpgDJDI262hYq/IQVYgf8hd8IHUs93Ol0kvMBAx4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -180,8 +420,8 @@ github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= +github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -189,28 +429,62 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY= +github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= +github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/ko v0.15.4 h1:0blRbIdPmSy6v4LvedGxbI/8krdJYQgbSih3v6Y8V1c= +github.com/google/ko v0.15.4/go.mod h1:ZkcmfV91Xt6ZzOBHc/cXXGYnqWdNWDVy/gHoUU9sjag= github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/rpmpack v0.6.1-0.20240329070804-c2247cbb881a h1:JJBdjSfqSy3mnDT0940ASQFghwcZ4y4cb6ttjAoXqwE= +github.com/google/rpmpack v0.6.1-0.20240329070804-c2247cbb881a/go.mod h1:uqVAUVQLq8UY2hCDfmJ/+rtO3aw7qyhc90rCVEabEfI= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 h1:SJ+NtwL6QaZ21U+IrK7d0gGgpjGGvd2kz+FzTHVzdqI= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC9t8wNnpPdctvtSUOPUUg4SHeE6vR1Ir2hmg= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= +github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= +github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/goreleaser/chglog v0.6.1 h1:NZKiX8l0FTQPRzBgKST7knvNZmZ04f7PEGkN2wInfhE= +github.com/goreleaser/chglog v0.6.1/go.mod h1:Bnnfo07jMZkaAb0uRNASMZyOsX6ROW6X1qbXqN3guUo= +github.com/goreleaser/fileglob v1.3.0 h1:/X6J7U8lbDpQtBvGcwwPS6OpzkNVlVEsFUVRx9+k+7I= +github.com/goreleaser/fileglob v1.3.0/go.mod h1:Jx6BoXv3mbYkEzwm9THo7xbr5egkAraxkGorbJb4RxU= +github.com/goreleaser/goreleaser v1.26.2 h1:1iY1HaXtRiMTrwy6KE1sNjkRjsjMi+9l0k6WUX8GpWw= +github.com/goreleaser/goreleaser v1.26.2/go.mod h1:mHi6zr6fuuOh5eHdWWgyo/N8BWED5WEVtb/4GETc9jQ= +github.com/goreleaser/nfpm/v2 v2.37.1 h1:RUmeEt8OlEVeSzKRrO5Vl5qVWCtUwx4j9uivGuRo5fw= +github.com/goreleaser/nfpm/v2 v2.37.1/go.mod h1:q8+sZXFqn106/eGw+9V+I8+izFxZ/sJjrhwmEUxXhUg= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= @@ -219,21 +493,68 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= +github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= +github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= +github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= +github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= +github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= +github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= +github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= +github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= +github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= +github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= +github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= +github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0= +github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs= +github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs= +github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk= +github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U= +github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg= +github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= +github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= +github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= +github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= +github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -246,6 +567,7 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -257,13 +579,20 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -273,32 +602,61 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/letsencrypt/boulder v0.0.0-20231026200631-000cd05d5491 h1:WGrKdjHtWC67RX96eTkYD2f53NDHhrq/7robWTAfk4s= +github.com/letsencrypt/boulder v0.0.0-20231026200631-000cd05d5491/go.mod h1:o158RFmdEbYyIZmXAbrvmJWesbyxlLKee6X64VPVuOc= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/panicparse v1.6.2 h1:tZuGQTlbOY5jCprrWMJTikREqKPn+UAKdR4CHSpj834= github.com/maruel/panicparse v1.6.2/go.mod h1:uoxI4w9gJL6XahaYPMq/z9uadrdr1SyHuQwV2q80Mm0= github.com/maruel/panicparse/v2 v2.1.1/go.mod h1:AeTWdCE4lcq8OKsLb6cHSj1RWHVSnV9HBCk7sKLF4Jg= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.0.10-0.20170816031813-ad5389df28cd/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.2/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-mastodon v0.0.8 h1:UgKs4SmQ5JeawxMIPP7NQ9xncmOXA+5q6jYk4erR7vk= +github.com/mattn/go-mastodon v0.0.8/go.mod h1:8YkqetHoAVEktRkK15qeiv/aaIMfJ/Gc89etisPZtHU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -314,6 +672,36 @@ github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA= +github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/mango v0.1.0 h1:DZQK45d2gGbql1arsYA4vfg4d7I9Hfx5rX/GCmzsAvI= +github.com/muesli/mango v0.1.0/go.mod h1:5XFpbC8jY5UUv89YQciiXNlbi+iJgt29VDC5xbzrLL4= +github.com/muesli/mango-cobra v1.2.0 h1:DQvjzAM0PMZr85Iv9LIMaYISpTOliMEg+uMFtNbYvWg= +github.com/muesli/mango-cobra v1.2.0/go.mod h1:vMJL54QytZAJhCT13LPVDfkvCUJ5/4jNUKF/8NC2UjA= +github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe7Sg= +github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= +github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= +github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739/go.mod h1:Bd5NYQ7pd+SrtBSrSNoBBmXlcY8+Xj4BMJgh8qcZrvs= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= @@ -322,6 +710,8 @@ github.com/nsf/termbox-go v1.1.1 h1:nksUPLCb73Q++DwbYUBEglYBRPZyoXJdrj5L+TkjyZY= github.com/nsf/termbox-go v1.1.1/go.mod h1:T0cTdVuOwf7pHQNtfhnEbzHbcNyCEcVU4YPpouCbVxo= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olivere/elastic/v7 v7.0.12/go.mod h1:14rWX28Pnh3qCKYRVnSGXWLf9MbLonYS/4FDCY3LAPo= github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= @@ -332,12 +722,23 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pborman/getopt v0.0.0-20180811024354-2b5b3bfb099b/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -345,6 +746,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f h1:VXTQfuJj9vKR4TCkEuWIckKvdHFeJH/huIFJ9/cXOB0= +github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -356,34 +759,69 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e h1:7q6NSFZDeGfvvtIRwBrU/aegEYJYmvev0cHAwo17zZQ= +github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs= +github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbmfHkLguCE9laoZCUzEEpIZXA= +github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sigstore/cosign/v2 v2.2.4 h1:iY4vtEacmu2hkNj1Fh+8EBqBwKs2DHM27/lbNWDFJro= +github.com/sigstore/cosign/v2 v2.2.4/go.mod h1:JZlRD2uaEjVAvZ1XJ3QkkZJhTqSDVtLaet+C/TMR81Y= +github.com/sigstore/rekor v1.3.6 h1:QvpMMJVWAp69a3CHzdrLelqEqpTM3ByQRt5B5Kspbi8= +github.com/sigstore/rekor v1.3.6/go.mod h1:JDTSNNMdQ/PxdsS49DJkJ+pRJCO/83nbR5p3aZQteXc= +github.com/sigstore/sigstore v1.8.3 h1:G7LVXqL+ekgYtYdksBks9B38dPoIsbscjQJX/MGWkA4= +github.com/sigstore/sigstore v1.8.3/go.mod h1:mqbTEariiGA94cn6G3xnDiV6BD8eSLdL/eA7bvJ0fVs= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/slack-go/slack v0.13.0 h1:7my/pR2ubZJ9912p9FtvALYpbt0cQPAqkRy2jaSI1PQ= +github.com/slack-go/slack v0.13.0/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -395,6 +833,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -404,21 +844,48 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= +github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= +github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= +github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 h1:jIVmlAFIqV3d+DOxazTR9v+zgj8+VYuQBzPgBZvWBHA= +github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651/go.mod h1:b26F2tHLqaoRQf8DywqzVaV1MQ9yvjb0OMcNl7Nxu20= +github.com/wagoodman/go-progress v0.0.0-20220614130704-4b1c25a33c7c h1:gFwUKtkv6QzQsFdIjvPqd0Qdw42DHUEbbUdiUTI1uco= +github.com/wagoodman/go-progress v0.0.0-20220614130704-4b1c25a33c7c/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/cbor-gen v0.1.1-0.20240311221002-68b9f235c302 h1:MhInbXe4SzcImAKktUvWBCWZgcw6MYf5NfumTj1BhAw= +github.com/whyrusleeping/cbor-gen v0.1.1-0.20240311221002-68b9f235c302/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xanzy/go-gitlab v0.105.0 h1:3nyLq0ESez0crcaM19o5S//SvezOQguuIHZ3wgX64hM= +github.com/xanzy/go-gitlab v0.105.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -430,9 +897,12 @@ github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 h1:tBiBTKHnIjovYoLX/ github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76/go.mod h1:SQliXeA7Dhkt//vS29v3zpbEwoa+zb2Cn5xj5uO4K5U= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +gitlab.com/digitalxero/go-conventional-commit v1.0.7 h1:8/dO6WWG+98PMhlZowt/YjuiKhqhGlOCwlIV8SqqGh8= +gitlab.com/digitalxero/go-conventional-commit v1.0.7/go.mod h1:05Xc2BFsSyC5tKhK0y+P3bs0AwUtNuTp+mTpbCU/DZ0= go.einride.tech/aip v0.67.1 h1:d/4TW92OxXBngkSOwWS2CH5rez869KpKMaN44mdxkFI= go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI= go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= @@ -458,28 +928,64 @@ go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5 go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +gocloud.dev v0.37.0 h1:XF1rN6R0qZI/9DYjN16Uy0durAmSlf58DHOcb28GPro= +gocloud.dev v0.37.0/go.mod h1:7/O4kqdInCNsc6LqgmuFnS0GRew4XNNYWpA44yQnwco= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= +golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -493,9 +999,16 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -509,7 +1022,11 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -520,34 +1037,59 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -557,19 +1099,29 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.0.0-20170921000349-586095a6e407/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.186.0 h1:n2OPp+PPXX0Axh4GuSsL5QL8xQCTb2oDwyzPnQvqUug= google.golang.org/api v0.186.0/go.mod h1:hvRbBmgoje49RV3xqVXrmP6w93n6ehGgIVPYrGtBFFc= @@ -609,18 +1161,30 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= +gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk= +gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw= gopkg.in/olivere/elastic.v5 v5.0.86 h1:xFy6qRCGAmo5Wjx96srho9BitLhZl2fcnpuidPwduXM= gopkg.in/olivere/elastic.v5 v5.0.86/go.mod h1:M3WNlsF+WhYn7api4D87NIflwTV/c0iVs8cqfWhK+68= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -631,6 +1195,7 @@ gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= k8s.io/api v0.31.0-beta.0 h1:4qPGblT8h5w81f+dbj8gbYTnZATmHDRapPcmSfqKJec= k8s.io/api v0.31.0-beta.0/go.mod h1:0Zff1rZco/hYJZmBdGxE+Q0KBGRxIwPAFK6fHYxjvk8= k8s.io/apimachinery v0.31.0-beta.0 h1:KoBE9f7sPz67HclZC/JgH1pIBvOlMZQAwoHQuriE/5E= @@ -643,9 +1208,15 @@ k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f h1:0LQagt0gDpKqvIkAMPaRGc k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f/go.mod h1:S9tOR0FxgyusSNR+MboCuiDpVWkAifZvaYI1Q2ubgro= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kind v0.23.0 h1:8fyDGWbWTeCcCTwA04v4Nfr45KKxbSPH1WO9K+jVrBg= +sigs.k8s.io/kind v0.23.0/go.mod h1:ZQ1iZuJLh3T+O8fzhdi3VWcFTzsdXtNv2ppsHc8JQ7s= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= +software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/website/content/docs/interop/nextflow.md b/website/content/docs/integrations/nextflow.md similarity index 98% rename from website/content/docs/interop/nextflow.md rename to website/content/docs/integrations/nextflow.md index cf9a6687c..37cac2b23 100644 --- a/website/content/docs/interop/nextflow.md +++ b/website/content/docs/integrations/nextflow.md @@ -2,7 +2,7 @@ title: Nextflow menu: main: - parent: Interop + parent: Integrations --- > ⚠️ Nextflow support is currently in development and requires a few additional steps to run which are included below. diff --git a/website/content/docs/interop/py-tes.md b/website/content/docs/integrations/py-tes.md similarity index 97% rename from website/content/docs/interop/py-tes.md rename to website/content/docs/integrations/py-tes.md index e07d2bb9d..385255729 100644 --- a/website/content/docs/interop/py-tes.md +++ b/website/content/docs/integrations/py-tes.md @@ -2,7 +2,7 @@ title: py-tes menu: main: - parent: Interop + parent: Integrations --- > ⚠️ py-tes support is in active development and may be subject to change. diff --git a/website/content/docs/interop/hpc.md b/website/content/docs/interop/hpc.md deleted file mode 100644 index 5f19d7827..000000000 --- a/website/content/docs/interop/hpc.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -title: HPC -menu: - main: - parent: Interop ---- - -> ⚠️ HPC support is in active development and commands may change. - -# HPC - -## Getting Started - -Funnel supports submitting tasks to HPC (High-Performance Computing) backends via job schedulers like Slurm. - -To get started with using HPC with Funnel, you will need to configure Funnel to use a suitable container engine and job scheduler. This guide will walk you through setting up the necessary configurations and running tasks on an HPC system using Slurm. - -## Example - -Below is an example configuration for setting up Funnel to run tasks on an HPC system using Slurm as the job scheduler and exadocker as the container engine. This configuration includes settings for local storage, worker behavior, server details, and Slurm job submission templates. - -
- config.yml - -```yaml -LocalStorage: - # Whitelist of local directory paths which Funnel is allowed to access. - AllowedDirs: - - ./ - - /home/users/example - -Worker: - LeaveWorkDir: true - ContainerDriver: sudo /opt/acc/sbin/exadocker - -Server: - HostName: exahead1 - -Compute: slurm - -# Custom template for Slurm -Slurm: - Template: | - #!/bin/bash - #SBATCH --job-name {{.TaskId}} - #SBATCH --ntasks 1 - #SBATCH --error {{.WorkDir}}/funnel-stderr - #SBATCH --output {{.WorkDir}}/funnel-stdout - #SBATCH --gres disk:1 - {{if ne .Cpus 0 -}} - {{printf "#SBATCH --cpus-per-task %d" .Cpus}} - {{- end}} - {{if ne .RamGb 0.0 -}} - {{printf "#SBATCH --mem %.0fGB" .RamGb}} - {{- end}} - {{if ne .DiskGb 0.0 -}} - {{printf "#SBATCH --tmp %.0fGB" .DiskGb}} - {{- end}} - - srun /usr/local/bin/mkdir-scratch.sh > /dev/null - SCRATCH_PATH="/mnt/scratch/${SLURM_JOB_ID}" - - cd $SCRATCH_PATH - srun bash -c 'funnel worker run --taskID {{.TaskId}} {{.Args}} --Worker.ScratchPath .' --temp-dir $SCRATCH_PATH - - cd - > /dev/null - srun /usr/local/bin/rmdir-scratch.sh > /dev/null -``` -
- -
- md5sum.json - -```json -{ - "name": "md5sum example", - "description": "Demonstrates input and output files using a simple md5sum command.", - "inputs": [ - { - "name": "md5sum input", - "description": "Input to md5sum. /tmp/md5sum_input must exist on the host system.", - "url": "file:///tmp/md5sum_input", - "type": "FILE", - "path": "/tmp/in" - } - ], - "outputs": [ - { - "name": "md5sum stdout", - "description": "Stdout of md5sum is captured and copied to /tmp/md5_output on the host system.", - "url": "file:///tmp/md5sum_output", - "type": "FILE", - "path": "/tmp/out" - } - ], - "executors": [ - { - "image": "alpine", - "command": ["md5sum", "/tmp/in"], - "stdout": "/tmp/out" - } - ] -} -``` -
- -## Start Funnel Server - -```sh -funnel server run --config config.yml -``` - -## Submit Task - -```sh -funnel task create md5sum.json -``` \ No newline at end of file From 0bdd6043952789915f2ccb2e90c2f79bad19febb Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 26 Jul 2024 18:57:34 -0700 Subject: [PATCH 57/67] Add container command event to match existing pattern --- worker/docker.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/worker/docker.go b/worker/docker.go index 527243f23..56072461d 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -95,6 +95,9 @@ func (docker Docker) executeCommand(ctx context.Context, commandTemplate string, } } + if usingCommand { + docker.Event.Info("Running command", "cmd", cmd.String()) + } return cmd.Run() } From eac39ef048a8299f2b6742fb06f03902b7baf0de Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 26 Jul 2024 19:00:57 -0700 Subject: [PATCH 58/67] Update .goreleaser.yaml to replace deprecated fields --- .goreleaser.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 54dbde04d..47c496f38 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -28,10 +28,10 @@ archives: name_template: "{{.ProjectName}}-{{.Os}}-{{.Arch}}-{{.Version}}" brews: - - tap: + - repository: owner: ohsu-comp-bio name: homebrew-formula - folder: Formula + directory: Formula skip_upload: true description: "distributed task execution toolkit" homepage: "https://ohsu-comp-bio.github.io/funnel/" From d7272d4056186501ab98b775478eea1789af4ec5 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Wed, 31 Jul 2024 11:06:55 -0700 Subject: [PATCH 59/67] Update Nextflow instructions --- website/content/docs/integrations/nextflow.md | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/website/content/docs/integrations/nextflow.md b/website/content/docs/integrations/nextflow.md index 37cac2b23..3090a9415 100644 --- a/website/content/docs/integrations/nextflow.md +++ b/website/content/docs/integrations/nextflow.md @@ -23,38 +23,62 @@ To set up Nextflow to use Funnel as the TES executor, run the following steps: ### 1. Install Nextflow +*Adapted from the [Nextflow Documentation](https://nextflow.io/docs/latest/install.html)* + +#### a. Install Nextflow: + +```sh +curl -s https://get.nextflow.io | bash +``` + +This will create the nextflow executable in the current directory. + +#### b. Make Nextflow executable: + ```sh -git clone https://github.com/nextflow-io/nextflow -b tes-update-1.1 +chmod +x nextflow +``` -cd nextflow +#### c. Move Nextflow into an executable path: -make compile +```sh +sudo mv nextflow /usr/local/bin ``` -This will create a new `launch.sh` file that can be used to run the Nextflow workflow below. +#### d. Confirm that Nextflow is installed correctly: + +```sh +nextflow info +``` ### 2. Update Nextflow Config Add the following to your `nextflow.config` in order to use the GA4GH TES plugin: ```yaml +cat <> nextflow.config plugins { id 'nf-ga4gh' } process.executor = 'tes' tes.endpoint = 'http://localhost:8000' # <--- Funnel's default address +EOF ``` -### 3. Start Funnel and Run Workflow +### 3. Start the Funnel Server -Finally, start the Funnel server and launch Nextflow: +Start the Funnel server: ```sh funnel server run +``` + +### 4. Run Nextflow -alias nextflow=~/nextflow/launch.sh # <--- Change this line to match your local nextflow directory +In another window, run the workflow: +```sh nextflow run main.nf -c nextflow.config ``` From 87fbe0b9271c94abe36e381cbc74adc1c6c07913 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 9 Sep 2024 18:21:30 -0700 Subject: [PATCH 60/67] Update K8s support and documentation - Update docker images to use quay.io/ohsu-comp-bio/funnel --- Dockerfile | 2 +- Dockerfile.dind | 4 +- Dockerfile.dind-rootless | 4 +- compute/kubernetes/backend.go | 33 +++++++--- compute/kubernetes/backend_test.go | 61 ++++++++++++++++++- deployments/kubernetes/README.md | 52 +++++++++------- deployments/kubernetes/funnel-deployment.yml | 14 +++-- .../kubernetes/funnel-server-config.yml | 7 +-- 8 files changed, 134 insertions(+), 43 deletions(-) diff --git a/Dockerfile b/Dockerfile index 39c481ab5..695461b78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # build stage -FROM golang:1.21-alpine AS build-env +FROM golang:1.23-alpine AS build-env RUN apk add make git bash build-base ENV GOPATH=/go ENV PATH="/go/bin:${PATH}" diff --git a/Dockerfile.dind b/Dockerfile.dind index 29390cf7e..fc57ab1a2 100644 --- a/Dockerfile.dind +++ b/Dockerfile.dind @@ -1,5 +1,5 @@ # build stage -FROM golang:1.20-alpine AS build-env +FROM golang:1.23-alpine AS build-env RUN apk add make git bash build-base ENV GOPATH=/go ENV PATH="/go/bin:${PATH}" @@ -7,7 +7,7 @@ ADD ./ /go/src/github.com/ohsu-comp-bio/funnel RUN cd /go/src/github.com/ohsu-comp-bio/funnel && make build # final stage -FROM docker:stable-dind +FROM docker:dind WORKDIR /opt/funnel VOLUME /opt/funnel/funnel-work-dir EXPOSE 8000 9090 diff --git a/Dockerfile.dind-rootless b/Dockerfile.dind-rootless index 3daa08593..a7c4c83c5 100644 --- a/Dockerfile.dind-rootless +++ b/Dockerfile.dind-rootless @@ -1,5 +1,5 @@ # build stage -FROM golang:1.20-alpine AS build-env +FROM golang:1.23-alpine AS build-env RUN apk add make git bash build-base ENV GOPATH=/go ENV PATH="/go/bin:${PATH}" @@ -7,7 +7,7 @@ ADD ./ /go/src/github.com/ohsu-comp-bio/funnel RUN cd /go/src/github.com/ohsu-comp-bio/funnel && make build # final stage -FROM docker:stable-dind-rootless +FROM docker:dind-rootless WORKDIR /opt/funnel VOLUME /opt/funnel/funnel-work-dir EXPOSE 8000 9090 diff --git a/compute/kubernetes/backend.go b/compute/kubernetes/backend.go index 9a29c263e..d2cd3159d 100644 --- a/compute/kubernetes/backend.go +++ b/compute/kubernetes/backend.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io/ioutil" "text/template" @@ -82,15 +83,32 @@ func NewBackend(ctx context.Context, conf config.Kubernetes, reader tes.ReadOnly // Backend represents the local backend. type Backend struct { - client batchv1.JobInterface - namespace string - template string - event events.Writer - database tes.ReadOnlyServer - log *logger.Logger + client batchv1.JobInterface + namespace string + template string + event events.Writer + database tes.ReadOnlyServer + log *logger.Logger + backendParameters map[string]string events.Computer } +func (b Backend) CheckBackendParameterSupport(task *tes.Task) error { + if !task.Resources.GetBackendParametersStrict() { + return nil + } + + taskBackendParameters := task.Resources.GetBackendParameters() + for k := range taskBackendParameters { + _, ok := b.backendParameters[k] + if !ok { + return errors.New("backend parameters not supported") + } + } + + return nil +} + // WriteEvent writes an event to the compute backend. // Currently, only TASK_CREATED is handled, which calls Submit. func (b *Backend) WriteEvent(ctx context.Context, ev *events.Event) error { @@ -153,7 +171,8 @@ func (b *Backend) Submit(ctx context.Context, task *tes.Task) error { if err != nil { return fmt.Errorf("creating job spec: %v", err) } - _, err = b.client.Create(ctx, job, metav1.CreateOptions{ + ctx = context.Background() + job, err = b.client.Create(ctx, job, metav1.CreateOptions{ FieldManager: task.Id, }) if err != nil { diff --git a/compute/kubernetes/backend_test.go b/compute/kubernetes/backend_test.go index 313247ea6..d1e0768c5 100644 --- a/compute/kubernetes/backend_test.go +++ b/compute/kubernetes/backend_test.go @@ -1,15 +1,62 @@ package kubernetes import ( + "context" "fmt" "io/ioutil" "testing" + "path/filepath" + "github.com/ohsu-comp-bio/funnel/config" "github.com/ohsu-comp-bio/funnel/logger" "github.com/ohsu-comp-bio/funnel/tes" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" ) +// getKubernetesClient creates a Kubernetes clientset for communication with the Kubernetes API +func getKubernetesClient() (*kubernetes.Clientset, error) { + var config *rest.Config + var err error + var contextName string + + // Try to create an in-cluster client + config, err = rest.InClusterConfig() + if err != nil { + // If in-cluster config fails, fallback to kubeconfig file for out-of-cluster + kubeconfig := filepath.Join(homedir.HomeDir(), ".kube", "config") + kubeConfig, err := clientcmd.LoadFromFile(kubeconfig) + if err != nil { + return nil, fmt.Errorf("failed to load kubeconfig: %v", err) + } + + // Get the current context name + contextName = kubeConfig.CurrentContext + + // Build config from flags or kubeconfig + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, fmt.Errorf("failed to create kubernetes config: %v", err) + } + } else { + contextName = "in-cluster" + } + + // Create the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create kubernetes clientset: %v", err) + } + + // Log or print the current context being used + fmt.Printf("Using Kubernetes context: %s\n", contextName) + + return clientset, nil +} + func TestCreateJobc(t *testing.T) { conf := config.DefaultConfig().Kubernetes content, err := ioutil.ReadFile("../../config/kubernetes-template.yaml") @@ -18,8 +65,15 @@ func TestCreateJobc(t *testing.T) { } conf.Template = string(content) log := logger.NewLogger("test", logger.DefaultConfig()) + + // Create Kubernetes client + clientset, err := getKubernetesClient() + if err != nil { + t.Fatal(fmt.Errorf("creating kubernetes client: %v", err)) + } + b := &Backend{ - client: nil, + client: clientset.BatchV1().Jobs(conf.Namespace), namespace: conf.Namespace, template: conf.Template, event: nil, @@ -42,4 +96,9 @@ func TestCreateJobc(t *testing.T) { t.Fatal(err) } t.Logf("%+v", job) + + err = b.Submit(context.Background(), task) + if err != nil { + t.Fatal(err) + } } diff --git a/deployments/kubernetes/README.md b/deployments/kubernetes/README.md index 2f6a54a24..b3443a2ac 100644 --- a/deployments/kubernetes/README.md +++ b/deployments/kubernetes/README.md @@ -1,3 +1,6 @@ +> [!NOTE] +> Kubernetes support is in active development and may involve frequent updates + # Using Funnel with Kubernetes Examples can be found here: https://github.com/ohsu-comp-bio/funnel/tree/master/deployments/kubernetes @@ -6,7 +9,7 @@ Examples can be found here: https://github.com/ohsu-comp-bio/funnel/tree/master/ *funnel-service.yml* -``` +```yaml apiVersion: v1 kind: Service metadata: @@ -23,18 +26,17 @@ spec: protocol: TCP port: 9090 targetPort: 9090 - ``` Deploy it: -``` +```sh kubectl apply -f funnel-service.yml ``` Get the clusterIP: -``` +```sh kubectl get services funnel --output=yaml | grep clusterIP ``` @@ -42,13 +44,17 @@ Use this value to configure the server hostname of the worker config. #### Create Funnel config files -*Note*: The configures job template uses the image, `ohsucompbio/funnel-dind:latest`, which is built on docker's official [docker-in-docker image (dind)](https://hub.docker.com/_/docker). You can also use the experimental [rootless dind variant](https://docs.docker.com/engine/security/rootless/) by changing the image to `ohsucompbio/funnel-dind-rootless:latest`. +> [!TIP] +> The configures job template uses the image, `ohsucompbio/funnel-dind:latest`, which is built on docker's official [docker-in-docker image (dind)](https://hub.docker.com/_/docker). You can also use the experimental [rootless dind variant](https://docs.docker.com/engine/security/rootless/) by changing the image to `quay.io/ohsu-comp-bio/funnel-dind-rootless:latest` *funnel-server-config.yml* -``` +```yaml Database: boltdb +BoltDB: + Path: /opt/funnel/funnel-work-dir/funnel.bolt.db + Compute: kubernetes Logger: @@ -74,7 +80,7 @@ Kubernetes: restartPolicy: Never containers: - name: {{printf "funnel-worker-%s" .TaskId}} - image: ohsucompbio/funnel-kube-dind:latest + image: quay.io/ohsu-comp-bio/funnel-dind:latest imagePullPolicy: IfNotPresent args: - "funnel" @@ -87,8 +93,8 @@ Kubernetes: resources: requests: cpu: {{if ne .Cpus 0 -}}{{.Cpus}}{{ else }}{{"100m"}}{{end}} - memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16M"}}{{end}} - ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100M"}}{{end}} + memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16Mi"}}{{end}} + ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100Mi"}}{{end}} volumeMounts: - name: {{printf "funnel-storage-%s" .TaskId}} mountPath: {{printf "/opt/funnel/funnel-work-dir/%s" .TaskId}} @@ -112,7 +118,7 @@ I recommend setting `DisableJobCleanup` to `true` for debugging - otherwise fail ***Remember to modify the template below to have the actual server hostname.*** -``` +```yaml Database: boltdb BoltDB: @@ -138,7 +144,7 @@ Server: #### Create a ConfigMap -``` +```sh kubectl create configmap funnel-config --from-file=funnel-server-config.yml --from-file=funnel-worker-config.yml ``` @@ -148,7 +154,7 @@ Define a Role and RoleBinding: *role.yml* -``` +```yaml kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -168,7 +174,7 @@ rules: *role_binding.yml* -``` +```yaml kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -195,7 +201,7 @@ kubectl create -f role_binding.yml *funnel-deployment.yml* -``` +```yaml apiVersion: apps/v1 kind: Deployment metadata: @@ -215,7 +221,7 @@ spec: serviceAccountName: funnel-sa containers: - name: funnel - image: ohsucompbio/funnel:latest + image: quay.io/ohsu-comp-bio/funnel:latest imagePullPolicy: IfNotPresent command: - 'funnel' @@ -225,9 +231,13 @@ spec: - '/etc/config/funnel-server-config.yml' resources: requests: - cpu: 2 + cpu: 500m + memory: 1G + ephemeral-storage: 25G + limits: + cpu: 2000m memory: 4G - ephemeral-storage: 25G # needed since we are using boltdb + ephemeral-storage: 25G volumeMounts: - name: funnel-deployment-storage mountPath: /opt/funnel/funnel-work-dir @@ -247,25 +257,25 @@ spec: Deploy it: -``` +```sh kubectl apply -f funnel-deployment.yml ``` #### Proxy the Service for local testing -``` +```sh kubectl port-forward service/funnel 8000:8000 ``` Now you can access the funnel server locally. Verify by running: -``` +```sh funnel task list ``` Now try running a task: -``` +```sh funnel examples hello-world > hello.json funnel task create hello.json ``` diff --git a/deployments/kubernetes/funnel-deployment.yml b/deployments/kubernetes/funnel-deployment.yml index c861bc3fe..cc41ac518 100644 --- a/deployments/kubernetes/funnel-deployment.yml +++ b/deployments/kubernetes/funnel-deployment.yml @@ -17,7 +17,7 @@ spec: serviceAccountName: funnel-sa containers: - name: funnel - image: ohsucompbio/funnel:latest + image: quay.io/ohsu-comp-bio/funnel:latest imagePullPolicy: IfNotPresent command: - 'funnel' @@ -27,9 +27,13 @@ spec: - '/etc/config/funnel-server-config.yml' resources: requests: - cpu: 2 + cpu: 500m + memory: 1G + ephemeral-storage: 25G + limits: + cpu: 2000m memory: 4G - ephemeral-storage: 25G # needed since we are using boltdb + ephemeral-storage: 25G volumeMounts: - name: funnel-deployment-storage mountPath: /opt/funnel/funnel-work-dir @@ -38,10 +42,10 @@ spec: ports: - containerPort: 8000 - containerPort: 9090 - + volumes: - name: funnel-deployment-storage emptyDir: {} - name: config-volume configMap: - name: funnel-config + name: funnel-config \ No newline at end of file diff --git a/deployments/kubernetes/funnel-server-config.yml b/deployments/kubernetes/funnel-server-config.yml index ae30805ff..c9c10f6c8 100644 --- a/deployments/kubernetes/funnel-server-config.yml +++ b/deployments/kubernetes/funnel-server-config.yml @@ -28,7 +28,7 @@ Kubernetes: restartPolicy: Never containers: - name: {{printf "funnel-worker-%s" .TaskId}} - image: ohsucompbio/funnel-dind:latest + image: quay.io/ohsu-comp-bio/funnel-dind:latest imagePullPolicy: IfNotPresent args: - "funnel" @@ -41,8 +41,8 @@ Kubernetes: resources: requests: cpu: {{if ne .Cpus 0 -}}{{.Cpus}}{{ else }}{{"100m"}}{{end}} - memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16M"}}{{end}} - ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100M"}}{{end}} + memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16Mi"}}{{end}} + ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100Mi"}}{{end}} volumeMounts: - name: {{printf "funnel-storage-%s" .TaskId}} mountPath: {{printf "/opt/funnel/funnel-work-dir/%s" .TaskId}} @@ -58,4 +58,3 @@ Kubernetes: - name: config-volume configMap: name: funnel-config - From 976aee0f6ab511b8cf00f5a36dbaf4a67245f891 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 10 Sep 2024 13:19:15 -0700 Subject: [PATCH 61/67] Update README.md --- deployments/kubernetes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/kubernetes/README.md b/deployments/kubernetes/README.md index b3443a2ac..7a7eb9adf 100644 --- a/deployments/kubernetes/README.md +++ b/deployments/kubernetes/README.md @@ -1,5 +1,5 @@ > [!NOTE] -> Kubernetes support is in active development and may involve frequent updates +> Funnel's Kubernetes support is in active development and may involve frequent updates # Using Funnel with Kubernetes From f5ed646e738276785cb86a37827301b8905c7a67 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 12 Sep 2024 14:29:09 -0700 Subject: [PATCH 62/67] fix: add support for wildcards in AWS S3 paths --- Makefile | 2 +- examples/s3.json | 8 ++++---- storage/amazon_s3.go | 22 +++++++++++++++++++--- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 317970114..8a114743c 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ git_upstream := $(shell git remote get-url $(shell git config branch.$(shell git export GIT_BRANCH = $(git_branch) export GIT_UPSTREAM = $(git_upstream) -export FUNNEL_VERSION=0.11.1-rc.1 +export FUNNEL_VERSION=0.11.1-rc.3 # LAST_PR_NUMBER is used by the release notes builder to generate notes # based on pull requests (PR) up until the last release. diff --git a/examples/s3.json b/examples/s3.json index c89c885dc..9f0e3064f 100644 --- a/examples/s3.json +++ b/examples/s3.json @@ -4,15 +4,15 @@ "executors": [ { "image": "ubuntu", - "command": ["md5sum", "/tmp/file.xml"] + "command": ["md5sum", "/tmp/release.json"] } ], "inputs": [ { "name": "input", - "description": "Download a public file from S3 Storage (IRS form 990 data)", - "url": "s3://irs-form-990/200931393493000150_public.xml", - "path": "/tmp/file.xml" + "description": "Download a public file from S3 Storage (CZ CELLxGENE Discover Census Data: https://registry.opendata.aws/tag/bioinformatics/)", + "url": "s3://cellxgene-census-public-us-west-2/cell-census/release.json", + "path": "/tmp/release.json" } ] } diff --git a/storage/amazon_s3.go b/storage/amazon_s3.go index 4d6aa2753..e67d68359 100644 --- a/storage/amazon_s3.go +++ b/storage/amazon_s3.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "regexp" "strings" @@ -232,9 +233,24 @@ func (s3b *AmazonS3) Put(ctx context.Context, url, path string) (*Object, error) sess := s3b.sess.Copy(&aws.Config{Region: aws.String(region)}) manager := s3manager.NewUploader(sess) - hf, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("amazonS3: opening file: %v", err) + var hf *os.File + // If path contains a wildcard, then handle globbing + if strings.Contains(path, "*") { + globs, err := filepath.Glob(path) + if err != nil { + return nil, fmt.Errorf("amazonS3: failed to resolve path %v: %v", path, err) + } + for _, glob := range globs { + hf, err = os.Open(glob) + if err != nil { + return nil, fmt.Errorf("amazonS3: opening file %v: %v", glob, err) + } + } + } else { + hf, err = os.Open(path) + if err != nil { + return nil, fmt.Errorf("amazonS3: opening fil %v: %v", path, err) + } } defer hf.Close() From 42fe5973d590d7f3555346b3e4a4b837ba1a32c9 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 17 Sep 2024 16:47:12 -0700 Subject: [PATCH 63/67] Add support for wildcards in AWS S3 output paths --- worker/storage.go | 39 +++++++++++++++++++++++++++++++++++++++ worker/worker.go | 7 +++++++ 2 files changed, 46 insertions(+) diff --git a/worker/storage.go b/worker/storage.go index 0edba8d34..7bacfa262 100644 --- a/worker/storage.go +++ b/worker/storage.go @@ -3,10 +3,12 @@ package worker import ( "context" "fmt" + "net/url" "os" "path/filepath" "strings" + proto "github.com/golang/protobuf/proto" "github.com/ohsu-comp-bio/funnel/events" "github.com/ohsu-comp-bio/funnel/storage" "github.com/ohsu-comp-bio/funnel/tes" @@ -246,3 +248,40 @@ func fixLinks(mapper *FileMapper, basepath string) { return nil }) } + +// resolveWildcards resolves any wildcards/globs in the output paths. +func resolveWildcards(mapper *FileMapper, output *tes.Output) error { + var outputs []*tes.Output + + // If path contains a wildcard, handle globbing and upload each file individually + if strings.Contains(output.Path, "*") { + globs, err := filepath.Glob(output.Path) + if err != nil { + return fmt.Errorf("failed to resolve path %v: %v", output.Path, err) + } + + for _, glob := range globs { + out := proto.Clone(output).(*tes.Output) + out.Path = glob + + // Construct URL by: + // - removing the mapper.WorkDir from the output.Path + // - then removing the PathPrefix + // - then joining the output.URL + globPath := strings.TrimPrefix(glob, mapper.WorkDir) + fixPath := strings.TrimPrefix(globPath, output.PathPrefix) + out.Url, err = url.JoinPath(output.Url, fixPath) + if err != nil { + return fmt.Errorf("failed to join URL: %v", err) + } + + outputs = append(outputs, out) + } + } else { + outputs = append(outputs, output) + } + + mapper.Outputs = outputs + + return nil +} diff --git a/worker/worker.go b/worker/worker.go index dfdff7c96..1c5189dd9 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -206,6 +206,13 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { } } + if run.ok() { + // Resolve wildcards in the output paths + for _, output := range mapper.Outputs { + resolveWildcards(mapper, output) + } + } + if run.ok() && r.Conf.ScratchPath != "" { mapper.CopyOutputsToWorkDir(r.Conf.ScratchPath) } From a570bf3fa12be4e1f6ff690eaceca6d75b8be311 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Wed, 18 Sep 2024 12:49:17 -0700 Subject: [PATCH 64/67] Update actions/upload-artifact to v4 in Github Actions - https://github.blog/changelog/2024-02-13-deprecation-notice-v1-and-v2-of-the-artifact-actions/ --- .github/workflows/compliance-test.yaml | 2 +- .github/workflows/nextflow.yaml | 2 +- .github/workflows/s3-test.yaml | 2 +- .github/workflows/tests.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/compliance-test.yaml b/.github/workflows/compliance-test.yaml index b1ade0a6d..95ed4ecbd 100644 --- a/.github/workflows/compliance-test.yaml +++ b/.github/workflows/compliance-test.yaml @@ -42,7 +42,7 @@ jobs: run: make build - name: Store funnel - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: funnelBin path: funnel diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index fd30af166..33c2c4615 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -19,7 +19,7 @@ jobs: run: make build - name: Store Funnel - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: funnelBin path: funnel diff --git a/.github/workflows/s3-test.yaml b/.github/workflows/s3-test.yaml index e84622a23..1ac1cc241 100644 --- a/.github/workflows/s3-test.yaml +++ b/.github/workflows/s3-test.yaml @@ -22,7 +22,7 @@ jobs: run: make build - name: Store funnel - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: funnelBin path: funnel diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0d749da5d..dedb8e7ab 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -34,7 +34,7 @@ jobs: run: make build - name: Store funnel - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: funnelBin path: funnel From 119b55436dc9dd575d338740fe97286fc6c660d2 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Wed, 18 Sep 2024 12:50:44 -0700 Subject: [PATCH 65/67] Update version to 0.11.1-rc.4 in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8a114743c..abe6d0c0f 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ git_upstream := $(shell git remote get-url $(shell git config branch.$(shell git export GIT_BRANCH = $(git_branch) export GIT_UPSTREAM = $(git_upstream) -export FUNNEL_VERSION=0.11.1-rc.3 +export FUNNEL_VERSION=0.11.1-rc.4 # LAST_PR_NUMBER is used by the release notes builder to generate notes # based on pull requests (PR) up until the last release. From f8db3a523116ddb1dba56baee256f4741dddc816 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 19 Sep 2024 11:02:54 -0700 Subject: [PATCH 66/67] Fix bug where FileMapper output was being overwritten in every loop --- Makefile | 2 +- worker/storage.go | 49 ++++++++++++++++++++++++----------------------- worker/worker.go | 4 +--- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index abe6d0c0f..49bd35bb0 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ git_upstream := $(shell git remote get-url $(shell git config branch.$(shell git export GIT_BRANCH = $(git_branch) export GIT_UPSTREAM = $(git_upstream) -export FUNNEL_VERSION=0.11.1-rc.4 +export FUNNEL_VERSION=0.11.1-rc.5 # LAST_PR_NUMBER is used by the release notes builder to generate notes # based on pull requests (PR) up until the last release. diff --git a/worker/storage.go b/worker/storage.go index 7bacfa262..a82bcc845 100644 --- a/worker/storage.go +++ b/worker/storage.go @@ -250,38 +250,39 @@ func fixLinks(mapper *FileMapper, basepath string) { } // resolveWildcards resolves any wildcards/globs in the output paths. -func resolveWildcards(mapper *FileMapper, output *tes.Output) error { +func resolveWildcards(mapper *FileMapper) error { var outputs []*tes.Output - // If path contains a wildcard, handle globbing and upload each file individually - if strings.Contains(output.Path, "*") { - globs, err := filepath.Glob(output.Path) - if err != nil { - return fmt.Errorf("failed to resolve path %v: %v", output.Path, err) - } - - for _, glob := range globs { - out := proto.Clone(output).(*tes.Output) - out.Path = glob - - // Construct URL by: - // - removing the mapper.WorkDir from the output.Path - // - then removing the PathPrefix - // - then joining the output.URL - globPath := strings.TrimPrefix(glob, mapper.WorkDir) - fixPath := strings.TrimPrefix(globPath, output.PathPrefix) - out.Url, err = url.JoinPath(output.Url, fixPath) + for _, output := range mapper.Outputs { + // If path contains a wildcard, handle globbing and upload each file individually + if strings.Contains(output.Path, "*") { + globs, err := filepath.Glob(output.Path) if err != nil { - return fmt.Errorf("failed to join URL: %v", err) + return fmt.Errorf("failed to resolve path %v: %v", output.Path, err) } - outputs = append(outputs, out) + for _, glob := range globs { + out := proto.Clone(output).(*tes.Output) + out.Path = glob + + // Construct URL by: + // - removing the mapper.WorkDir from the output.Path + // - then removing the PathPrefix + // - then joining the output.URL + globPath := strings.TrimPrefix(glob, mapper.WorkDir) + fixPath := strings.TrimPrefix(globPath, output.PathPrefix) + out.Url, err = url.JoinPath(output.Url, fixPath) + if err != nil { + return fmt.Errorf("failed to join URL: %v", err) + } + + outputs = append(outputs, out) + } + } else { + outputs = append(outputs, output) } - } else { - outputs = append(outputs, output) } mapper.Outputs = outputs - return nil } diff --git a/worker/worker.go b/worker/worker.go index 1c5189dd9..8e4e68223 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -208,9 +208,7 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { if run.ok() { // Resolve wildcards in the output paths - for _, output := range mapper.Outputs { - resolveWildcards(mapper, output) - } + resolveWildcards(mapper) } if run.ok() && r.Conf.ScratchPath != "" { From b2579415a42d44bf277c0864bed2289261550e15 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 30 Sep 2024 16:23:35 -0700 Subject: [PATCH 67/67] Add initial search functionality to website - Uses Pagefind: https://pagefind.app/docs/ --- .github/workflows/hugo.yml | 5 +++++ website/layouts/partials/head.html | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 0e3d40fc5..81986c025 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -59,6 +59,11 @@ jobs: --baseURL "${{ steps.pages.outputs.base_url }}/" \ --source website \ --destination public + + # Run pagefind to add search functionality + # Reference: https://pagefind.app/docs/ + npx -y pagefind --site public + - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: diff --git a/website/layouts/partials/head.html b/website/layouts/partials/head.html index 683833b07..e7189503b 100644 --- a/website/layouts/partials/head.html +++ b/website/layouts/partials/head.html @@ -28,6 +28,15 @@ + + + + + @@ -48,3 +57,5 @@ + +