From 5b0ceba57997d80073fcb85c49f7fca8dab38f3a Mon Sep 17 00:00:00 2001 From: apoorvam Date: Fri, 22 Mar 2019 23:31:43 -0400 Subject: [PATCH 01/41] Add a command `validate` to check for parse errors #31 #55 Running command `dunner validate` will check for any parse errors and warnings. If there are any errors, it fails the cmd, if not it finishes by listing any warnings if present. --- Makefile | 12 ++++++++-- cmd/validate.go | 18 +++++++++++++++ pkg/config/config.go | 25 +++++++++++++++++++++ pkg/config/config_test.go | 47 +++++++++++++++++++++++++++++++++++++++ pkg/config/validate.go | 25 +++++++++++++++++++++ pkg/dunner/dunner.go | 11 +++++---- 6 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 cmd/validate.go create mode 100644 pkg/config/validate.go diff --git a/Makefile b/Makefile index 84223f6..58f61dc 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GOCMD=go GOINSTALL=$(GOCMD) install GOTEST=$(GOCMD) test DEP=dep -.PHONY : all install test +.PHONY : all install vet fmt test all : def @@ -18,4 +18,12 @@ test: @$(GOTEST) -v ./... clean: - rm -rf * \ No newline at end of file + rm -rf * + +vet: + @echo "=== go vet ===" + @go vet ./... + +fmt: + @echo "=== go fmt ===" + @go fmt ./... diff --git a/cmd/validate.go b/cmd/validate.go new file mode 100644 index 0000000..913ed74 --- /dev/null +++ b/cmd/validate.go @@ -0,0 +1,18 @@ +package cmd + +import ( + "github.com/leopardslab/Dunner/pkg/config" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(validateCmd) +} + +var validateCmd = &cobra.Command{ + Use: "validate", + Short: "Validate the dunner task file `.dunner.yaml`", + Long: "You can validate task file `.dunner.yaml` with this command to see if there are any parse errors", + Run: config.Validate, + Args: cobra.MinimumNArgs(0), +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 46d3b48..4837a8a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -41,6 +41,31 @@ type Configs struct { Tasks map[string][]Task } +// Validates config and returns a list of errors and warnings. If errors are not critical/only warnings, it returns param `ok` as true, else false +func (configs *Configs) Validate() ([]error, bool) { + var errs []error + var warnings []error + if len(configs.Tasks) == 0 { + warnings = append(warnings, fmt.Errorf("dunner: No tasks defined")) + } + + for taskName, tasks := range configs.Tasks { + for _, task := range tasks { + if task.Image == "" { + errs = append(errs, fmt.Errorf(`dunner: [%s] Image repository name cannot be empty`, taskName)) + } + if len(task.Command) == 0 { + errs = append(errs, fmt.Errorf("dunner: [%s] Commands not defined for task with image %s", taskName, task.Image)) + } + } + } + if len(errs) > 0 { + errs = append(errs, warnings...) + return errs, false + } + return warnings, true +} + // GetConfigs reads and parses tasks from the dunner file func GetConfigs(filename string) (*Configs, error) { fileContents, err := ioutil.ReadFile(filename) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 0058078..d2461c5 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -54,3 +54,50 @@ test: } } + +func TestConfigs_Validate(t *testing.T) { + tasks := make(map[string][]Task, 0) + tasks["stats"] = []Task{getSampleTask()} + configs := &Configs{Tasks: tasks} + + errs, ok := configs.Validate() + + if !ok || len(errs) != 0 { + t.Fatalf("Configs Validation failed, expected to pass") + } +} + +func TestConfigs_ValidateWithNoTasks(t *testing.T) { + tasks := make(map[string][]Task, 0) + configs := &Configs{Tasks: tasks} + + errs, ok := configs.Validate() + + if !ok || len(errs) != 1 { + t.Fatalf("Configs validation failed") + } + if errs[0].Error() != "dunner: No tasks defined" { + t.Fatalf("Configs Validation error message not as expected") + } +} + +func TestConfigs_ValidateWithParseErrors(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := Task{Image: "", Command: []string{}} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs, ok := configs.Validate() + + if ok || len(errs) != 2 { + t.Fatalf("Configs validation failed") + } + + if errs[0].Error() != "dunner: [stats] Image repository name cannot be empty" || errs[1].Error() != "dunner: [stats] Commands not defined for task with image " { + t.Fatalf("Configs Validation error message not as expected") + } +} + +func getSampleTask() Task { + return Task{Image: "image_name", Command: []string{"node", "--version"}} +} diff --git a/pkg/config/validate.go b/pkg/config/validate.go new file mode 100644 index 0000000..a735d6d --- /dev/null +++ b/pkg/config/validate.go @@ -0,0 +1,25 @@ +package config + +import ( + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func Validate(_ *cobra.Command, args []string) { + var dunnerFile = viper.GetString("DunnerTaskFile") + + configs, err := GetConfigs(dunnerFile) + if err != nil { + log.Fatal(err) + } + + errs, ok := configs.Validate() + for _, err := range errs { + log.Error(err) + } + if !ok { + os.Exit(1) + } +} diff --git a/pkg/dunner/dunner.go b/pkg/dunner/dunner.go index 5e3152c..f6d64fa 100644 --- a/pkg/dunner/dunner.go +++ b/pkg/dunner/dunner.go @@ -32,6 +32,13 @@ func Do(_ *cobra.Command, args []string) { if err != nil { log.Fatal(err) } + errs, ok := configs.Validate() + for _, err := range errs { + log.Error(err) + } + if !ok { + os.Exit(1) + } execTask(configs, args[0], args[1:]) } @@ -91,10 +98,6 @@ func process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args [ log.Fatal(err) } - if s.Image == "" { - log.Fatalf(`dunner: image repository name cannot be empty`) - } - pout, err := (*s).Exec() if err != nil { log.Fatal(err) From 114cea041f4e9e0bc00204d9a3fe9e046674de23 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Sun, 12 May 2019 15:11:07 -0400 Subject: [PATCH 02/41] Add commands to makefile to build, lint, vet, format, test coverage --- .gitignore | 6 ++++++ Makefile | 40 ++++++++++++++++++++++++++-------------- pkg/config/config.go | 8 +------- pkg/config/validate.go | 1 + 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 540eee7..899734a 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,9 @@ # Environment file .env + +# Coverage files +coverage* + +# IDE files +.idea/* \ No newline at end of file diff --git a/Makefile b/Makefile index 58f61dc..4c3f245 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,41 @@ -#Go parameters +ALL_PACKAGES=$(shell go list ./... | grep -v "vendor") +#Go parameters GOCMD=go GOINSTALL=$(GOCMD) install GOTEST=$(GOCMD) test -DEP=dep -.PHONY : all install vet fmt test +DEP=dep +.PHONY : all install vet fmt test lint build -all : def +all: build test fmt lint vet -def : - @$(GOINSTALL) -ldflags '-s' +setup: install + @go get -u golang.org/x/lint/golint install: @$(DEP) ensure -test: - @$(GOTEST) -v ./... +build: + @go build ./... -clean: - rm -rf * +test: build + @$(GOTEST) -v $(ALL_PACKAGES) vet: - @echo "=== go vet ===" - @go vet ./... + @go vet $(ALL_PACKAGES) fmt: - @echo "=== go fmt ===" - @go fmt ./... + @go fmt $(ALL_PACKAGES) + +lint: + @golint -set_exit_status $(ALL_PACKAGES) + +precommit: build test fmt lint vet + +test-coverage: + @echo "mode: count" > coverage-all.out + + $(foreach pkg, $(ALL_PACKAGES),\ + go test -coverprofile=coverage.out -covermode=count $(pkg);\ + tail -n +2 coverage.out >> coverage-all.out;) + @go tool cover -html=coverage-all.out -o coverage.html diff --git a/pkg/config/config.go b/pkg/config/config.go index 4837a8a..32c9c44 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -19,12 +19,6 @@ import ( var log = logger.Log -type DirMount struct { - Src string `yaml:"src"` - Dest string `yaml:"dest"` - ReadOnly bool `yaml:"read-only"` -} - // Task describes a single task to be run in a docker container type Task struct { Name string `yaml:"name"` @@ -41,7 +35,7 @@ type Configs struct { Tasks map[string][]Task } -// Validates config and returns a list of errors and warnings. If errors are not critical/only warnings, it returns param `ok` as true, else false +// Validate config and return a list of errors and warnings. If errors are not critical/only warnings, it returns param `ok` as true, else false func (configs *Configs) Validate() ([]error, bool) { var errs []error var warnings []error diff --git a/pkg/config/validate.go b/pkg/config/validate.go index a735d6d..70970b8 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/viper" ) +// Validate the configuration, fails if there are any errors func Validate(_ *cobra.Command, args []string) { var dunnerFile = viper.GetString("DunnerTaskFile") From 015fed67f19c71903845033e5f59b19e9e4d0a6d Mon Sep 17 00:00:00 2001 From: apoorvam Date: Sun, 12 May 2019 15:18:12 -0400 Subject: [PATCH 03/41] Remove changes from validate branch --- Makefile | 2 +- cmd/validate.go | 18 --------------- pkg/config/config.go | 25 --------------------- pkg/config/config_test.go | 47 --------------------------------------- pkg/config/validate.go | 26 ---------------------- pkg/dunner/dunner.go | 11 ++++----- 6 files changed, 5 insertions(+), 124 deletions(-) delete mode 100644 cmd/validate.go delete mode 100644 pkg/config/validate.go diff --git a/Makefile b/Makefile index 4c3f245..193f114 100644 --- a/Makefile +++ b/Makefile @@ -38,4 +38,4 @@ test-coverage: $(foreach pkg, $(ALL_PACKAGES),\ go test -coverprofile=coverage.out -covermode=count $(pkg);\ tail -n +2 coverage.out >> coverage-all.out;) - @go tool cover -html=coverage-all.out -o coverage.html + @go tool cover -html=coverage-all.out -o coverage.html \ No newline at end of file diff --git a/cmd/validate.go b/cmd/validate.go deleted file mode 100644 index 913ed74..0000000 --- a/cmd/validate.go +++ /dev/null @@ -1,18 +0,0 @@ -package cmd - -import ( - "github.com/leopardslab/Dunner/pkg/config" - "github.com/spf13/cobra" -) - -func init() { - rootCmd.AddCommand(validateCmd) -} - -var validateCmd = &cobra.Command{ - Use: "validate", - Short: "Validate the dunner task file `.dunner.yaml`", - Long: "You can validate task file `.dunner.yaml` with this command to see if there are any parse errors", - Run: config.Validate, - Args: cobra.MinimumNArgs(0), -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 32c9c44..d549751 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -35,31 +35,6 @@ type Configs struct { Tasks map[string][]Task } -// Validate config and return a list of errors and warnings. If errors are not critical/only warnings, it returns param `ok` as true, else false -func (configs *Configs) Validate() ([]error, bool) { - var errs []error - var warnings []error - if len(configs.Tasks) == 0 { - warnings = append(warnings, fmt.Errorf("dunner: No tasks defined")) - } - - for taskName, tasks := range configs.Tasks { - for _, task := range tasks { - if task.Image == "" { - errs = append(errs, fmt.Errorf(`dunner: [%s] Image repository name cannot be empty`, taskName)) - } - if len(task.Command) == 0 { - errs = append(errs, fmt.Errorf("dunner: [%s] Commands not defined for task with image %s", taskName, task.Image)) - } - } - } - if len(errs) > 0 { - errs = append(errs, warnings...) - return errs, false - } - return warnings, true -} - // GetConfigs reads and parses tasks from the dunner file func GetConfigs(filename string) (*Configs, error) { fileContents, err := ioutil.ReadFile(filename) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index d2461c5..0058078 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -54,50 +54,3 @@ test: } } - -func TestConfigs_Validate(t *testing.T) { - tasks := make(map[string][]Task, 0) - tasks["stats"] = []Task{getSampleTask()} - configs := &Configs{Tasks: tasks} - - errs, ok := configs.Validate() - - if !ok || len(errs) != 0 { - t.Fatalf("Configs Validation failed, expected to pass") - } -} - -func TestConfigs_ValidateWithNoTasks(t *testing.T) { - tasks := make(map[string][]Task, 0) - configs := &Configs{Tasks: tasks} - - errs, ok := configs.Validate() - - if !ok || len(errs) != 1 { - t.Fatalf("Configs validation failed") - } - if errs[0].Error() != "dunner: No tasks defined" { - t.Fatalf("Configs Validation error message not as expected") - } -} - -func TestConfigs_ValidateWithParseErrors(t *testing.T) { - tasks := make(map[string][]Task, 0) - task := Task{Image: "", Command: []string{}} - tasks["stats"] = []Task{task} - configs := &Configs{Tasks: tasks} - - errs, ok := configs.Validate() - - if ok || len(errs) != 2 { - t.Fatalf("Configs validation failed") - } - - if errs[0].Error() != "dunner: [stats] Image repository name cannot be empty" || errs[1].Error() != "dunner: [stats] Commands not defined for task with image " { - t.Fatalf("Configs Validation error message not as expected") - } -} - -func getSampleTask() Task { - return Task{Image: "image_name", Command: []string{"node", "--version"}} -} diff --git a/pkg/config/validate.go b/pkg/config/validate.go deleted file mode 100644 index 70970b8..0000000 --- a/pkg/config/validate.go +++ /dev/null @@ -1,26 +0,0 @@ -package config - -import ( - "os" - - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -// Validate the configuration, fails if there are any errors -func Validate(_ *cobra.Command, args []string) { - var dunnerFile = viper.GetString("DunnerTaskFile") - - configs, err := GetConfigs(dunnerFile) - if err != nil { - log.Fatal(err) - } - - errs, ok := configs.Validate() - for _, err := range errs { - log.Error(err) - } - if !ok { - os.Exit(1) - } -} diff --git a/pkg/dunner/dunner.go b/pkg/dunner/dunner.go index f6d64fa..5e3152c 100644 --- a/pkg/dunner/dunner.go +++ b/pkg/dunner/dunner.go @@ -32,13 +32,6 @@ func Do(_ *cobra.Command, args []string) { if err != nil { log.Fatal(err) } - errs, ok := configs.Validate() - for _, err := range errs { - log.Error(err) - } - if !ok { - os.Exit(1) - } execTask(configs, args[0], args[1:]) } @@ -98,6 +91,10 @@ func process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args [ log.Fatal(err) } + if s.Image == "" { + log.Fatalf(`dunner: image repository name cannot be empty`) + } + pout, err := (*s).Exec() if err != nil { log.Fatal(err) From 49925ab4297e072aea312cccf9c34900074b4efd Mon Sep 17 00:00:00 2001 From: apoorvam Date: Sun, 12 May 2019 15:34:30 -0400 Subject: [PATCH 04/41] Reference go binary --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 193f114..f550130 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ build: @go build ./... test: build - @$(GOTEST) -v $(ALL_PACKAGES) + @go -v $(ALL_PACKAGES) vet: @go vet $(ALL_PACKAGES) From b13a19dcc18bf7cc254a487691aba51ec75bc2d2 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Sun, 12 May 2019 20:56:07 -0400 Subject: [PATCH 05/41] Update build command flags in makefile --- Makefile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f550130..56e2e84 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,13 @@ ALL_PACKAGES=$(shell go list ./... | grep -v "vendor") +SHA=$(shell git rev-list HEAD --max-count=1 --abbrev-commit) +TAG?=$(shell git tag -l --contains HEAD) +VERSION=$(TAG) + +ifeq ($(VERSION),) +VERSION := latest +endif + #Go parameters GOCMD=go GOINSTALL=$(GOCMD) install @@ -15,8 +23,8 @@ setup: install install: @$(DEP) ensure -build: - @go build ./... +build: install + @$(GOINSTALL) -ldflags "-X main.version=$(VERSION)-$(SHA) -s" test: build @go -v $(ALL_PACKAGES) From 3e727d3b19c64bdcb7c4cdcd9a3567549d5074bc Mon Sep 17 00:00:00 2001 From: apoorvam Date: Tue, 14 May 2019 16:59:05 -0400 Subject: [PATCH 06/41] Add validation using goplayground/validator --- Gopkg.lock | 51 +++++++++++++++++++++++++ Gopkg.toml | 12 ++++++ cmd/validate.go | 23 ++++++++++- pkg/config/config.go | 80 +++++++++++++++++++++++++++++---------- pkg/config/config_test.go | 32 +++++++++------- pkg/config/validate.go | 25 ------------ pkg/dunner/dunner.go | 10 ++--- 7 files changed, 170 insertions(+), 63 deletions(-) delete mode 100644 pkg/config/validate.go diff --git a/Gopkg.lock b/Gopkg.lock index eeddb63..6c33aad 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -97,6 +97,34 @@ revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" +[[projects]] + digest = "1:fd9d04d4fe18d02ecd2cdd0b8335a3c27210b10a9a5b92b9f01ac3caf8648e36" + name = "github.com/go-playground/locales" + packages = [ + ".", + "currency", + "en", + ] + pruneopts = "UT" + revision = "f63010822830b6fe52288ee52d5a1151088ce039" + version = "v0.12.1" + +[[projects]] + digest = "1:e022cf244bcac1b6ef933f1a2e0adcf6a6dfd7b872d8d41e4d4179bb09a87cbc" + name = "github.com/go-playground/universal-translator" + packages = ["."] + pruneopts = "UT" + revision = "b32fa301c9fe55953584134cb6853a13c87ec0a1" + version = "v0.16.0" + +[[projects]] + digest = "1:7bb1ad654b21fb8f44c77c07840007d6796f70d42d1b470600f943a508ff8a72" + name = "github.com/go-playground/validator" + packages = ["."] + pruneopts = "UT" + revision = "46b4b1e301c24cac870ffcb4ba5c8a703d1ef475" + version = "v9.28.0" + [[projects]] digest = "1:701239373bbf22998c5a36c76d2ff4ee309b3980e39b7a66d72cabd3aab748fa" name = "github.com/gogo/protobuf" @@ -148,6 +176,14 @@ revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242" version = "v1.0.1" +[[projects]] + digest = "1:6782ffc812e8e700e6952ede1e60487ff1fd9da489eff762985be662a7cfc431" + name = "github.com/leodido/go-urn" + packages = ["."] + pruneopts = "UT" + revision = "70078a794e8ea4b497ba7c19a78cd60f90ccf0f4" + version = "v1.1.0" + [[projects]] digest = "1:c568d7727aa262c32bdf8a3f7db83614f7af0ed661474b24588de635c20024c7" name = "github.com/magiconair/properties" @@ -303,6 +339,17 @@ revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" +[[projects]] + digest = "1:d9ed69c1d4d528e4a7779accc3ee070e68760cdd822f69ff2cf8ae0f677afbe2" + name = "gopkg.in/go-playground/validator.v9" + packages = [ + ".", + "translations/en", + ] + pruneopts = "UT" + revision = "46b4b1e301c24cac870ffcb4ba5c8a703d1ef475" + version = "v9.28.0" + [[projects]] digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96" name = "gopkg.in/yaml.v2" @@ -322,10 +369,14 @@ "github.com/docker/docker/pkg/jsonmessage", "github.com/docker/docker/pkg/stdcopy", "github.com/docker/docker/pkg/term", + "github.com/go-playground/locales/en", + "github.com/go-playground/universal-translator", + "github.com/go-playground/validator", "github.com/joho/godotenv", "github.com/sirupsen/logrus", "github.com/spf13/cobra", "github.com/spf13/viper", + "gopkg.in/go-playground/validator.v9/translations/en", "gopkg.in/yaml.v2", ] solver-name = "gps-cdcl" diff --git a/Gopkg.toml b/Gopkg.toml index 18489df..c2ffed3 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -41,3 +41,15 @@ [prune] go-tests = true unused-packages = true + +[[constraint]] + name = "github.com/go-playground/validator" + version = "9.28.0" + +[[constraint]] + name = "github.com/go-playground/locales" + version = "0.12.1" + +[[constraint]] + name = "gopkg.in/go-playground/validator.v9" + version = "9.28.0" diff --git a/cmd/validate.go b/cmd/validate.go index 913ed74..26d06bd 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -1,8 +1,11 @@ package cmd import ( + "os" + "github.com/leopardslab/Dunner/pkg/config" "github.com/spf13/cobra" + "github.com/spf13/viper" ) func init() { @@ -13,6 +16,24 @@ var validateCmd = &cobra.Command{ Use: "validate", Short: "Validate the dunner task file `.dunner.yaml`", Long: "You can validate task file `.dunner.yaml` with this command to see if there are any parse errors", - Run: config.Validate, + Run: Validate, Args: cobra.MinimumNArgs(0), } + +// Validate command invoked from command line, validates the dunner task file. If there are errors, it fails with non-zero exit code. +func Validate(_ *cobra.Command, args []string) { + var dunnerFile = viper.GetString("DunnerTaskFile") + + configs, err := config.GetConfigs(dunnerFile) + if err != nil { + log.Fatal(err) + } + + errs := configs.Validate() + if len(errs) != 0 { + for _, err := range errs { + log.Error(err) + } + os.Exit(1) + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 4837a8a..dd8e245 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -6,19 +6,30 @@ import ( "os" "path" "path/filepath" + "reflect" "regexp" "strings" "github.com/docker/docker/api/types/mount" + "github.com/go-playground/locales/en" + ut "github.com/go-playground/universal-translator" "github.com/joho/godotenv" "github.com/leopardslab/Dunner/internal/logger" "github.com/leopardslab/Dunner/pkg/docker" "github.com/spf13/viper" + "gopkg.in/go-playground/validator.v9" + en_translations "gopkg.in/go-playground/validator.v9/translations/en" yaml "gopkg.in/yaml.v2" ) var log = logger.Log +var ( + uni *ut.UniversalTranslator + govalidator *validator.Validate + trans ut.Translator +) + type DirMount struct { Src string `yaml:"src"` Dest string `yaml:"dest"` @@ -28,9 +39,9 @@ type DirMount struct { // Task describes a single task to be run in a docker container type Task struct { Name string `yaml:"name"` - Image string `yaml:"image"` + Image string `yaml:"image" validate:"required"` SubDir string `yaml:"dir"` - Command []string `yaml:"command"` + Command []string `yaml:"command" validate:"required,min=1,dive,required"` Envs []string `yaml:"envs"` Mounts []string `yaml:"mounts"` Args []string `yaml:"args"` @@ -38,32 +49,63 @@ type Task struct { // Configs describes the parsed information from the dunner file type Configs struct { - Tasks map[string][]Task + Tasks map[string][]Task `validate:"required,min=1,dive,keys,required,endkeys,required,min=1,required"` } -// Validates config and returns a list of errors and warnings. If errors are not critical/only warnings, it returns param `ok` as true, else false -func (configs *Configs) Validate() ([]error, bool) { - var errs []error - var warnings []error - if len(configs.Tasks) == 0 { - warnings = append(warnings, fmt.Errorf("dunner: No tasks defined")) +// Validate validates config and returns errors. +func (configs *Configs) Validate() []error { + err := initValidator() + if err != nil { + return []error{err} } + valErrs := govalidator.Struct(configs) + errs := formatErrors(valErrs, "") for taskName, tasks := range configs.Tasks { - for _, task := range tasks { - if task.Image == "" { - errs = append(errs, fmt.Errorf(`dunner: [%s] Image repository name cannot be empty`, taskName)) - } - if len(task.Command) == 0 { - errs = append(errs, fmt.Errorf("dunner: [%s] Commands not defined for task with image %s", taskName, task.Image)) + taskValErrs := govalidator.Var(tasks, "dive") + errs = append(errs, formatErrors(taskValErrs, taskName)...) + } + return errs +} + +func formatErrors(valErrs error, taskName string) []error { + var errs []error + if valErrs != nil { + if _, ok := valErrs.(*validator.InvalidValidationError); ok { + errs = append(errs, valErrs) + } else { + for _, e := range valErrs.(validator.ValidationErrors) { + if taskName == "" { + errs = append(errs, fmt.Errorf(e.Translate(trans))) + } else { + errs = append(errs, fmt.Errorf("task '%s': %s", taskName, e.Translate(trans))) + } } } } - if len(errs) > 0 { - errs = append(errs, warnings...) - return errs, false + return errs +} + +func initValidator() error { + govalidator = validator.New() + govalidator.RegisterTagNameFunc(func(fld reflect.StructField) string { + name := strings.SplitN(fld.Tag.Get("yaml"), ",", 2)[0] + if name == "-" { + return "" + } + return name + }) + + translator := en.New() + uni = ut.New(translator, translator) + + var translatorFound bool + trans, translatorFound = uni.GetTranslator("en") + if !translatorFound { + return fmt.Errorf("failed to initialize validator with translator") } - return warnings, true + en_translations.RegisterDefaultTranslations(govalidator, trans) + return nil } // GetConfigs reads and parses tasks from the dunner file diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index d2461c5..2552234 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -60,10 +60,10 @@ func TestConfigs_Validate(t *testing.T) { tasks["stats"] = []Task{getSampleTask()} configs := &Configs{Tasks: tasks} - errs, ok := configs.Validate() + errs := configs.Validate() - if !ok || len(errs) != 0 { - t.Fatalf("Configs Validation failed, expected to pass") + if len(errs) != 0 { + t.Fatalf("Configs Validation failed, expected to pass. got: %s", errs) } } @@ -71,13 +71,14 @@ func TestConfigs_ValidateWithNoTasks(t *testing.T) { tasks := make(map[string][]Task, 0) configs := &Configs{Tasks: tasks} - errs, ok := configs.Validate() + errs := configs.Validate() - if !ok || len(errs) != 1 { - t.Fatalf("Configs validation failed") + if len(errs) != 1 { + t.Fatalf("Configs validation failed, expected 1 error, got %s", errs) } - if errs[0].Error() != "dunner: No tasks defined" { - t.Fatalf("Configs Validation error message not as expected") + expected := "Tasks must contain at least 1 item" + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) } } @@ -87,14 +88,19 @@ func TestConfigs_ValidateWithParseErrors(t *testing.T) { tasks["stats"] = []Task{task} configs := &Configs{Tasks: tasks} - errs, ok := configs.Validate() + errs := configs.Validate() - if ok || len(errs) != 2 { - t.Fatalf("Configs validation failed") + if len(errs) != 2 { + t.Fatalf("expected 2 errors, got %d", len(errs)) } - if errs[0].Error() != "dunner: [stats] Image repository name cannot be empty" || errs[1].Error() != "dunner: [stats] Commands not defined for task with image " { - t.Fatalf("Configs Validation error message not as expected") + expected1 := "task 'stats': image is a required field" + expected2 := "task 'stats': command must contain at least 1 item" + if errs[0].Error() != expected1 { + t.Fatalf("expected: %s, got: %s", expected1, errs[0].Error()) + } + if errs[1].Error() != expected2 { + t.Fatalf("expected: %s, got: %s", expected2, errs[1].Error()) } } diff --git a/pkg/config/validate.go b/pkg/config/validate.go deleted file mode 100644 index a735d6d..0000000 --- a/pkg/config/validate.go +++ /dev/null @@ -1,25 +0,0 @@ -package config - -import ( - "os" - - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -func Validate(_ *cobra.Command, args []string) { - var dunnerFile = viper.GetString("DunnerTaskFile") - - configs, err := GetConfigs(dunnerFile) - if err != nil { - log.Fatal(err) - } - - errs, ok := configs.Validate() - for _, err := range errs { - log.Error(err) - } - if !ok { - os.Exit(1) - } -} diff --git a/pkg/dunner/dunner.go b/pkg/dunner/dunner.go index f6d64fa..a7c749b 100644 --- a/pkg/dunner/dunner.go +++ b/pkg/dunner/dunner.go @@ -32,11 +32,11 @@ func Do(_ *cobra.Command, args []string) { if err != nil { log.Fatal(err) } - errs, ok := configs.Validate() - for _, err := range errs { - log.Error(err) - } - if !ok { + errs := configs.Validate() + if len(errs) != 0 { + for _, err := range errs { + log.Error(err) + } os.Exit(1) } From 2445611ce744131def93aa0d7167f56a3d721944 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Tue, 14 May 2019 23:12:23 -0400 Subject: [PATCH 07/41] Add validation to check that mount dir is in right format --- pkg/config/config.go | 56 +++++++++++++++++++++++++++++++++++++-- pkg/config/config_test.go | 37 +++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index dd8e245..2bfe181 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -30,6 +30,18 @@ var ( trans ut.Translator ) +var customValidations = []struct { + tag string + translation string + validationFn func(fl validator.FieldLevel) bool +}{ + { + tag: "mountdir", + translation: "mount directory '{0}' is invalid. Use '::'", + validationFn: ValidateMountDir, + }, +} + type DirMount struct { Src string `yaml:"src"` Dest string `yaml:"dest"` @@ -43,7 +55,7 @@ type Task struct { SubDir string `yaml:"dir"` Command []string `yaml:"command" validate:"required,min=1,dive,required"` Envs []string `yaml:"envs"` - Mounts []string `yaml:"mounts"` + Mounts []string `yaml:"mounts" validate:"omitempty,dive,mountdir"` Args []string `yaml:"args"` } @@ -96,18 +108,41 @@ func initValidator() error { return name }) + // Register default translators translator := en.New() uni = ut.New(translator, translator) - var translatorFound bool trans, translatorFound = uni.GetTranslator("en") if !translatorFound { return fmt.Errorf("failed to initialize validator with translator") } en_translations.RegisterDefaultTranslations(govalidator, trans) + + // Register Custom validators and translations + for _, t := range customValidations { + err := govalidator.RegisterValidation(t.tag, t.validationFn) + if err != nil { + return fmt.Errorf("failed to register validation: %s", err.Error()) + } + err = govalidator.RegisterTranslation(t.tag, trans, registrationFunc(t.tag, t.translation), translateFunc) + if err != nil { + return fmt.Errorf("failed to register translations: %s", err.Error()) + } + } return nil } +// ValidateMountDir verifies that mount values are in proper format +func ValidateMountDir(fl validator.FieldLevel) bool { + value := fl.Field().String() + f := func(c rune) bool { return c == ':' } + mountValues := strings.FieldsFunc(value, f) + if len(mountValues) != 3 { + return false + } + return true +} + // GetConfigs reads and parses tasks from the dunner file func GetConfigs(filename string) (*Configs, error) { fileContents, err := ioutil.ReadFile(filename) @@ -232,3 +267,20 @@ func joinPathRelToHome(p string) string { } return p } + +func registrationFunc(tag string, translation string) validator.RegisterTranslationsFunc { + return func(ut ut.Translator) (err error) { + if err = ut.Add(tag, translation, true); err != nil { + return + } + return + } +} + +func translateFunc(ut ut.Translator, fe validator.FieldError) string { + t, err := ut.T(fe.Tag(), reflect.ValueOf(fe.Value()).String(), fe.Param()) + if err != nil { + return fe.(error).Error() + } + return t +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 2552234..67d0b4e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "io/ioutil" "os" "reflect" @@ -91,7 +92,7 @@ func TestConfigs_ValidateWithParseErrors(t *testing.T) { errs := configs.Validate() if len(errs) != 2 { - t.Fatalf("expected 2 errors, got %d", len(errs)) + t.Fatalf("expected 2 errors, got %d : %s", len(errs), errs) } expected1 := "task 'stats': image is a required field" @@ -104,6 +105,40 @@ func TestConfigs_ValidateWithParseErrors(t *testing.T) { } } +func TestConfigs_ValidateWithInvalidMountDirectory(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + task.Mounts = []string{"invalid_dir"} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d : %s", len(errs), errs) + } + + expected := "task 'stats': mount directory 'invalid_dir' is invalid. Use '::'" + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + +func TestConfigs_ValidateWithValidMountDirectory(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + wd, _ := os.Getwd() + task.Mounts = []string{fmt.Sprintf("%s:%s:w", wd, wd)} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if errs != nil { + t.Fatalf("expected no errors, got %s", errs) + } +} + func getSampleTask() Task { return Task{Image: "image_name", Command: []string{"node", "--version"}} } From b09ec14216d77bb3c843694c84a9b57f4b719fc4 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Wed, 15 May 2019 14:07:58 -0400 Subject: [PATCH 08/41] Add validations to check that host directory exists in mount option --- internal/util/util.go | 22 ++++++++++++ internal/util/util_test.go | 51 ++++++++++++++++++++++++++++ pkg/config/config.go | 51 ++++++++++++++++------------ pkg/config/config_test.go | 68 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 168 insertions(+), 24 deletions(-) create mode 100644 internal/util/util.go create mode 100644 internal/util/util_test.go diff --git a/internal/util/util.go b/internal/util/util.go new file mode 100644 index 0000000..a414a41 --- /dev/null +++ b/internal/util/util.go @@ -0,0 +1,22 @@ +package util + +import ( + "os" + "path" + "strings" +) + +// HomeDir is the environment variable HOME +var HomeDir = os.Getenv("HOME") + +// DirExists returns true if the given param is a valid existing directory +func DirExists(dir string) bool { + if dir[0] == '~' { + dir = path.Join(HomeDir, strings.Trim(dir, "~")) + } + src, err := os.Stat(dir) + if err != nil { + return false + } + return src.IsDir() +} diff --git a/internal/util/util_test.go b/internal/util/util_test.go new file mode 100644 index 0000000..ce7993a --- /dev/null +++ b/internal/util/util_test.go @@ -0,0 +1,51 @@ +package util + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestDirExistsSuccess(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestDir") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + exists := DirExists(tmpdir) + + if !exists { + t.Fatalf("Directory exists; but got false") + } +} + +func TestDirExistsFail(t *testing.T) { + exists := DirExists("this path is invalid") + + if exists { + t.Fatalf("Directory invalid; but got as exists") + } +} + +func TestDirExistsFailForFile(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "TestFileExists") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpfile.Name()) + + exists := DirExists(tmpfile.Name()) + + if exists { + t.Fatalf("Not a directory; but got as true") + } +} + +func TestDirExistsIfNotAbsPath(t *testing.T) { + exists := DirExists("~/invalidpathfortesting") + + if exists { + t.Fatalf("Not a directory; but got as true") + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 2bfe181..d53812d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -15,6 +15,7 @@ import ( ut "github.com/go-playground/universal-translator" "github.com/joho/godotenv" "github.com/leopardslab/Dunner/internal/logger" + "github.com/leopardslab/Dunner/internal/util" "github.com/leopardslab/Dunner/pkg/docker" "github.com/spf13/viper" "gopkg.in/go-playground/validator.v9" @@ -25,19 +26,23 @@ import ( var log = logger.Log var ( - uni *ut.UniversalTranslator - govalidator *validator.Validate - trans ut.Translator + uni *ut.UniversalTranslator + govalidator *validator.Validate + trans ut.Translator + defaultPermissionMode = "r" + validDirPermissionModes = []string{defaultPermissionMode, "wr", "rw", "w"} ) -var customValidations = []struct { +type customValidation struct { tag string translation string validationFn func(fl validator.FieldLevel) bool -}{ +} + +var customValidations = []customValidation{ { tag: "mountdir", - translation: "mount directory '{0}' is invalid. Use '::'", + translation: "mount directory '{0}' is invalid. Check format is '::' and has right permission level", validationFn: ValidateMountDir, }, } @@ -66,7 +71,7 @@ type Configs struct { // Validate validates config and returns errors. func (configs *Configs) Validate() []error { - err := initValidator() + err := initValidator(customValidations) if err != nil { return []error{err} } @@ -98,7 +103,7 @@ func formatErrors(valErrs error, taskName string) []error { return errs } -func initValidator() error { +func initValidator(customValidations []customValidation) error { govalidator = validator.New() govalidator.RegisterTagNameFunc(func(fld reflect.StructField) string { name := strings.SplitN(fld.Tag.Get("yaml"), ",", 2)[0] @@ -132,15 +137,28 @@ func initValidator() error { return nil } -// ValidateMountDir verifies that mount values are in proper format +// ValidateMountDir verifies that mount values are in proper format :: +// Format should match, is optional which is `readOnly` by default and `src` directory exists in host machine func ValidateMountDir(fl validator.FieldLevel) bool { value := fl.Field().String() f := func(c rune) bool { return c == ':' } mountValues := strings.FieldsFunc(value, f) + if len(mountValues) != 3 { + mountValues = append(mountValues, defaultPermissionMode) + } if len(mountValues) != 3 { return false } - return true + validPerm := false + for _, perm := range validDirPermissionModes { + if mountValues[2] == perm { + validPerm = true + } + } + if !validPerm { + return false + } + return util.DirExists(mountValues[0]) } // GetConfigs reads and parses tasks from the dunner file @@ -228,21 +246,10 @@ func DecodeMount(mounts []string, step *docker.Step) error { strings.Trim(strings.Trim(m, `'`), `"`), ":", ) - if len(arr) != 3 && len(arr) != 2 { - return fmt.Errorf( - `config: invalid format for mount %s`, - m, - ) - } var readOnly = true if len(arr) == 3 { if arr[2] == "wr" || arr[2] == "w" { readOnly = false - } else if arr[2] != "r" { - return fmt.Errorf( - `config: invalid format of read-write mode for mount '%s'`, - m, - ) } } src, err := filepath.Abs(joinPathRelToHome(arr[0])) @@ -263,7 +270,7 @@ func DecodeMount(mounts []string, step *docker.Step) error { func joinPathRelToHome(p string) string { if p[0] == '~' { - return path.Join(os.Getenv("HOME"), strings.Trim(p, "~")) + return path.Join(util.HomeDir, strings.Trim(p, "~")) } return p } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 67d0b4e..f005e9e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -105,7 +105,7 @@ func TestConfigs_ValidateWithParseErrors(t *testing.T) { } } -func TestConfigs_ValidateWithInvalidMountDirectory(t *testing.T) { +func TestConfigs_ValidateWithInvalidMountFormat(t *testing.T) { tasks := make(map[string][]Task, 0) task := getSampleTask() task.Mounts = []string{"invalid_dir"} @@ -118,7 +118,7 @@ func TestConfigs_ValidateWithInvalidMountDirectory(t *testing.T) { t.Fatalf("expected 1 error, got %d : %s", len(errs), errs) } - expected := "task 'stats': mount directory 'invalid_dir' is invalid. Use '::'" + expected := "task 'stats': mount directory 'invalid_dir' is invalid. Check format is '::' and has right permission level" if errs[0].Error() != expected { t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) } @@ -139,6 +139,70 @@ func TestConfigs_ValidateWithValidMountDirectory(t *testing.T) { } } +func TestConfigs_ValidateWithNoModeGiven(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + wd, _ := os.Getwd() + task.Mounts = []string{fmt.Sprintf("%s:%s", wd, wd)} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if errs != nil { + t.Fatalf("expected no errors, got %s", errs) + } +} + +func TestConfigs_ValidateWithInvalidMode(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + wd, _ := os.Getwd() + task.Mounts = []string{fmt.Sprintf("%s:%s:ab", wd, wd)} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + expected := fmt.Sprintf("task 'stats': mount directory '%s' is invalid. Check format is '::' and has right permission level", task.Mounts[0]) + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + +func TestConfigs_ValidateWithInvalidMountDirectory(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + task.Mounts = []string{"blah:foo:w"} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d : %s", len(errs), errs) + } + + expected := "task 'stats': mount directory 'blah:foo:w' is invalid. Check format is '::' and has right permission level" + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + func getSampleTask() Task { return Task{Image: "image_name", Command: []string{"node", "--version"}} } + +func TestInitValidatorForNilTranslation(t *testing.T) { + vals := []customValidation{{tag: "foo", translation: "", validationFn: nil}} + + err := initValidator(vals) + + expected := "failed to register validation: Function cannot be empty" + if err == nil { + t.Fatalf("expected %s, got %s", expected, err) + } + if err.Error() != expected { + t.Fatalf("expected %s, got %s", expected, err.Error()) + } +} From d148bbf47ef760c949896a2b5ce7ab3a2d9ae60b Mon Sep 17 00:00:00 2001 From: Milindu Sanoj Kumarage Date: Thu, 21 Mar 2019 23:43:11 +0530 Subject: [PATCH 09/41] [#59] Build a Snap package Refer https://snapcraft.io/first-snap/golang --- snapcraft.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 snapcraft.yaml diff --git a/snapcraft.yaml b/snapcraft.yaml new file mode 100644 index 0000000..f3daa74 --- /dev/null +++ b/snapcraft.yaml @@ -0,0 +1,21 @@ +name: dunner +version: git +summary: A Docker based tas runner tool +description: | + Dunner is a task runner tool like Grunt but used Docker images like CircleCI do. | + You can define tasks and steps of the tasks in your `.dunner.yaml` file and then run these steps with `Dunner do taskname` +confinement: devmode +base: core18 + +parts: + dunner: + plugin: go + go-importpath: github.com/leopardslab/Dunner + source: . + source-type: git + build-packages: + - gcc + +apps: + dunner: + command: Dunner From be0fa74df4d3c65df7647300fd714a068030a4a8 Mon Sep 17 00:00:00 2001 From: Milindu Sanoj Kumarage Date: Thu, 21 Mar 2019 23:43:11 +0530 Subject: [PATCH 10/41] [#59] Build a Snap package Refer https://snapcraft.io/first-snap/golang --- snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index f3daa74..6a30d31 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -1,6 +1,6 @@ name: dunner version: git -summary: A Docker based tas runner tool +summary: A Docker based task runner tool description: | Dunner is a task runner tool like Grunt but used Docker images like CircleCI do. | You can define tasks and steps of the tasks in your `.dunner.yaml` file and then run these steps with `Dunner do taskname` From c9a8727e9cef6568adf4742a455bbb5ef65e667d Mon Sep 17 00:00:00 2001 From: Milindu Sanoj Kumarage Date: Fri, 22 Feb 2019 01:05:44 +0530 Subject: [PATCH 11/41] [#22] Improve Makefile, Add better versions cmd --- Makefile | 22 +++++++++++++++++----- cmd/version.go | 4 ++-- main.go | 4 ++++ pkg/global/global.go | 5 +++++ 4 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 pkg/global/global.go diff --git a/Makefile b/Makefile index 58f61dc..dbb7acc 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,15 @@ -#Go parameters +.PHONY: all install test build clean + +SHA=$(shell git rev-list HEAD --max-count=1 --abbrev-commit) +TAG?=$(shell git tag -l --contains HEAD) +VERSION=$(TAG) + +ifeq ($(VERSION),) +VERSION := latest +endif + +#Go parameters GOCMD=go GOINSTALL=$(GOCMD) install GOTEST=$(GOCMD) test @@ -8,15 +18,17 @@ DEP=dep all : def -def : - @$(GOINSTALL) -ldflags '-s' +all: build -install: +install: @$(DEP) ensure -test: +test: install @$(GOTEST) -v ./... +build: install + @$(GOINSTALL) -ldflags "-X main.version=$(VERSION)-$(SHA) -s" + clean: rm -rf * diff --git a/cmd/version.go b/cmd/version.go index 0802c2e..d65c46d 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -2,8 +2,8 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" + G "github.com/leopardslab/Dunner/pkg/global" ) func init() { @@ -15,6 +15,6 @@ var versionCmd = &cobra.Command{ Short: "Print the version number of Dunner", Long: `All software has versions. This is Dunners's`, Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v1.0.0") + fmt.Println(G.VERSION) }, } diff --git a/main.go b/main.go index 225014b..0469e03 100644 --- a/main.go +++ b/main.go @@ -2,8 +2,12 @@ package main import ( "github.com/leopardslab/Dunner/cmd" + G "github.com/leopardslab/Dunner/pkg/global" ) +var version string = "" + func main() { + G.VERSION = version cmd.Execute() } diff --git a/pkg/global/global.go b/pkg/global/global.go new file mode 100644 index 0000000..6cfa267 --- /dev/null +++ b/pkg/global/global.go @@ -0,0 +1,5 @@ +package global + +var ( + VERSION string +) From e000965cbef500306c821c4d9e41b27bd472ffbe Mon Sep 17 00:00:00 2001 From: Milindu Sanoj Kumarage Date: Fri, 22 Mar 2019 00:21:05 +0530 Subject: [PATCH 12/41] [#22] Add lint fixes --- main.go | 2 +- pkg/global/global.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 0469e03..9f59cd6 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,7 @@ import ( G "github.com/leopardslab/Dunner/pkg/global" ) -var version string = "" +var version string func main() { G.VERSION = version diff --git a/pkg/global/global.go b/pkg/global/global.go index 6cfa267..94caf27 100644 --- a/pkg/global/global.go +++ b/pkg/global/global.go @@ -1,5 +1,6 @@ package global +// VERSION is to hold the Dunner version var ( VERSION string ) From 1fc564503ba57f01da9e747d617fe5c4b1696f60 Mon Sep 17 00:00:00 2001 From: Ayush Jain Date: Mon, 25 Mar 2019 15:28:51 +0530 Subject: [PATCH 13/41] Add support for executing multiple commands --- .dunner.yaml | 18 ++++++----- pkg/config/config.go | 14 ++++----- pkg/docker/docker.go | 74 +++++++++++++++++++++++++++++++++++++------- pkg/dunner/dunner.go | 67 +++++++++++++++++++-------------------- 4 files changed, 114 insertions(+), 59 deletions(-) diff --git a/.dunner.yaml b/.dunner.yaml index c64c3a1..14fd7d7 100644 --- a/.dunner.yaml +++ b/.dunner.yaml @@ -1,15 +1,18 @@ build: - image: node:10.15.0 - command: ["node", "--version"] - - image: node:10.15.0 - command: ["npm", "--version"] + commands: + - ["node", "--version"] + - ["npm", "--version"] - image: alpine dir: pkg - command: ["pwd"] + commands: + - ["pwd"] - image: alpine - command: ["apk", "update"] + commands: + - ["apk", "update"] - image: alpine - command: ["printenv"] + commands: + - ["printenv"] envs: - PERM=775 - ID=dunner @@ -19,7 +22,8 @@ build: - '/root' show: - image: alpine - command: ["ls", "$1"] + commands: + - ["ls", "$1"] mounts: - '~/Downloads:/root/down' - ~/Pictures:/root/pics:wr diff --git a/pkg/config/config.go b/pkg/config/config.go index aeaa83f..b475bca 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -27,13 +27,13 @@ type DirMount struct { // Task describes a single task to be run in a docker container type Task struct { - Name string `yaml:"name"` - Image string `yaml:"image"` - SubDir string `yaml:"dir"` - Command []string `yaml:"command"` - Envs []string `yaml:"envs"` - Mounts []string `yaml:"mounts"` - Args []string `yaml:"args"` + Name string `yaml:"name"` + Image string `yaml:"image"` + SubDir string `yaml:"dir"` + Commands [][]string `yaml:"commands"` + Envs []string `yaml:"envs"` + Mounts []string `yaml:"mounts"` + Args []string `yaml:"args"` } // Configs describes the parsed information from the dunner file diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index a7dae36..b4abdc9 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -1,17 +1,21 @@ package docker import ( + "bytes" "context" - "io" + "fmt" "io/ioutil" "os" "path/filepath" + "strings" + "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" "github.com/docker/docker/pkg/jsonmessage" + "github.com/docker/docker/pkg/stdcopy" "github.com/docker/docker/pkg/term" "github.com/leopardslab/dunner/internal/logger" "github.com/spf13/viper" @@ -24,7 +28,7 @@ type Step struct { Task string Name string Image string - Command []string + Commands [][]string Env []string WorkDir string Volumes map[string]string @@ -32,8 +36,15 @@ type Step struct { Args []string } +// Result stores the output of commands run using docker exec +type Result struct { + Command string + Output string + Error string +} + // Exec method is used to execute the task described in the corresponding step -func (step Step) Exec() (*io.ReadCloser, error) { +func (step Step) Exec() (*[]Result, error) { var ( hostMountFilepath = "./" @@ -87,7 +98,7 @@ func (step Step) Exec() (*io.ReadCloser, error) { ctx, &container.Config{ Image: step.Image, - Cmd: step.Command, + Cmd: []string{"tail", "-f", "/dev/null"}, Env: step.Env, WorkingDir: containerWorkingDir, }, @@ -103,27 +114,66 @@ func (step Step) Exec() (*io.ReadCloser, error) { log.Fatal(err) } + if len(resp.Warnings) > 0 { + for warning := range resp.Warnings { + log.Warn(warning) + } + } + if err = cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { log.Fatal(err) } - statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) - select { - case err = <-errCh: + defer func() { + dur, err := time.ParseDuration("-1ns") // Negative duration means no force termination + if err != nil { + log.Fatal(err) + } + if err = cli.ContainerStop(ctx, resp.ID, &dur); err != nil { + log.Fatal(err) + } + }() + + var results []Result + for _, cmd := range step.Commands { + r, err := runCmd(ctx, cli, resp.ID, cmd) if err != nil { log.Fatal(err) } - case <-statusCh: + results = append(results, *r) } - out, err = cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ - ShowStdout: true, - ShowStderr: true, + return &results, nil +} + +func runCmd(ctx context.Context, cli *client.Client, containerID string, command []string) (*Result, error) { + if len(command) == 0 { + return nil, fmt.Errorf(`config: Command cannot be empty`) + } + + exec, err := cli.ContainerExecCreate(ctx, containerID, types.ExecConfig{ + Cmd: command, + AttachStdout: true, + AttachStderr: true, }) if err != nil { log.Fatal(err) } - return &out, nil + resp, err := cli.ContainerExecAttach(ctx, exec.ID, types.ExecStartCheck{}) + if err != nil { + log.Fatal(err) + } + defer resp.Close() + var out, errOut bytes.Buffer + if _, err = stdcopy.StdCopy(&out, &errOut, resp.Reader); err != nil { + log.Fatal(err) + } + var result = Result{ + Command: strings.Join(command, " "), + Output: out.String(), + Error: errOut.String(), + } + return &result, nil } diff --git a/pkg/dunner/dunner.go b/pkg/dunner/dunner.go index c30e2ce..0a334cf 100644 --- a/pkg/dunner/dunner.go +++ b/pkg/dunner/dunner.go @@ -1,13 +1,12 @@ package dunner import ( - "os" + "fmt" "regexp" "strconv" "strings" "sync" - "github.com/docker/docker/pkg/stdcopy" "github.com/leopardslab/dunner/internal/logger" "github.com/leopardslab/dunner/pkg/config" "github.com/leopardslab/dunner/pkg/docker" @@ -44,13 +43,13 @@ func execTask(configs *config.Configs, taskName string, args []string) { wg.Add(1) } step := docker.Step{ - Task: taskName, - Name: stepDefinition.Name, - Image: stepDefinition.Image, - Command: stepDefinition.Command, - Env: stepDefinition.Envs, - WorkDir: stepDefinition.SubDir, - Args: stepDefinition.Args, + Task: taskName, + Name: stepDefinition.Name, + Image: stepDefinition.Image, + Commands: stepDefinition.Commands, + Env: stepDefinition.Envs, + WorkDir: stepDefinition.SubDir, + Args: stepDefinition.Args, } if err := config.DecodeMount(stepDefinition.Mounts, &step); err != nil { @@ -95,38 +94,40 @@ func process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args [ log.Fatalf(`dunner: image repository name cannot be empty`) } - pout, err := (*s).Exec() + results, err := (*s).Exec() if err != nil { log.Fatal(err) } - log.Infof( - "Running task '%+v' on '%+v' Docker with command '%+v'", - s.Task, - s.Image, - strings.Join(s.Command, " "), - ) - - if _, err = stdcopy.StdCopy(os.Stdout, os.Stderr, *pout); err != nil { - log.Fatal(err) - } - - if err = (*pout).Close(); err != nil { - log.Fatal(err) + for _, res := range *results { + log.Infof( + "Running task '%+v' on '%+v' Docker with command '%+v'", + s.Task, + s.Image, + res.Command, + ) + if res.Output != "" { + fmt.Printf(`OUT: %s`, res.Output) + } + if res.Error != "" { + fmt.Printf(`ERR: %s`, res.Error) + } } } func passArgs(s *docker.Step, args *[]string) error { - for i, subStr := range s.Command { - regex := regexp.MustCompile(`\$[1-9][0-9]*`) - subStr = regex.ReplaceAllStringFunc(subStr, func(str string) string { - j, err := strconv.Atoi(strings.Trim(str, "$")) - if err != nil { - log.Fatal(err) - } - return (*args)[j-1] - }) - s.Command[i] = subStr + for i, cmd := range s.Commands { + for j, subStr := range cmd { + regex := regexp.MustCompile(`\$[1-9][0-9]*`) + subStr = regex.ReplaceAllStringFunc(subStr, func(str string) string { + j, err := strconv.Atoi(strings.Trim(str, "$")) + if err != nil { + log.Fatal(err) + } + return (*args)[j-1] + }) + s.Commands[i][j] = subStr + } } return nil } From db7f92f649a148b1e06478e104258285daa263d7 Mon Sep 17 00:00:00 2001 From: Ayush Jain Date: Mon, 25 Mar 2019 15:29:14 +0530 Subject: [PATCH 14/41] Update unit tests for config and docker modules --- pkg/config/config_test.go | 12 +++++++----- pkg/docker/docker_test.go | 22 ++++++++-------------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 0058078..222b7d5 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -13,7 +13,9 @@ func TestGetConfigs(t *testing.T) { var content = []byte(` test: - image: node - command: ["node", "--version"] + commands: + - ["node", "--version"] + - ["npm", "--version"] envs: - MYVAR=MYVAL`) @@ -38,10 +40,10 @@ test: } var task = Task{ - Name: "", - Image: "node", - Command: []string{"node", "--version"}, - Envs: []string{"MYVAR=MYVAL"}, + Name: "", + Image: "node", + Commands: [][]string{{"node", "--version"}, {"npm", "--version"}}, + Envs: []string{"MYVAR=MYVAL"}, } var tasks = make(map[string][]Task) tasks["test"] = []Task{task} diff --git a/pkg/docker/docker_test.go b/pkg/docker/docker_test.go index a8fd7d9..a804ee4 100644 --- a/pkg/docker/docker_test.go +++ b/pkg/docker/docker_test.go @@ -1,7 +1,6 @@ package docker import ( - "bytes" "strings" "testing" ) @@ -11,25 +10,20 @@ func TestStep_Do(t *testing.T) { var testNodeVersion = "10.15.0" step := &Step{ - Task: "test", - Name: "node", - Image: "node:" + testNodeVersion, - Command: []string{"node", "--version"}, - Env: nil, - Volumes: nil, + Task: "test", + Name: "node", + Image: "node:" + testNodeVersion, + Commands: [][]string{{"node", "--version"}}, + Env: nil, + Volumes: nil, } - pout, err := step.Exec() - if err != nil { - t.Error(err) - } - buffer := new(bytes.Buffer) - _, err = buffer.ReadFrom(*pout) + results, err := step.Exec() if err != nil { t.Error(err) } - strOut := buffer.String() + strOut := (*results)[0].Output var result = strings.Trim(strings.Split(strOut, "v")[1], "\n") if result != testNodeVersion { t.Fatalf("Detected version of node container: '%s'; Expected output: '%s'", result, testNodeVersion) From 3566857d7a9528059bb4bd1c741a668c6d5b2972 Mon Sep 17 00:00:00 2001 From: Ayush Jain Date: Mon, 25 Mar 2019 15:29:38 +0530 Subject: [PATCH 15/41] Update README.md --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 25636ee..807be79 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,14 @@ Example `.dunner.yaml` ```yaml deploy: - image: 'emeraldsquad/sonar-scanner' - command: ['sonar', 'scan'] + commands: + - ['sonar', 'scan'] - image: 'golang' - command: ['go', 'install'] + commands: + - ['go', 'install'] - image: 'mesosphere/aws-cli' - command: ['aws', 'elasticbeanstalk update-application --application-name myapp'] + commands: + - ['aws', 'elasticbeanstalk update-application --application-name myapp'] envs: - AWS_ACCESS_KEY_ID=`$AWS_KEY` - AWS_SECRET_ACCESS_KEY=`$AWS_SECRET` @@ -23,7 +26,8 @@ deploy: args: 'prod' status: - image: 'mesosphere/aws-cli' - command: ['aws', 'elasticbeanstalk describe-events --environment-name $1'] + commands: + - ['aws', 'elasticbeanstalk describe-events --environment-name $1'] # This uses args passed to the task, `$1` means first arg envs: - AWS_ACCESS_KEY_ID=`$AWS_KEY` From 6545aeadfdc1b856acc1da6c6037b7a71ba6dfd5 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Wed, 15 May 2019 16:25:41 -0400 Subject: [PATCH 16/41] Remove changes not part of this feature --- Makefile | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Makefile b/Makefile index dbb7acc..188d169 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,6 @@ GOCMD=go GOINSTALL=$(GOCMD) install GOTEST=$(GOCMD) test DEP=dep -.PHONY : all install vet fmt test - -all : def all: build @@ -31,11 +28,3 @@ build: install clean: rm -rf * - -vet: - @echo "=== go vet ===" - @go vet ./... - -fmt: - @echo "=== go fmt ===" - @go fmt ./... From 3f1f0ab25d75a1dc7b683c8b189e6b1615583579 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Wed, 15 May 2019 16:27:34 -0400 Subject: [PATCH 17/41] Fix case in dunner link --- Makefile | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 188d169..041a287 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ endif GOCMD=go GOINSTALL=$(GOCMD) install GOTEST=$(GOCMD) test -DEP=dep +DEP=dep all: build diff --git a/README.md b/README.md index 6a68357..4ad2556 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ This work is still in progress. See the development plan. - [ ] Ability to install as a RPM package - [ ] Ability to install as a Brew package -### [`v2.0`](https://github.com/leopardslab/Dunner/milestone/3) +### [`v2.0`](https://github.com/leopardslab/dunner/milestone/3) - [ ] Ability to Dry Run - [ ] Ability to verfiy the `.dunner.yaml` file - [ ] Ability to define multiple commands for the same step From 0dd5d1e8b42a2ae42458ce6427c9436348d91953 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Wed, 15 May 2019 17:59:12 -0400 Subject: [PATCH 18/41] Pass configs in context for reference to other tasks --- internal/util/util.go | 2 +- pkg/config/config.go | 18 ++++++++++++------ pkg/config/config_test.go | 6 +++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/internal/util/util.go b/internal/util/util.go index a414a41..8eb8338 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -11,7 +11,7 @@ var HomeDir = os.Getenv("HOME") // DirExists returns true if the given param is a valid existing directory func DirExists(dir string) bool { - if dir[0] == '~' { + if strings.HasPrefix(dir, "~") { dir = path.Join(HomeDir, strings.Trim(dir, "~")) } src, err := os.Stat(dir) diff --git a/pkg/config/config.go b/pkg/config/config.go index 6dedabd..24d770c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,6 +1,7 @@ package config import ( + "context" "fmt" "io/ioutil" "os" @@ -36,7 +37,7 @@ var ( type customValidation struct { tag string translation string - validationFn func(fl validator.FieldLevel) bool + validationFn func(context.Context, validator.FieldLevel) bool } var customValidations = []customValidation{ @@ -58,9 +59,9 @@ type Task struct { Name string `yaml:"name"` Image string `yaml:"image" validate:"required"` SubDir string `yaml:"dir"` - Command []string `yaml:"command" validate:"required,min=1,dive,required"` + Command []string `yaml:"command" validate:"omitempty,dive,required"` Envs []string `yaml:"envs"` - Mounts []string `yaml:"mounts" validate:"omitempty,dive,mountdir"` + Mounts []string `yaml:"mounts" validate:"omitempty,dive,min=1,mountdir"` Args []string `yaml:"args"` } @@ -69,6 +70,10 @@ type Configs struct { Tasks map[string][]Task `validate:"required,min=1,dive,keys,required,endkeys,required,min=1,required"` } +type contextKey string + +var configsKey = contextKey("dunnerConfigs") + // Validate validates config and returns errors. func (configs *Configs) Validate() []error { err := initValidator(customValidations) @@ -77,10 +82,11 @@ func (configs *Configs) Validate() []error { } valErrs := govalidator.Struct(configs) errs := formatErrors(valErrs, "") + ctx := context.WithValue(context.Background(), configsKey, configs) // Each task is validated separately so that task name can be added in error messages for taskName, tasks := range configs.Tasks { - taskValErrs := govalidator.Var(tasks, "dive") + taskValErrs := govalidator.VarCtx(ctx, tasks, "dive") errs = append(errs, formatErrors(taskValErrs, taskName)...) } return errs @@ -126,7 +132,7 @@ func initValidator(customValidations []customValidation) error { // Register Custom validators and translations for _, t := range customValidations { - err := govalidator.RegisterValidation(t.tag, t.validationFn) + err := govalidator.RegisterValidationCtx(t.tag, t.validationFn) if err != nil { return fmt.Errorf("failed to register validation: %s", err.Error()) } @@ -140,7 +146,7 @@ func initValidator(customValidations []customValidation) error { // ValidateMountDir verifies that mount values are in proper format :: // Format should match, is optional which is `readOnly` by default and `src` directory exists in host machine -func ValidateMountDir(fl validator.FieldLevel) bool { +func ValidateMountDir(ctx context.Context, fl validator.FieldLevel) bool { value := fl.Field().String() f := func(c rune) bool { return c == ':' } mountValues := strings.FieldsFunc(value, f) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index f005e9e..c2727a8 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -83,9 +83,9 @@ func TestConfigs_ValidateWithNoTasks(t *testing.T) { } } -func TestConfigs_ValidateWithParseErrors(t *testing.T) { +func TestConfigs_ValidateWithEmptyImageAndCommand(t *testing.T) { tasks := make(map[string][]Task, 0) - task := Task{Image: "", Command: []string{}} + task := Task{Image: "", Command: []string{""}} tasks["stats"] = []Task{task} configs := &Configs{Tasks: tasks} @@ -96,7 +96,7 @@ func TestConfigs_ValidateWithParseErrors(t *testing.T) { } expected1 := "task 'stats': image is a required field" - expected2 := "task 'stats': command must contain at least 1 item" + expected2 := "task 'stats': command[0] is a required field" if errs[0].Error() != expected1 { t.Fatalf("expected: %s, got: %s", expected1, errs[0].Error()) } From 329dc9c20c7c52b22b4f7ae1bc24a3bee154bb55 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Wed, 15 May 2019 19:59:19 -0400 Subject: [PATCH 19/41] Add validation status message --- cmd/validate.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/validate.go b/cmd/validate.go index d4e5957..893a906 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -1,6 +1,7 @@ package cmd import ( + "fmt" "os" "github.com/leopardslab/dunner/pkg/config" @@ -31,9 +32,11 @@ func Validate(_ *cobra.Command, args []string) { errs := configs.Validate() if len(errs) != 0 { + fmt.Println("Validation failed with following errors:") for _, err := range errs { - log.Error(err) + fmt.Println(err.Error()) } os.Exit(1) } + fmt.Println("Validation successful!") } From d386ee78a583cd914e6c9af6e3d63c015d16f369 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Thu, 16 May 2019 12:40:05 -0400 Subject: [PATCH 20/41] Integrate with codecov to get code coverage for master --- .travis.yml | 6 +++++- README.md | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a5a610a..e835ef5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,8 @@ before_install: install: - dep ensure -v -script: go test -v ./... \ No newline at end of file +script: + - go test ./... -race -coverprofile=coverage.txt -covermode=atomic + +after_success: + - bash <(curl -s https://codecov.io/bash) \ No newline at end of file diff --git a/README.md b/README.md index 25636ee..20598b0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Dunner [![Codacy Badge](https://api.codacy.com/project/badge/Grade/b2275e331d2745dc9527d45efbbf2da2)](https://app.codacy.com/app/Leopardslab/dunner?utm_source=github.com&utm_medium=referral&utm_content=leopardslab/dunner&utm_campaign=Badge_Grade_Dashboard) +[![Codecov branch](https://img.shields.io/codecov/c/github/leopardslab/dunner/master.svg?style=for-the-badge)](https://codecov.io/gh/leopardslab/dunner) Dunner is a task runner tool like Grunt but uses Docker images like CircleCI do. You can define tasks and steps of the tasks in your `.dunner.yaml` file and then run these steps with `Dunner do taskname` From f4692ca061a6920c538250f5e165a4855ae37736 Mon Sep 17 00:00:00 2001 From: Ayush Jain Date: Thu, 16 May 2019 23:38:39 +0530 Subject: [PATCH 21/41] Resolve settings intialisation bug --- internal/settings/settings.go | 5 ++++- main.go | 2 ++ pkg/docker/docker_test.go | 5 +++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 0f246b0..4d13197 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -4,7 +4,10 @@ import ( "github.com/spf13/viper" ) -func init() { +// Init function initializes the default settings for dunner +// These settings can tweaked using appropriate environment variables, or +// defining the configuration in conf present in the appropriate config files +func Init() { // Settings file viper.SetConfigName("settings") viper.SetConfigType("yaml") diff --git a/main.go b/main.go index bc95223..0bc36a2 100644 --- a/main.go +++ b/main.go @@ -2,12 +2,14 @@ package main import ( "github.com/leopardslab/dunner/cmd" + "github.com/leopardslab/dunner/internal/settings" G "github.com/leopardslab/dunner/pkg/global" ) var version string func main() { + settings.Init() G.VERSION = version cmd.Execute() } diff --git a/pkg/docker/docker_test.go b/pkg/docker/docker_test.go index a804ee4..886b4bb 100644 --- a/pkg/docker/docker_test.go +++ b/pkg/docker/docker_test.go @@ -3,12 +3,13 @@ package docker import ( "strings" "testing" + + "github.com/leopardslab/dunner/internal/settings" ) func TestStep_Do(t *testing.T) { - + settings.Init() var testNodeVersion = "10.15.0" - step := &Step{ Task: "test", Name: "node", From 5c87fc0e754f2023410f1a0d1bbdb24998d2e55c Mon Sep 17 00:00:00 2001 From: Ayush Jain Date: Fri, 17 May 2019 00:14:12 +0530 Subject: [PATCH 22/41] [#66] Auto detect Docker API version --- cmd/root.go | 7 +++---- internal/settings/settings.go | 3 --- pkg/docker/docker.go | 6 ++---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 261c853..49f2390 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "os" @@ -18,13 +19,11 @@ var rootCmd = &cobra.Command{ Long: `You can define a set of commands and on what Docker images these commands should run as steps. A task has many steps. Then you can run these tasks with 'dunner do nameoftask'`, Run: func(cmd *cobra.Command, args []string) { - _, err := client.NewClientWithOpts( - client.FromEnv, - client.WithVersion(viper.GetString("DockerAPIVersion")), - ) + cli, err := client.NewClientWithOpts(client.FromEnv) if err != nil { log.Fatal(err) } + cli.NegotiateAPIVersion(context.Background()) fmt.Println("Dunner running!") }, diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 0f246b0..1c10dc8 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -26,7 +26,4 @@ func init() { // Modes viper.SetDefault("Async", false) viper.SetDefault("Verbose", false) - - // Constants - viper.SetDefault("DockerAPIVersion", "1.39") } diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index a7dae36..fefefb0 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -42,13 +42,11 @@ func (step Step) Exec() (*io.ReadCloser, error) { ) ctx := context.Background() - cli, err := client.NewClientWithOpts( - client.FromEnv, - client.WithVersion(viper.GetString("DockerAPIVersion")), - ) + cli, err := client.NewClientWithOpts(client.FromEnv) if err != nil { log.Fatal(err) } + cli.NegotiateAPIVersion(ctx) path, err := filepath.Abs(hostMountFilepath) if err != nil { From e500224213f8e4fb001ea4dd23b1236ed6aa13fe Mon Sep 17 00:00:00 2001 From: apoorvam Date: Thu, 16 May 2019 16:42:21 -0400 Subject: [PATCH 23/41] Integrate with goreleaser to release dunner as binary, tar, deb, rpm, snap for all os/arch --- .gitignore | 3 +++ .goreleaser.yml | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ .travis.yml | 24 ++++++++++++++++++- Makefile | 2 ++ 4 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 .goreleaser.yml diff --git a/.gitignore b/.gitignore index 540eee7..765097a 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ # Environment file .env + +# Packages +dist/* diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..a476d3e --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,63 @@ +# Goreleaser documentation at http://goreleaser.com +project_name: dunner +builds: +- env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - 386 + - amd64 + - arm + - arm64 +archives: +- replacements: + 386: i386 + amd64: x86_64 +checksum: + name_template: '{{ .ProjectName }}_checksums.txt' +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + - Merge pull request + - Merge branch + - Update readme +snapshot: + name_template: "{{.ProjectName}}_{{.Tag}}" + +brew: + github: + owner: leopardslab + name: homebrew-dunner + folder: Formula + homepage: https://github.com/leopardslab/Dunner + description: A Docker based task runner tool + test: | + system "#{bin}/dunner version" + +nfpm: + name_template: '{{ .ProjectName }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' + homepage: https://github.com/leopardslab/Dunner + description: A Docker based task runner tool + license: MIT + formats: + - deb + - rpm + dependencies: + - git + recommends: + - rpm +snapcraft: + name_template: '{{ .ProjectName }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' + summary: A Docker based task runner tool + description: | + Dunner is a task runner tool like Grunt but used Docker images like CircleCI do. | + You can define tasks and steps of the tasks in your `.dunner.yaml` file and then run these steps with `Dunner do taskname` + grade: stable + confinement: devmode + publish: false \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index a5a610a..e0b100d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,13 @@ env: services: - docker +addons: + apt: + packages: + - rpm + - snapd +env: + - PATH=/snap/bin:$PATH before_install: - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep @@ -16,5 +23,20 @@ before_install: install: - dep ensure -v + - sudo snap install snapcraft --classic + +script: + - make ci + +after_success: + - test -n "$TRAVIS_TAG" && snapcraft login --with snap.login -script: go test -v ./... \ No newline at end of file +deploy: +- provider: script + skip_cleanup: true + script: curl -sL https://git.io/goreleaser | bash + verbose: true + on: + tags: true + condition: $TRAVIS_OS_NAME = linux + master: true \ No newline at end of file diff --git a/Makefile b/Makefile index 041a287..16c563e 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,5 @@ build: install clean: rm -rf * + +ci: build test \ No newline at end of file From b95b385f735436e68dbe32bc20f1bc03b7cc8508 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Thu, 16 May 2019 17:19:36 -0400 Subject: [PATCH 24/41] Print dep version to confirm installation --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e0b100d..5ab6557 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ env: before_install: - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep - chmod +x $GOPATH/bin/dep + - dep version install: - dep ensure -v From 3ab04072d1d4e8a620e8a6d86691e79da511254b Mon Sep 17 00:00:00 2001 From: apoorvam Date: Thu, 16 May 2019 17:22:32 -0400 Subject: [PATCH 25/41] add verbose mode to debug --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5ab6557..9643faa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,8 +18,9 @@ env: - PATH=/snap/bin:$PATH before_install: - - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep + - curl -L -s -v https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep - chmod +x $GOPATH/bin/dep + - ls -al $GOPATH/bin - dep version install: From 105f8f874d77d54cc786b849662557e49b75729a Mon Sep 17 00:00:00 2001 From: apoorvam Date: Thu, 16 May 2019 17:41:48 -0400 Subject: [PATCH 26/41] use dep from gopath --- .travis.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9643faa..f8e4240 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ go: env: - DEP_VERSION="0.5.0" + - PATH=/snap/bin:$PATH services: - docker @@ -14,17 +15,14 @@ addons: packages: - rpm - snapd -env: - - PATH=/snap/bin:$PATH before_install: - curl -L -s -v https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep - chmod +x $GOPATH/bin/dep - - ls -al $GOPATH/bin - dep version install: - - dep ensure -v + - $GOPATH/bin/dep ensure -v - sudo snap install snapcraft --classic script: @@ -41,4 +39,4 @@ deploy: on: tags: true condition: $TRAVIS_OS_NAME = linux - master: true \ No newline at end of file + master: true From 375d971943e64b3d7634a4bd0999c0d65c952208 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Thu, 16 May 2019 17:45:01 -0400 Subject: [PATCH 27/41] debug ci - use dep from gopath --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f8e4240..c9c136b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ addons: before_install: - curl -L -s -v https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep - chmod +x $GOPATH/bin/dep - - dep version + - $GOPATH/bin/dep version install: - $GOPATH/bin/dep ensure -v From c3dd2c476e7eb42a8775ef382223667a244984cd Mon Sep 17 00:00:00 2001 From: apoorvam Date: Thu, 16 May 2019 17:52:18 -0400 Subject: [PATCH 28/41] remove all debug commands in ci file --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c9c136b..69a4912 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,9 @@ go: - master env: - - DEP_VERSION="0.5.0" - - PATH=/snap/bin:$PATH + global: + - DEP_VERSION="0.5.0" + - PATH=/snap/bin:$PATH services: - docker @@ -17,9 +18,8 @@ addons: - snapd before_install: - - curl -L -s -v https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep + - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep - chmod +x $GOPATH/bin/dep - - $GOPATH/bin/dep version install: - $GOPATH/bin/dep ensure -v From 966c4f4461dffe50fa2aef6ea0bc06781c839c55 Mon Sep 17 00:00:00 2001 From: Ayush Jain Date: Sun, 19 May 2019 12:15:21 +0530 Subject: [PATCH 29/41] Add support for both 'command' and 'commands' --- .dunner.yaml | 3 +-- cmd/version.go | 3 ++- pkg/config/config.go | 1 + pkg/docker/docker.go | 48 ++++++++++++++++++++++++++++++++++++-------- pkg/dunner/dunner.go | 1 + 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/.dunner.yaml b/.dunner.yaml index 14fd7d7..86badf1 100644 --- a/.dunner.yaml +++ b/.dunner.yaml @@ -5,8 +5,7 @@ build: - ["npm", "--version"] - image: alpine dir: pkg - commands: - - ["pwd"] + command: ["pwd"] - image: alpine commands: - ["apk", "update"] diff --git a/cmd/version.go b/cmd/version.go index 07017ec..1be06fa 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -2,8 +2,9 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" + G "github.com/leopardslab/dunner/pkg/global" + "github.com/spf13/cobra" ) func init() { diff --git a/pkg/config/config.go b/pkg/config/config.go index b475bca..454255a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -30,6 +30,7 @@ type Task struct { Name string `yaml:"name"` Image string `yaml:"image"` SubDir string `yaml:"dir"` + Command []string `yaml:"command"` Commands [][]string `yaml:"commands"` Envs []string `yaml:"envs"` Mounts []string `yaml:"mounts"` diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index b4abdc9..329811b 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -28,6 +29,7 @@ type Step struct { Task string Name string Image string + Command []string Commands [][]string Env []string WorkDir string @@ -50,6 +52,8 @@ func (step Step) Exec() (*[]Result, error) { hostMountFilepath = "./" containerDefaultWorkingDir = "/dunner" hostMountTarget = "/dunner" + defaultCommand = []string{"tail", "-f", "/dev/null"} + multipleCommands = false ) ctx := context.Background() @@ -67,7 +71,6 @@ func (step Step) Exec() (*[]Result, error) { } log.Infof("Pulling an image: '%s'", step.Image) - out, err := cli.ImagePull(ctx, step.Image, types.ImagePullOptions{}) if err != nil { log.Fatal(err) @@ -94,11 +97,15 @@ func (step Step) Exec() (*[]Result, error) { containerWorkingDir = filepath.Join(hostMountTarget, step.WorkDir) } + multipleCommands = len(step.Commands) > 0 + if !multipleCommands { + defaultCommand = step.Command + } resp, err := cli.ContainerCreate( ctx, &container.Config{ Image: step.Image, - Cmd: []string{"tail", "-f", "/dev/null"}, + Cmd: defaultCommand, Env: step.Env, WorkingDir: containerWorkingDir, }, @@ -135,14 +142,34 @@ func (step Step) Exec() (*[]Result, error) { }() var results []Result - for _, cmd := range step.Commands { - r, err := runCmd(ctx, cli, resp.ID, cmd) + if multipleCommands { + for _, cmd := range step.Commands { + r, err := runCmd(ctx, cli, resp.ID, cmd) + if err != nil { + log.Fatal(err) + } + results = append(results, *r) + } + } else { + statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) + select { + case err = <-errCh: + if err != nil { + log.Fatal(err) + } + case <-statusCh: + } + + out, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) if err != nil { log.Fatal(err) } - results = append(results, *r) - } + results = []Result{*extractResult(out, step.Command)} + } return &results, nil } @@ -166,8 +193,13 @@ func runCmd(ctx context.Context, cli *client.Client, containerID string, command } defer resp.Close() + return extractResult(resp.Reader, command), nil +} + +func extractResult(reader io.Reader, command []string) *Result { + var out, errOut bytes.Buffer - if _, err = stdcopy.StdCopy(&out, &errOut, resp.Reader); err != nil { + if _, err := stdcopy.StdCopy(&out, &errOut, reader); err != nil { log.Fatal(err) } var result = Result{ @@ -175,5 +207,5 @@ func runCmd(ctx context.Context, cli *client.Client, containerID string, command Output: out.String(), Error: errOut.String(), } - return &result, nil + return &result } diff --git a/pkg/dunner/dunner.go b/pkg/dunner/dunner.go index 0a334cf..5475684 100644 --- a/pkg/dunner/dunner.go +++ b/pkg/dunner/dunner.go @@ -46,6 +46,7 @@ func execTask(configs *config.Configs, taskName string, args []string) { Task: taskName, Name: stepDefinition.Name, Image: stepDefinition.Image, + Command: stepDefinition.Command, Commands: stepDefinition.Commands, Env: stepDefinition.Envs, WorkDir: stepDefinition.SubDir, From 085588503dad8d93fa8d1f24f2cf8804e7254e74 Mon Sep 17 00:00:00 2001 From: Ayush Jain Date: Wed, 27 Mar 2019 18:24:19 +0530 Subject: [PATCH 30/41] Change syntax for task to be used as step of another task --- .dunner.yaml | 2 +- pkg/config/config.go | 1 + pkg/docker/docker.go | 1 + pkg/dunner/dunner.go | 8 ++++---- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.dunner.yaml b/.dunner.yaml index 86badf1..ab9b36d 100644 --- a/.dunner.yaml +++ b/.dunner.yaml @@ -16,7 +16,7 @@ build: - PERM=775 - ID=dunner - DIR=`$HOME` - - name: '@show' + - follow: 'show' args: - '/root' show: diff --git a/pkg/config/config.go b/pkg/config/config.go index 454255a..bfc07c2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -34,6 +34,7 @@ type Task struct { Commands [][]string `yaml:"commands"` Envs []string `yaml:"envs"` Mounts []string `yaml:"mounts"` + Follow string `yaml:"follow"` Args []string `yaml:"args"` } diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index 329811b..de9aa89 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -35,6 +35,7 @@ type Step struct { WorkDir string Volumes map[string]string ExtMounts []mount.Mount + Follow string Args []string } diff --git a/pkg/dunner/dunner.go b/pkg/dunner/dunner.go index 5475684..d0c4274 100644 --- a/pkg/dunner/dunner.go +++ b/pkg/dunner/dunner.go @@ -50,6 +50,7 @@ func execTask(configs *config.Configs, taskName string, args []string) { Commands: stepDefinition.Commands, Env: stepDefinition.Envs, WorkDir: stepDefinition.SubDir, + Follow: stepDefinition.Follow, Args: stepDefinition.Args, } @@ -73,16 +74,15 @@ func process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args [ defer wg.Done() } - if newTask := regexp.MustCompile(`^@\w+$`).FindString(s.Name); newTask != "" { - newTask = strings.Trim(newTask, "@") + if s.Follow != "" { if async { wg.Add(1) go func(wg *sync.WaitGroup) { - execTask(configs, newTask, s.Args) + execTask(configs, s.Follow, s.Args) wg.Done() }(wg) } else { - execTask(configs, newTask, s.Args) + execTask(configs, s.Follow, s.Args) } return } From 466e51d57f1b163bab5ec592f3dcc70f01c56f43 Mon Sep 17 00:00:00 2001 From: Ayush Jain Date: Wed, 27 Mar 2019 18:24:53 +0530 Subject: [PATCH 31/41] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 807be79..7bc08c4 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ deploy: - AWS_ACCESS_KEY_ID=`$AWS_KEY` - AWS_SECRET_ACCESS_KEY=`$AWS_SECRET` - AWS_DEFAULT_REGION=us-east1 -- name: '@status' #This refers to another task and can pass args too +- follow: 'status' #This refers to another task and can pass args too args: 'prod' status: - image: 'mesosphere/aws-cli' From 32cdbcc825da2418b2da700928e092e0d8dcaef2 Mon Sep 17 00:00:00 2001 From: Ayush Jain Date: Thu, 28 Mar 2019 00:54:28 +0530 Subject: [PATCH 32/41] Implement dry run mode of do command --- cmd/do.go | 8 +++++- internal/settings/settings.go | 1 + pkg/docker/docker.go | 47 +++++++++++++++++++---------------- pkg/dunner/dunner.go | 4 +++ 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/cmd/do.go b/cmd/do.go index ed414b3..f1b2069 100644 --- a/cmd/do.go +++ b/cmd/do.go @@ -10,11 +10,17 @@ func init() { rootCmd.AddCommand(doCmd) // Async Mode - doCmd.Flags().BoolP("async", "A", false, "Async mode") + doCmd.Flags().BoolP("async", "A", false, "Asynchronous mode") if err := viper.BindPFlag("Async", doCmd.Flags().Lookup("async")); err != nil { log.Fatal(err) } + // Dry-run mode + doCmd.Flags().Bool("dry-run", false, "Dry-run of the command") + if err := viper.BindPFlag("Dry-run", doCmd.Flags().Lookup("dry-run")); err != nil { + log.Fatal(err) + } + } var doCmd = &cobra.Command{ diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 4d13197..3d2d68a 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -29,6 +29,7 @@ func Init() { // Modes viper.SetDefault("Async", false) viper.SetDefault("Verbose", false) + viper.SetDefault("Dry-run", false) // Constants viper.SetDefault("DockerAPIVersion", "1.39") diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index de9aa89..f21e48f 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -143,35 +143,38 @@ func (step Step) Exec() (*[]Result, error) { }() var results []Result - if multipleCommands { - for _, cmd := range step.Commands { - r, err := runCmd(ctx, cli, resp.ID, cmd) - if err != nil { - log.Fatal(err) + if dryRun := viper.GetBool("Dry-run"); !dryRun { + if multipleCommands { + for _, cmd := range step.Commands { + r, err := runCmd(ctx, cli, resp.ID, cmd) + if err != nil { + log.Fatal(err) + } + results = append(results, *r) } - results = append(results, *r) - } - } else { - statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) - select { - case err = <-errCh: + } else { + statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) + select { + case err = <-errCh: + if err != nil { + log.Fatal(err) + } + case <-statusCh: + } + + out, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) if err != nil { log.Fatal(err) } - case <-statusCh: - } - out, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ - ShowStdout: true, - ShowStderr: true, - }) - if err != nil { - log.Fatal(err) + results = []Result{*extractResult(out, step.Command)} } - - results = []Result{*extractResult(out, step.Command)} + return &results, nil } - return &results, nil + return nil, nil } func runCmd(ctx context.Context, cli *client.Client, containerID string, command []string) (*Result, error) { diff --git a/pkg/dunner/dunner.go b/pkg/dunner/dunner.go index d0c4274..5fd2309 100644 --- a/pkg/dunner/dunner.go +++ b/pkg/dunner/dunner.go @@ -100,6 +100,10 @@ func process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args [ log.Fatal(err) } + if results == nil { + return + } + for _, res := range *results { log.Infof( "Running task '%+v' on '%+v' Docker with command '%+v'", From 65516692d1797151a554ea8f69e94d5f05710db5 Mon Sep 17 00:00:00 2001 From: Apoorva M Date: Mon, 20 May 2019 10:10:27 -0400 Subject: [PATCH 33/41] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eb310b0..4f43dbf 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ build: install ci: test test: build - @go -v $(ALL_PACKAGES) + @go test -v $(ALL_PACKAGES) vet: @go vet $(ALL_PACKAGES) From de5c166bf2aac0f592d6aabaa03eea64cdfb1949 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Mon, 20 May 2019 10:32:43 -0400 Subject: [PATCH 34/41] add formatting changes --- pkg/config/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index ad0625a..536caad 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -35,6 +35,7 @@ var ( ) type contextKey string + var configsKey = contextKey("dunnerConfigs") type customValidation struct { From ed6af4e0a4ed28a175e5259fb7ce200a38cb6e09 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Mon, 20 May 2019 10:41:09 -0400 Subject: [PATCH 35/41] fix vet warnings --- pkg/config/config.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 536caad..01b554f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -52,12 +52,6 @@ var customValidations = []customValidation{ }, } -type DirMount struct { - Src string `yaml:"src"` - Dest string `yaml:"dest"` - ReadOnly bool `yaml:"read-only"` -} - // Task describes a single task to be run in a docker container type Task struct { Name string `yaml:"name"` From b586e36d62cdc4efa90c2c659c3b957b850d6b93 Mon Sep 17 00:00:00 2001 From: Apoorva M Date: Mon, 20 May 2019 11:55:28 -0400 Subject: [PATCH 36/41] Add command to make release `make release v3.0.0` will add a git tag which can trigger the dunner release. --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 4f43dbf..5440eb8 100644 --- a/Makefile +++ b/Makefile @@ -50,3 +50,8 @@ test-coverage: tail -n +2 coverage.out >> coverage-all.out;) @go tool cover -html=coverage-all.out -o coverage.html +release: + @echo "Make sure you run this on master branch to make a release" + @echo "Adding tag for version: $(VERSION)" + git tag -a $(VERSION) -m "Release version $(VERSION)" + @echo "Run \"git push origin $(VERSION)\" to push tag to remote which makes a dunner release!" From e1deab59632061f4d76a48c77c43a814fa5fd566 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Mon, 20 May 2019 13:35:52 -0400 Subject: [PATCH 37/41] Update make ci command --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5440eb8..e5c63b5 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,8 @@ install: build: install @$(GOINSTALL) -ldflags "-X main.version=$(VERSION)-$(SHA) -s" -ci: test +ci: build fmt lint vet + @go test -v $(ALL_PACKAGES) -race -coverprofile=coverage.txt -covermode=atomic test: build @go test -v $(ALL_PACKAGES) From ddb5715e2f1cb2c358d5bd6c6726a7da4e6edb97 Mon Sep 17 00:00:00 2001 From: apoorvam Date: Mon, 20 May 2019 13:43:36 -0400 Subject: [PATCH 38/41] Run setup as part of ci --- .travis.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9816f24..13c7639 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ before_install: - chmod +x $GOPATH/bin/dep install: - - $GOPATH/bin/dep ensure -v + - make setup - sudo snap install snapcraft --classic script: diff --git a/Makefile b/Makefile index e5c63b5..9e47d25 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ setup: install @go get -u golang.org/x/lint/golint install: - @$(DEP) ensure + @$(DEP) ensure -v build: install @$(GOINSTALL) -ldflags "-X main.version=$(VERSION)-$(SHA) -s" From 2aea51d84eccf84f7f25dd191779c2cfebbc5d16 Mon Sep 17 00:00:00 2001 From: agentmilindu Date: Sat, 25 May 2019 12:32:14 +0530 Subject: [PATCH 39/41] Remove Dev mode and make the release stable --- snapcraft.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index 2cbd30a..9ca7070 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -4,7 +4,8 @@ summary: A Docker based task runner tool description: | Dunner is a task runner tool like Grunt but used Docker images like CircleCI do. | You can define tasks and steps of the tasks in your `.dunner.yaml` file and then run these steps with `Dunner do taskname` -confinement: devmode +grade: stable +confinement: strict base: core18 parts: @@ -18,4 +19,4 @@ parts: apps: dunner: - command: Dunner + command: bin/dunner From 06a036e4f7a1382f6486441e3b38bb228e78bab7 Mon Sep 17 00:00:00 2001 From: agentmilindu Date: Sun, 26 May 2019 13:43:35 +0530 Subject: [PATCH 40/41] Update goReleser for Snapcraft --- .goreleaser.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index a476d3e..896eca4 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -59,5 +59,5 @@ snapcraft: Dunner is a task runner tool like Grunt but used Docker images like CircleCI do. | You can define tasks and steps of the tasks in your `.dunner.yaml` file and then run these steps with `Dunner do taskname` grade: stable - confinement: devmode - publish: false \ No newline at end of file + confinement: strict + publish: true From b1bc1a4b58da5aa20fbe4572a23bd95ad54a1857 Mon Sep 17 00:00:00 2001 From: agentmilindu Date: Sun, 26 May 2019 14:06:39 +0530 Subject: [PATCH 41/41] Added snap.login.enc for SnapCraft auto build on Travis CI --- .travis.yml | 37 ++++++++++++++++--------------------- snap.login.enc | Bin 0 -> 2384 bytes 2 files changed, 16 insertions(+), 21 deletions(-) create mode 100644 snap.login.enc diff --git a/.travis.yml b/.travis.yml index 13c7639..17b3dc8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,37 +1,32 @@ language: go - go: - - "1.11.x" - - master - +- 1.11.x +- master env: global: - - DEP_VERSION="0.5.0" - - PATH=/snap/bin:$PATH - + - DEP_VERSION="0.5.0" + - PATH=/snap/bin:$PATH services: - - docker +- docker addons: apt: packages: - rpm - snapd - before_install: - - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep - - chmod +x $GOPATH/bin/dep - +- openssl aes-256-cbc -K $encrypted_12c8071d2874_key -iv $encrypted_12c8071d2874_iv + -in snap.login.enc -out snap.login -d +- curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 + -o $GOPATH/bin/dep +- chmod +x $GOPATH/bin/dep install: - - make setup - - sudo snap install snapcraft --classic - +- make setup +- sudo snap install snapcraft --classic script: - - make ci - +- make ci after_success: - - bash <(curl -s https://codecov.io/bash) - - test -n "$TRAVIS_TAG" && snapcraft login --with snap.login - +- bash <(curl -s https://codecov.io/bash) +- test -n "$TRAVIS_TAG" && snapcraft login --with snap.login deploy: - provider: script skip_cleanup: true @@ -39,5 +34,5 @@ deploy: verbose: true on: tags: true - condition: $TRAVIS_OS_NAME = linux + condition: "$TRAVIS_OS_NAME = linux" master: true diff --git a/snap.login.enc b/snap.login.enc new file mode 100644 index 0000000000000000000000000000000000000000..ec5c6b72f52199c5d9b62fa03031011f84851a3d GIT binary patch literal 2384 zcmV-W39t5+dI$0hngtj4Tc>_37{2mshKQpEoMDbE7G6hAQ%Du`kftR~o6VFb+n|*_ zi@*uPZ_`jGXpn>NEJ?$ACVB3xSeQ^JyND4Swg|ei5iC+FFON8BheQ$23e-$Y?>?!d zFMp|;L+rr@yoVz5J#;+NUXJ+P+-Y3YrqXd<`*h38zZ@d4`rs0Jo$Wq+<&7$uT2wxq z!`&crrd*+Zu*cRGDqA9fA5O53-$o;}gjenq&3S-MF{Xn^ELRzd}gT-qd|<7_f0HaES3 z7Jmo?W1dmS8>*vLu^%PkrHr7cJFyub<|827lNm_!OUdu@GyrygD=Xt*jpp`r;F$*H z5|e~$%ot3Qu_eSCxliq`8pNt?<{3r6OIbXEKDXPC0;B72;#0mOpg&9NzA)l;$tng% zFk+uuxLIPfCo&P>%Ge(;bRJ5O^7-6Bri3(&n)+<;xNMW5MD#u)V@4`Bi(h_MU!{=91Qh+9XZhH@X2lR}%u+5~ zb-;NDg#fNvm4}KTe|rlH#B`A=K|=|lBzfMPAO**K03u;|um@^D`q=q{i}kqW?M=8Nv!&tx)mM>1_@x$Z>Mmd%gj@ z>e9WX&ZcZaEnTg>PIaM|ec;jpdx_t>1nA?alzs#Wns)agEazfLxGyr@md^OvsJJ&6 zq-k!eJct&+LRMP{lM?hzw`H%K=rvCTT)6AE8&5m46r@?qgD6Ol5FBj1j7l&~N{VQE{n| z;@E#Ia;4zycsS5uX564QscMRs6)zkGdel9nEZz|Qrjq)cnM5^M%Y1W-C=e0H^oi~8 zb+e)eHS;JoXra^QH5FkFKn4=&#`v~mDb)Ud)es^ErGa=stu!%b*E zHUYNw!duOBNQgJM*WD(Nrs?X*>TSB#jPgrW;?A9e`|Df3u_%Hd(gD+18#y45bx-3whSv_EMzLdauUC-fZ&VfnUXuASewnxc4E+MU*7y zQ6JmR%g1r$^$u$pJEZeS(@LBElIEdZwP5{nf)Mw=A9L@SL7umlY$VZ?8Zn~vDhkTT zgF?OIr;L4}y^ZG~Q4nh!{l7Dl6WMEGCjjT2Z*P3eUtkVkq0=*(SWsx=9@T}?k`>|M zptN4U=U_%*TU+>^xupL+hBB{l^@dlEp7s+?gYLrT3?-9`xDb4W*~xu4$%7~QdJdt< zJK_^F?--Q&JfdP{Fuy~M%N;C2FFeD?_T<18X- zoXA*Y-C6xp^}BqENwv?z`el+#!!gJ&++C8{t0%Qo4ph1BJB6z7_Qrqk%?)1JhyWrh z@dM;4OMM}Q6tj|8K0}S}>jha0>ko4gCkBuI{%zTz%<*MrSPe-}gL29W=z#DDc_)ud z`zSzds8-F`*Ohbxxi{`$m6PF~AX!ieH16JmP&GW_R3XUzm{+;(-Wzr{ZRF6~*k=jCzeQ!+|oZEIn2 z-bEYQlj2Ri?5v&-+>Iy@>(7992(9Yp!R`^^m3!|C?mnTfBfWf0@OWl3=PD}^T-Bbh&@TDYW(I-f#@WRq|-rr$02Ucz)jDm9 zBqzr!;4w1zmE$O>lUg#X!U@LbEs-ldQ~hnqUt;56enI+Z`6jKwoyBP%7MS`c*?$=Z zt||6bmS)K!T@XJt+sZpe_!P=$1XdK5FZuX1hOdd~fhHN}zjJ{ZPIo3F7N06gVnJ`( zSCdF*!e+!+OxaTUX$G^LJ&cEYp%abC4c(350|}rWHiPUPP}iphO8kLQPO$-nJ?45* z3%eU}(byzoV+QYj#kX8JG0ss8sy3w<_Yg@k(a6dRSm|?k=dC_K=R6 zGnO=XA*jh+KEJ(bq)(W%I=1Ylz4g44ge8y#tMb+R6!~Ml)2|Kps-V-382Cm(cYo~_ zqMz!XMqDEp);8j!Ouk_gn-MgZm8W5pPbHpN_@0y}`Tv$wuF6S%r2Io`HpZ8UfWAQO z6q#@G30MpBa)tN}PzInEzs~632;+3Cbl@6aWPO%APH5U`<3(fAm(Z;r2ARNw?n`_G zmb36uR3Fqi>KG|maMl{#n9CUf$0BofpAsF7@4<7bnr;>~6u;eFX2(xXhfam|85DVHP4D=W}SkS?F#rRRniCdHUtJWBr;j_m9Oj)jk*I znHC~s*7tjQ+mwlCe2+p4e|JS=OnVZ4_m?xSHvQ76-TS|vzQ_hP# ze@0+=^k^ov3FC7s*!eE^{d`;&q8sCk^;vb%z}3FM%%T28QsF9B)lgZ?{bEAD{shUT C8n5mE literal 0 HcmV?d00001