diff --git a/Gopkg.lock b/Gopkg.lock index bfddd17..a3f59b6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -34,7 +34,7 @@ [[projects]] branch = "master" - digest = "1:2239c4f8af92a5d5e65341b8537e799208cb442a67db36740ebb82996dce5e32" + digest = "1:15cce32b1a36005d784c500eaef4727a3d969ca9cebfe5aeda8b61b2db9d8836" name = "github.com/docker/docker" packages = [ "api", @@ -61,7 +61,7 @@ "pkg/term/windows", ] pruneopts = "UT" - revision = "3042254a87274ff5e9561f2da1a986a703dfc60f" + revision = "34b56728ed7101c6b3cc0405f5fd6351073a8253" [[projects]] digest = "1:811c86996b1ca46729bad2724d4499014c4b9effd05ef8c71b852aad90deb0ce" @@ -91,6 +91,26 @@ revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" +[[projects]] + digest = "1:fd9d04d4fe18d02ecd2cdd0b8335a3c27210b10a9a5b92b9f01ac3caf8648e36" + name = "github.com/go-playground/locales" + packages = [ + ".", + "currency", + "en", + ] + pruneopts = "UT" + revision = "f63010822830b6fe52288ee52d5a1151088ce039" + version = "v0.12.1" + +[[projects]] + digest = "1:e022cf244bcac1b6ef933f1a2e0adcf6a6dfd7b872d8d41e4d4179bb09a87cbc" + name = "github.com/go-playground/universal-translator" + packages = ["."] + pruneopts = "UT" + revision = "b32fa301c9fe55953584134cb6853a13c87ec0a1" + version = "v0.16.0" + [[projects]] digest = "1:48092bf6632f55839850666c33469f546f6d45fdbd59a66759ec12e84d853dc2" name = "github.com/gogo/protobuf" @@ -156,6 +176,14 @@ revision = "f55edac94c9bbba5d6182a4be46d86a2c9b5b50e" version = "v1.0.2" +[[projects]] + digest = "1:6782ffc812e8e700e6952ede1e60487ff1fd9da489eff762985be662a7cfc431" + name = "github.com/leodido/go-urn" + packages = ["."] + pruneopts = "UT" + revision = "70078a794e8ea4b497ba7c19a78cd60f90ccf0f4" + version = "v1.1.0" + [[projects]] digest = "1:5a0ef768465592efca0412f7e838cdc0826712f8447e70e6ccc52eb441e9ab13" name = "github.com/magiconair/properties" @@ -283,18 +311,18 @@ "proxy", ] pruneopts = "UT" - revision = "a4d6f7feada510cc50e69a37b484cb0fdc6b7876" + revision = "3ec19112720433827bbce8be9342797f5a6aaaf9" [[projects]] branch = "master" - digest = "1:646036a04e163f383d101f40e36fd3354676c7dcccc226d825200a6b9509727a" + digest = "1:a63ab7381e8941204bf91ce0bbdcb9f47767de756f98209a6a059dcbd52d57b6" name = "golang.org/x/sys" packages = [ "unix", "windows", ] pruneopts = "UT" - revision = "a5b02f93d862f065920dd6a40dddc66b60d0dec4" + revision = "87c872767d25fb96dfe96c794fd028b38a08440b" [[projects]] digest = "1:1093f2eb4b344996604f7d8b29a16c5b22ab9e1b25652140d3fede39f640d5cd" @@ -313,11 +341,11 @@ [[projects]] branch = "master" - digest = "1:fc5df4cb84cdf639ef702c176a50a9ee08ea3e68ebd67519c0f4f963c7d82978" + digest = "1:583a0c80f5e3a9343d33aea4aead1e1afcc0043db66fdf961ddd1fe8cd3a4faf" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] pruneopts = "UT" - revision = "b515fa19cec88c32f305a962f34ae60068947aea" + revision = "d00d292a067ce1aa0017b40ca75437b42461fa61" [[projects]] digest = "1:4a251721b698c0e6285aec21d4ba3f79ddeccd903690144461b360ea7f948106" @@ -330,6 +358,17 @@ revision = "25c4f928eaa6d96443009bd842389fb4fa48664e" version = "v1.20.1" +[[projects]] + digest = "1:d9ed69c1d4d528e4a7779accc3ee070e68760cdd822f69ff2cf8ae0f677afbe2" + name = "gopkg.in/go-playground/validator.v9" + packages = [ + ".", + "translations/en", + ] + pruneopts = "UT" + revision = "46b4b1e301c24cac870ffcb4ba5c8a703d1ef475" + version = "v9.28.0" + [[projects]] digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96" name = "gopkg.in/yaml.v2" @@ -349,10 +388,14 @@ "github.com/docker/docker/pkg/jsonmessage", "github.com/docker/docker/pkg/stdcopy", "github.com/docker/docker/pkg/term", + "github.com/go-playground/locales/en", + "github.com/go-playground/universal-translator", "github.com/joho/godotenv", "github.com/sirupsen/logrus", "github.com/spf13/cobra", "github.com/spf13/viper", + "gopkg.in/go-playground/validator.v9", + "gopkg.in/go-playground/validator.v9/translations/en", "gopkg.in/yaml.v2", ] solver-name = "gps-cdcl" diff --git a/Gopkg.toml b/Gopkg.toml index 18489df..af7967d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -41,3 +41,11 @@ [prune] go-tests = true unused-packages = true + +[[constraint]] + name = "gopkg.in/go-playground/validator.v9" + version = "9.28.0" + +[[constraint]] + name = "github.com/go-playground/universal-translator" + version = "0.16.0" \ No newline at end of file diff --git a/cmd/validate.go b/cmd/validate.go new file mode 100644 index 0000000..893a906 --- /dev/null +++ b/cmd/validate.go @@ -0,0 +1,42 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/leopardslab/dunner/pkg/config" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func init() { + rootCmd.AddCommand(validateCmd) +} + +var validateCmd = &cobra.Command{ + Use: "validate", + Short: "Validate the dunner task file `.dunner.yaml`", + Long: "You can validate task file `.dunner.yaml` with this command to see if there are any parse errors", + Run: Validate, + Args: cobra.MinimumNArgs(0), +} + +// Validate command invoked from command line, validates the dunner task file. If there are errors, it fails with non-zero exit code. +func Validate(_ *cobra.Command, args []string) { + var dunnerFile = viper.GetString("DunnerTaskFile") + + configs, err := config.GetConfigs(dunnerFile) + if err != nil { + log.Fatal(err) + } + + errs := configs.Validate() + if len(errs) != 0 { + fmt.Println("Validation failed with following errors:") + for _, err := range errs { + fmt.Println(err.Error()) + } + os.Exit(1) + } + fmt.Println("Validation successful!") +} diff --git a/internal/util/util.go b/internal/util/util.go new file mode 100644 index 0000000..8eb8338 --- /dev/null +++ b/internal/util/util.go @@ -0,0 +1,22 @@ +package util + +import ( + "os" + "path" + "strings" +) + +// HomeDir is the environment variable HOME +var HomeDir = os.Getenv("HOME") + +// DirExists returns true if the given param is a valid existing directory +func DirExists(dir string) bool { + if strings.HasPrefix(dir, "~") { + dir = path.Join(HomeDir, strings.Trim(dir, "~")) + } + src, err := os.Stat(dir) + if err != nil { + return false + } + return src.IsDir() +} diff --git a/internal/util/util_test.go b/internal/util/util_test.go new file mode 100644 index 0000000..ce7993a --- /dev/null +++ b/internal/util/util_test.go @@ -0,0 +1,51 @@ +package util + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestDirExistsSuccess(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestDir") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + exists := DirExists(tmpdir) + + if !exists { + t.Fatalf("Directory exists; but got false") + } +} + +func TestDirExistsFail(t *testing.T) { + exists := DirExists("this path is invalid") + + if exists { + t.Fatalf("Directory invalid; but got as exists") + } +} + +func TestDirExistsFailForFile(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "TestFileExists") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpfile.Name()) + + exists := DirExists(tmpfile.Name()) + + if exists { + t.Fatalf("Not a directory; but got as true") + } +} + +func TestDirExistsIfNotAbsPath(t *testing.T) { + exists := DirExists("~/invalidpathfortesting") + + if exists { + t.Fatalf("Not a directory; but got as true") + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index c5db987..01b554f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,40 +1,167 @@ package config import ( + "context" "fmt" "io/ioutil" "os" "path" "path/filepath" + "reflect" "regexp" "strings" "github.com/docker/docker/api/types/mount" + "github.com/go-playground/locales/en" + ut "github.com/go-playground/universal-translator" "github.com/joho/godotenv" "github.com/leopardslab/dunner/internal/logger" + "github.com/leopardslab/dunner/internal/util" "github.com/leopardslab/dunner/pkg/docker" "github.com/spf13/viper" + "gopkg.in/go-playground/validator.v9" + en_translations "gopkg.in/go-playground/validator.v9/translations/en" yaml "gopkg.in/yaml.v2" ) var log = logger.Log +var ( + uni *ut.UniversalTranslator + govalidator *validator.Validate + trans ut.Translator + defaultPermissionMode = "r" + validDirPermissionModes = []string{defaultPermissionMode, "wr", "rw", "w"} +) + +type contextKey string + +var configsKey = contextKey("dunnerConfigs") + +type customValidation struct { + tag string + translation string + validationFn func(context.Context, validator.FieldLevel) bool +} + +var customValidations = []customValidation{ + { + tag: "mountdir", + translation: "mount directory '{0}' is invalid. Check format is '::' and has right permission level", + validationFn: ValidateMountDir, + }, +} + // Task describes a single task to be run in a docker container type Task struct { Name string `yaml:"name"` - Image string `yaml:"image"` + Image string `yaml:"image" validate:"required"` SubDir string `yaml:"dir"` - Command []string `yaml:"command"` + Command []string `yaml:"command" validate:"omitempty,dive,required"` Commands [][]string `yaml:"commands"` Envs []string `yaml:"envs"` - Mounts []string `yaml:"mounts"` + Mounts []string `yaml:"mounts" validate:"omitempty,dive,min=1,mountdir"` Follow string `yaml:"follow"` Args []string `yaml:"args"` } // Configs describes the parsed information from the dunner file type Configs struct { - Tasks map[string][]Task + Tasks map[string][]Task `validate:"required,min=1,dive,keys,required,endkeys,required,min=1,required"` +} + +// Validate validates config and returns errors. +func (configs *Configs) Validate() []error { + err := initValidator(customValidations) + if err != nil { + return []error{err} + } + valErrs := govalidator.Struct(configs) + errs := formatErrors(valErrs, "") + ctx := context.WithValue(context.Background(), configsKey, configs) + + // Each task is validated separately so that task name can be added in error messages + for taskName, tasks := range configs.Tasks { + taskValErrs := govalidator.VarCtx(ctx, tasks, "dive") + errs = append(errs, formatErrors(taskValErrs, taskName)...) + } + return errs +} + +func formatErrors(valErrs error, taskName string) []error { + var errs []error + if valErrs != nil { + if _, ok := valErrs.(*validator.InvalidValidationError); ok { + errs = append(errs, valErrs) + } else { + for _, e := range valErrs.(validator.ValidationErrors) { + if taskName == "" { + errs = append(errs, fmt.Errorf(e.Translate(trans))) + } else { + errs = append(errs, fmt.Errorf("task '%s': %s", taskName, e.Translate(trans))) + } + } + } + } + return errs +} + +func initValidator(customValidations []customValidation) error { + govalidator = validator.New() + govalidator.RegisterTagNameFunc(func(fld reflect.StructField) string { + name := strings.SplitN(fld.Tag.Get("yaml"), ",", 2)[0] + if name == "-" { + return "" + } + return name + }) + + // Register default translators + translator := en.New() + uni = ut.New(translator, translator) + var translatorFound bool + trans, translatorFound = uni.GetTranslator("en") + if !translatorFound { + return fmt.Errorf("failed to initialize validator with translator") + } + en_translations.RegisterDefaultTranslations(govalidator, trans) + + // Register Custom validators and translations + for _, t := range customValidations { + err := govalidator.RegisterValidationCtx(t.tag, t.validationFn) + if err != nil { + return fmt.Errorf("failed to register validation: %s", err.Error()) + } + err = govalidator.RegisterTranslation(t.tag, trans, registrationFunc(t.tag, t.translation), translateFunc) + if err != nil { + return fmt.Errorf("failed to register translations: %s", err.Error()) + } + } + return nil +} + +// ValidateMountDir verifies that mount values are in proper format :: +// Format should match, is optional which is `readOnly` by default and `src` directory exists in host machine +func ValidateMountDir(ctx context.Context, fl validator.FieldLevel) bool { + value := fl.Field().String() + f := func(c rune) bool { return c == ':' } + mountValues := strings.FieldsFunc(value, f) + if len(mountValues) != 3 { + mountValues = append(mountValues, defaultPermissionMode) + } + if len(mountValues) != 3 { + return false + } + validPerm := false + for _, perm := range validDirPermissionModes { + if mountValues[2] == perm { + validPerm = true + } + } + if !validPerm { + return false + } + return util.DirExists(mountValues[0]) } // GetConfigs reads and parses tasks from the dunner file @@ -122,21 +249,10 @@ func DecodeMount(mounts []string, step *docker.Step) error { strings.Trim(strings.Trim(m, `'`), `"`), ":", ) - if len(arr) != 3 && len(arr) != 2 { - return fmt.Errorf( - `config: invalid format for mount %s`, - m, - ) - } var readOnly = true if len(arr) == 3 { if arr[2] == "wr" || arr[2] == "w" { readOnly = false - } else if arr[2] != "r" { - return fmt.Errorf( - `config: invalid format of read-write mode for mount '%s'`, - m, - ) } } src, err := filepath.Abs(joinPathRelToHome(arr[0])) @@ -157,7 +273,24 @@ func DecodeMount(mounts []string, step *docker.Step) error { func joinPathRelToHome(p string) string { if p[0] == '~' { - return path.Join(os.Getenv("HOME"), strings.Trim(p, "~")) + return path.Join(util.HomeDir, strings.Trim(p, "~")) } return p } + +func registrationFunc(tag string, translation string) validator.RegisterTranslationsFunc { + return func(ut ut.Translator) (err error) { + if err = ut.Add(tag, translation, true); err != nil { + return + } + return + } +} + +func translateFunc(ut ut.Translator, fe validator.FieldError) string { + t, err := ut.T(fe.Tag(), reflect.ValueOf(fe.Value()).String(), fe.Param()) + if err != nil { + return fe.(error).Error() + } + return t +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 222b7d5..83ffb04 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "io/ioutil" "os" "reflect" @@ -56,3 +57,154 @@ test: } } + +func TestConfigs_Validate(t *testing.T) { + tasks := make(map[string][]Task, 0) + tasks["stats"] = []Task{getSampleTask()} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 0 { + t.Fatalf("Configs Validation failed, expected to pass. got: %s", errs) + } +} + +func TestConfigs_ValidateWithNoTasks(t *testing.T) { + tasks := make(map[string][]Task, 0) + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 1 { + t.Fatalf("Configs validation failed, expected 1 error, got %s", errs) + } + expected := "Tasks must contain at least 1 item" + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + +func TestConfigs_ValidateWithEmptyImageAndCommand(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := Task{Image: "", Command: []string{""}} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 2 { + t.Fatalf("expected 2 errors, got %d : %s", len(errs), errs) + } + + expected1 := "task 'stats': image is a required field" + expected2 := "task 'stats': command[0] is a required field" + if errs[0].Error() != expected1 { + t.Fatalf("expected: %s, got: %s", expected1, errs[0].Error()) + } + if errs[1].Error() != expected2 { + t.Fatalf("expected: %s, got: %s", expected2, errs[1].Error()) + } +} + +func TestConfigs_ValidateWithInvalidMountFormat(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + task.Mounts = []string{"invalid_dir"} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d : %s", len(errs), errs) + } + + expected := "task 'stats': mount directory 'invalid_dir' is invalid. Check format is '::' and has right permission level" + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + +func TestConfigs_ValidateWithValidMountDirectory(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + wd, _ := os.Getwd() + task.Mounts = []string{fmt.Sprintf("%s:%s:w", wd, wd)} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if errs != nil { + t.Fatalf("expected no errors, got %s", errs) + } +} + +func TestConfigs_ValidateWithNoModeGiven(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + wd, _ := os.Getwd() + task.Mounts = []string{fmt.Sprintf("%s:%s", wd, wd)} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if errs != nil { + t.Fatalf("expected no errors, got %s", errs) + } +} + +func TestConfigs_ValidateWithInvalidMode(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + wd, _ := os.Getwd() + task.Mounts = []string{fmt.Sprintf("%s:%s:ab", wd, wd)} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + expected := fmt.Sprintf("task 'stats': mount directory '%s' is invalid. Check format is '::' and has right permission level", task.Mounts[0]) + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + +func TestConfigs_ValidateWithInvalidMountDirectory(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + task.Mounts = []string{"blah:foo:w"} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d : %s", len(errs), errs) + } + + expected := "task 'stats': mount directory 'blah:foo:w' is invalid. Check format is '::' and has right permission level" + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + +func getSampleTask() Task { + return Task{Image: "image_name", Command: []string{"node", "--version"}} +} + +func TestInitValidatorForNilTranslation(t *testing.T) { + vals := []customValidation{{tag: "foo", translation: "", validationFn: nil}} + + err := initValidator(vals) + + expected := "failed to register validation: Function cannot be empty" + if err == nil { + t.Fatalf("expected %s, got %s", expected, err) + } + if err.Error() != expected { + t.Fatalf("expected %s, got %s", expected, err.Error()) + } +} diff --git a/pkg/dunner/dunner.go b/pkg/dunner/dunner.go index 5fd2309..150c461 100644 --- a/pkg/dunner/dunner.go +++ b/pkg/dunner/dunner.go @@ -2,6 +2,7 @@ package dunner import ( "fmt" + "os" "regexp" "strconv" "strings" @@ -31,6 +32,14 @@ func Do(_ *cobra.Command, args []string) { if err != nil { log.Fatal(err) } + errs := configs.Validate() + if len(errs) != 0 { + fmt.Println("Validation failed with following errors:") + for _, err := range errs { + fmt.Println(err.Error()) + } + os.Exit(1) + } execTask(configs, args[0], args[1:]) }