diff --git a/.dunner.yaml b/.dunner.yaml index c64c3a1..ab9b36d 100644 --- a/.dunner.yaml +++ b/.dunner.yaml @@ -1,25 +1,28 @@ build: - image: node:10.15.0 - command: ["node", "--version"] - - image: node:10.15.0 - command: ["npm", "--version"] + commands: + - ["node", "--version"] + - ["npm", "--version"] - image: alpine dir: pkg command: ["pwd"] - image: alpine - command: ["apk", "update"] + commands: + - ["apk", "update"] - image: alpine - command: ["printenv"] + commands: + - ["printenv"] envs: - PERM=775 - ID=dunner - DIR=`$HOME` - - name: '@show' + - follow: 'show' args: - '/root' show: - image: alpine - command: ["ls", "$1"] + commands: + - ["ls", "$1"] mounts: - '~/Downloads:/root/down' - ~/Pictures:/root/pics:wr diff --git a/.gitignore b/.gitignore index 540eee7..7b287d0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,11 @@ # Environment file .env + +# Packages +dist/* +# Coverage files +coverage* + +# IDE files +.idea/* diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..896eca4 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,63 @@ +# Goreleaser documentation at http://goreleaser.com +project_name: dunner +builds: +- env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - 386 + - amd64 + - arm + - arm64 +archives: +- replacements: + 386: i386 + amd64: x86_64 +checksum: + name_template: '{{ .ProjectName }}_checksums.txt' +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + - Merge pull request + - Merge branch + - Update readme +snapshot: + name_template: "{{.ProjectName}}_{{.Tag}}" + +brew: + github: + owner: leopardslab + name: homebrew-dunner + folder: Formula + homepage: https://github.com/leopardslab/Dunner + description: A Docker based task runner tool + test: | + system "#{bin}/dunner version" + +nfpm: + name_template: '{{ .ProjectName }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' + homepage: https://github.com/leopardslab/Dunner + description: A Docker based task runner tool + license: MIT + formats: + - deb + - rpm + dependencies: + - git + recommends: + - rpm +snapcraft: + name_template: '{{ .ProjectName }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' + summary: A Docker based task runner tool + description: | + Dunner is a task runner tool like Grunt but used Docker images like CircleCI do. | + You can define tasks and steps of the tasks in your `.dunner.yaml` file and then run these steps with `Dunner do taskname` + grade: stable + confinement: strict + publish: true diff --git a/.travis.yml b/.travis.yml index a5a610a..17b3dc8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,38 @@ language: go - go: - - "1.11.x" - - master - +- 1.11.x +- master env: + global: - DEP_VERSION="0.5.0" - + - PATH=/snap/bin:$PATH services: - - docker - +- docker +addons: + apt: + packages: + - rpm + - snapd before_install: - - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep - - chmod +x $GOPATH/bin/dep - +- openssl aes-256-cbc -K $encrypted_12c8071d2874_key -iv $encrypted_12c8071d2874_iv + -in snap.login.enc -out snap.login -d +- curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 + -o $GOPATH/bin/dep +- chmod +x $GOPATH/bin/dep install: - - dep ensure -v - -script: go test -v ./... \ No newline at end of file +- make setup +- sudo snap install snapcraft --classic +script: +- make ci +after_success: +- bash <(curl -s https://codecov.io/bash) +- test -n "$TRAVIS_TAG" && snapcraft login --with snap.login +deploy: +- provider: script + skip_cleanup: true + script: curl -sL https://git.io/goreleaser | bash + verbose: true + on: + tags: true + condition: "$TRAVIS_OS_NAME = linux" + master: true diff --git a/Gopkg.lock b/Gopkg.lock index bfddd17..a3f59b6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -34,7 +34,7 @@ [[projects]] branch = "master" - digest = "1:2239c4f8af92a5d5e65341b8537e799208cb442a67db36740ebb82996dce5e32" + digest = "1:15cce32b1a36005d784c500eaef4727a3d969ca9cebfe5aeda8b61b2db9d8836" name = "github.com/docker/docker" packages = [ "api", @@ -61,7 +61,7 @@ "pkg/term/windows", ] pruneopts = "UT" - revision = "3042254a87274ff5e9561f2da1a986a703dfc60f" + revision = "34b56728ed7101c6b3cc0405f5fd6351073a8253" [[projects]] digest = "1:811c86996b1ca46729bad2724d4499014c4b9effd05ef8c71b852aad90deb0ce" @@ -91,6 +91,26 @@ revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" +[[projects]] + digest = "1:fd9d04d4fe18d02ecd2cdd0b8335a3c27210b10a9a5b92b9f01ac3caf8648e36" + name = "github.com/go-playground/locales" + packages = [ + ".", + "currency", + "en", + ] + pruneopts = "UT" + revision = "f63010822830b6fe52288ee52d5a1151088ce039" + version = "v0.12.1" + +[[projects]] + digest = "1:e022cf244bcac1b6ef933f1a2e0adcf6a6dfd7b872d8d41e4d4179bb09a87cbc" + name = "github.com/go-playground/universal-translator" + packages = ["."] + pruneopts = "UT" + revision = "b32fa301c9fe55953584134cb6853a13c87ec0a1" + version = "v0.16.0" + [[projects]] digest = "1:48092bf6632f55839850666c33469f546f6d45fdbd59a66759ec12e84d853dc2" name = "github.com/gogo/protobuf" @@ -156,6 +176,14 @@ revision = "f55edac94c9bbba5d6182a4be46d86a2c9b5b50e" version = "v1.0.2" +[[projects]] + digest = "1:6782ffc812e8e700e6952ede1e60487ff1fd9da489eff762985be662a7cfc431" + name = "github.com/leodido/go-urn" + packages = ["."] + pruneopts = "UT" + revision = "70078a794e8ea4b497ba7c19a78cd60f90ccf0f4" + version = "v1.1.0" + [[projects]] digest = "1:5a0ef768465592efca0412f7e838cdc0826712f8447e70e6ccc52eb441e9ab13" name = "github.com/magiconair/properties" @@ -283,18 +311,18 @@ "proxy", ] pruneopts = "UT" - revision = "a4d6f7feada510cc50e69a37b484cb0fdc6b7876" + revision = "3ec19112720433827bbce8be9342797f5a6aaaf9" [[projects]] branch = "master" - digest = "1:646036a04e163f383d101f40e36fd3354676c7dcccc226d825200a6b9509727a" + digest = "1:a63ab7381e8941204bf91ce0bbdcb9f47767de756f98209a6a059dcbd52d57b6" name = "golang.org/x/sys" packages = [ "unix", "windows", ] pruneopts = "UT" - revision = "a5b02f93d862f065920dd6a40dddc66b60d0dec4" + revision = "87c872767d25fb96dfe96c794fd028b38a08440b" [[projects]] digest = "1:1093f2eb4b344996604f7d8b29a16c5b22ab9e1b25652140d3fede39f640d5cd" @@ -313,11 +341,11 @@ [[projects]] branch = "master" - digest = "1:fc5df4cb84cdf639ef702c176a50a9ee08ea3e68ebd67519c0f4f963c7d82978" + digest = "1:583a0c80f5e3a9343d33aea4aead1e1afcc0043db66fdf961ddd1fe8cd3a4faf" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] pruneopts = "UT" - revision = "b515fa19cec88c32f305a962f34ae60068947aea" + revision = "d00d292a067ce1aa0017b40ca75437b42461fa61" [[projects]] digest = "1:4a251721b698c0e6285aec21d4ba3f79ddeccd903690144461b360ea7f948106" @@ -330,6 +358,17 @@ revision = "25c4f928eaa6d96443009bd842389fb4fa48664e" version = "v1.20.1" +[[projects]] + digest = "1:d9ed69c1d4d528e4a7779accc3ee070e68760cdd822f69ff2cf8ae0f677afbe2" + name = "gopkg.in/go-playground/validator.v9" + packages = [ + ".", + "translations/en", + ] + pruneopts = "UT" + revision = "46b4b1e301c24cac870ffcb4ba5c8a703d1ef475" + version = "v9.28.0" + [[projects]] digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96" name = "gopkg.in/yaml.v2" @@ -349,10 +388,14 @@ "github.com/docker/docker/pkg/jsonmessage", "github.com/docker/docker/pkg/stdcopy", "github.com/docker/docker/pkg/term", + "github.com/go-playground/locales/en", + "github.com/go-playground/universal-translator", "github.com/joho/godotenv", "github.com/sirupsen/logrus", "github.com/spf13/cobra", "github.com/spf13/viper", + "gopkg.in/go-playground/validator.v9", + "gopkg.in/go-playground/validator.v9/translations/en", "gopkg.in/yaml.v2", ] solver-name = "gps-cdcl" diff --git a/Gopkg.toml b/Gopkg.toml index 18489df..af7967d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -41,3 +41,11 @@ [prune] go-tests = true unused-packages = true + +[[constraint]] + name = "gopkg.in/go-playground/validator.v9" + version = "9.28.0" + +[[constraint]] + name = "github.com/go-playground/universal-translator" + version = "0.16.0" \ No newline at end of file diff --git a/Makefile b/Makefile index 041a287..9e47d25 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all install test build clean +ALL_PACKAGES=$(shell go list ./... | grep -v "vendor") SHA=$(shell git rev-list HEAD --max-count=1 --abbrev-commit) TAG?=$(shell git tag -l --contains HEAD) @@ -8,23 +8,51 @@ ifeq ($(VERSION),) VERSION := latest endif - #Go parameters GOCMD=go GOINSTALL=$(GOCMD) install GOTEST=$(GOCMD) test DEP=dep +.PHONY : all install vet fmt test lint build -all: build +all: build test fmt lint vet -install: - @$(DEP) ensure +setup: install + @go get -u golang.org/x/lint/golint -test: install - @$(GOTEST) -v ./... +install: + @$(DEP) ensure -v -build: install +build: install @$(GOINSTALL) -ldflags "-X main.version=$(VERSION)-$(SHA) -s" -clean: - rm -rf * +ci: build fmt lint vet + @go test -v $(ALL_PACKAGES) -race -coverprofile=coverage.txt -covermode=atomic + +test: build + @go test -v $(ALL_PACKAGES) + +vet: + @go vet $(ALL_PACKAGES) + +fmt: + @go fmt $(ALL_PACKAGES) + +lint: + @golint -set_exit_status $(ALL_PACKAGES) + +precommit: build test fmt lint vet + +test-coverage: + @echo "mode: count" > coverage-all.out + + $(foreach pkg, $(ALL_PACKAGES),\ + go test -coverprofile=coverage.out -covermode=count $(pkg);\ + tail -n +2 coverage.out >> coverage-all.out;) + @go tool cover -html=coverage-all.out -o coverage.html + +release: + @echo "Make sure you run this on master branch to make a release" + @echo "Adding tag for version: $(VERSION)" + git tag -a $(VERSION) -m "Release version $(VERSION)" + @echo "Run \"git push origin $(VERSION)\" to push tag to remote which makes a dunner release!" diff --git a/README.md b/README.md index 5976eec..816b356 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ # Dunner [![Codacy Badge](https://api.codacy.com/project/badge/Grade/b2275e331d2745dc9527d45efbbf2da2)](https://app.codacy.com/app/Leopardslab/dunner?utm_source=github.com&utm_medium=referral&utm_content=leopardslab/dunner&utm_campaign=Badge_Grade_Dashboard) +[![Codecov branch](https://img.shields.io/codecov/c/github/leopardslab/dunner/master.svg?style=for-the-badge)](https://codecov.io/gh/leopardslab/dunner) [![Build Status](https://travis-ci.org/leopardslab/Dunner.svg?branch=master)](https://travis-ci.org/leopardslab/Dunner) -Dunner is a task runner tool like Grunt but uses Docker images like CircleCI do. You can define tasks and steps of the tasks in your `.dunner.yaml` file and then run these steps with `Dunner do taskname` +Dunner is a task runner tool like Grunt but uses Docker images like CircleCI do. You can define tasks and steps of the tasks in your `.dunner.yaml` file and then run these steps with `dunner do taskname` Example `.dunner.yaml` @@ -11,20 +12,24 @@ Example `.dunner.yaml` ```yaml deploy: - image: 'emeraldsquad/sonar-scanner' - command: ['sonar', 'scan'] + commands: + - ['sonar', 'scan'] - image: 'golang' - command: ['go', 'install'] + commands: + - ['go', 'install'] - image: 'mesosphere/aws-cli' - command: ['aws', 'elasticbeanstalk update-application --application-name myapp'] + commands: + - ['aws', 'elasticbeanstalk update-application --application-name myapp'] envs: - AWS_ACCESS_KEY_ID=`$AWS_KEY` - AWS_SECRET_ACCESS_KEY=`$AWS_SECRET` - AWS_DEFAULT_REGION=us-east1 -- name: '@status' #This refers to another task and can pass args too +- follow: 'status' #This refers to another task and can pass args too args: 'prod' status: - image: 'mesosphere/aws-cli' - command: ['aws', 'elasticbeanstalk describe-events --environment-name $1'] + commands: + - ['aws', 'elasticbeanstalk describe-events --environment-name $1'] # This uses args passed to the task, `$1` means first arg envs: - AWS_ACCESS_KEY_ID=`$AWS_KEY` @@ -33,8 +38,8 @@ status: ``` Now you can use as, - 1. `Dunner do deploy` - 2. `Dunner do status prod` + 1. `dunner do deploy` + 2. `dunner do status prod` ## NOTE @@ -54,7 +59,7 @@ This work is still in progress. See the development plan. - [x] Ability to get ENV, param, etc values from host environment variables or `.env` file - [x] Ability to install as a Snap package -### [`v2.0`](https://github.com/leopardslab/Dunner/milestone/3) +### [`v2.0`](https://github.com/leopardslab/dunner/milestone/3) - [ ] Ability to Dry Run - [ ] Ability to verfiy the `.dunner.yaml` file - [ ] Ability to define multiple commands for the same step diff --git a/cmd/do.go b/cmd/do.go index ed414b3..f1b2069 100644 --- a/cmd/do.go +++ b/cmd/do.go @@ -10,11 +10,17 @@ func init() { rootCmd.AddCommand(doCmd) // Async Mode - doCmd.Flags().BoolP("async", "A", false, "Async mode") + doCmd.Flags().BoolP("async", "A", false, "Asynchronous mode") if err := viper.BindPFlag("Async", doCmd.Flags().Lookup("async")); err != nil { log.Fatal(err) } + // Dry-run mode + doCmd.Flags().Bool("dry-run", false, "Dry-run of the command") + if err := viper.BindPFlag("Dry-run", doCmd.Flags().Lookup("dry-run")); err != nil { + log.Fatal(err) + } + } var doCmd = &cobra.Command{ diff --git a/cmd/root.go b/cmd/root.go index 261c853..49f2390 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "os" @@ -18,13 +19,11 @@ var rootCmd = &cobra.Command{ Long: `You can define a set of commands and on what Docker images these commands should run as steps. A task has many steps. Then you can run these tasks with 'dunner do nameoftask'`, Run: func(cmd *cobra.Command, args []string) { - _, err := client.NewClientWithOpts( - client.FromEnv, - client.WithVersion(viper.GetString("DockerAPIVersion")), - ) + cli, err := client.NewClientWithOpts(client.FromEnv) if err != nil { log.Fatal(err) } + cli.NegotiateAPIVersion(context.Background()) fmt.Println("Dunner running!") }, diff --git a/cmd/validate.go b/cmd/validate.go new file mode 100644 index 0000000..893a906 --- /dev/null +++ b/cmd/validate.go @@ -0,0 +1,42 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/leopardslab/dunner/pkg/config" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func init() { + rootCmd.AddCommand(validateCmd) +} + +var validateCmd = &cobra.Command{ + Use: "validate", + Short: "Validate the dunner task file `.dunner.yaml`", + Long: "You can validate task file `.dunner.yaml` with this command to see if there are any parse errors", + Run: Validate, + Args: cobra.MinimumNArgs(0), +} + +// Validate command invoked from command line, validates the dunner task file. If there are errors, it fails with non-zero exit code. +func Validate(_ *cobra.Command, args []string) { + var dunnerFile = viper.GetString("DunnerTaskFile") + + configs, err := config.GetConfigs(dunnerFile) + if err != nil { + log.Fatal(err) + } + + errs := configs.Validate() + if len(errs) != 0 { + fmt.Println("Validation failed with following errors:") + for _, err := range errs { + fmt.Println(err.Error()) + } + os.Exit(1) + } + fmt.Println("Validation successful!") +} diff --git a/cmd/version.go b/cmd/version.go index 07017ec..1be06fa 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -2,8 +2,9 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" + G "github.com/leopardslab/dunner/pkg/global" + "github.com/spf13/cobra" ) func init() { diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 0f246b0..3d2d68a 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -4,7 +4,10 @@ import ( "github.com/spf13/viper" ) -func init() { +// Init function initializes the default settings for dunner +// These settings can tweaked using appropriate environment variables, or +// defining the configuration in conf present in the appropriate config files +func Init() { // Settings file viper.SetConfigName("settings") viper.SetConfigType("yaml") @@ -26,6 +29,7 @@ func init() { // Modes viper.SetDefault("Async", false) viper.SetDefault("Verbose", false) + viper.SetDefault("Dry-run", false) // Constants viper.SetDefault("DockerAPIVersion", "1.39") diff --git a/internal/util/util.go b/internal/util/util.go new file mode 100644 index 0000000..8eb8338 --- /dev/null +++ b/internal/util/util.go @@ -0,0 +1,22 @@ +package util + +import ( + "os" + "path" + "strings" +) + +// HomeDir is the environment variable HOME +var HomeDir = os.Getenv("HOME") + +// DirExists returns true if the given param is a valid existing directory +func DirExists(dir string) bool { + if strings.HasPrefix(dir, "~") { + dir = path.Join(HomeDir, strings.Trim(dir, "~")) + } + src, err := os.Stat(dir) + if err != nil { + return false + } + return src.IsDir() +} diff --git a/internal/util/util_test.go b/internal/util/util_test.go new file mode 100644 index 0000000..ce7993a --- /dev/null +++ b/internal/util/util_test.go @@ -0,0 +1,51 @@ +package util + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestDirExistsSuccess(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestDir") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + exists := DirExists(tmpdir) + + if !exists { + t.Fatalf("Directory exists; but got false") + } +} + +func TestDirExistsFail(t *testing.T) { + exists := DirExists("this path is invalid") + + if exists { + t.Fatalf("Directory invalid; but got as exists") + } +} + +func TestDirExistsFailForFile(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "TestFileExists") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpfile.Name()) + + exists := DirExists(tmpfile.Name()) + + if exists { + t.Fatalf("Not a directory; but got as true") + } +} + +func TestDirExistsIfNotAbsPath(t *testing.T) { + exists := DirExists("~/invalidpathfortesting") + + if exists { + t.Fatalf("Not a directory; but got as true") + } +} diff --git a/main.go b/main.go index bc95223..0bc36a2 100644 --- a/main.go +++ b/main.go @@ -2,12 +2,14 @@ package main import ( "github.com/leopardslab/dunner/cmd" + "github.com/leopardslab/dunner/internal/settings" G "github.com/leopardslab/dunner/pkg/global" ) var version string func main() { + settings.Init() G.VERSION = version cmd.Execute() } diff --git a/pkg/config/config.go b/pkg/config/config.go index aeaa83f..01b554f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,44 +1,167 @@ package config import ( + "context" "fmt" "io/ioutil" "os" "path" "path/filepath" + "reflect" "regexp" "strings" "github.com/docker/docker/api/types/mount" + "github.com/go-playground/locales/en" + ut "github.com/go-playground/universal-translator" "github.com/joho/godotenv" "github.com/leopardslab/dunner/internal/logger" + "github.com/leopardslab/dunner/internal/util" "github.com/leopardslab/dunner/pkg/docker" "github.com/spf13/viper" + "gopkg.in/go-playground/validator.v9" + en_translations "gopkg.in/go-playground/validator.v9/translations/en" yaml "gopkg.in/yaml.v2" ) var log = logger.Log -type DirMount struct { - Src string `yaml:"src"` - Dest string `yaml:"dest"` - ReadOnly bool `yaml:"read-only"` +var ( + uni *ut.UniversalTranslator + govalidator *validator.Validate + trans ut.Translator + defaultPermissionMode = "r" + validDirPermissionModes = []string{defaultPermissionMode, "wr", "rw", "w"} +) + +type contextKey string + +var configsKey = contextKey("dunnerConfigs") + +type customValidation struct { + tag string + translation string + validationFn func(context.Context, validator.FieldLevel) bool +} + +var customValidations = []customValidation{ + { + tag: "mountdir", + translation: "mount directory '{0}' is invalid. Check format is '::' and has right permission level", + validationFn: ValidateMountDir, + }, } // Task describes a single task to be run in a docker container type Task struct { - Name string `yaml:"name"` - Image string `yaml:"image"` - SubDir string `yaml:"dir"` - Command []string `yaml:"command"` - Envs []string `yaml:"envs"` - Mounts []string `yaml:"mounts"` - Args []string `yaml:"args"` + Name string `yaml:"name"` + Image string `yaml:"image" validate:"required"` + SubDir string `yaml:"dir"` + Command []string `yaml:"command" validate:"omitempty,dive,required"` + Commands [][]string `yaml:"commands"` + Envs []string `yaml:"envs"` + Mounts []string `yaml:"mounts" validate:"omitempty,dive,min=1,mountdir"` + Follow string `yaml:"follow"` + Args []string `yaml:"args"` } // Configs describes the parsed information from the dunner file type Configs struct { - Tasks map[string][]Task + Tasks map[string][]Task `validate:"required,min=1,dive,keys,required,endkeys,required,min=1,required"` +} + +// Validate validates config and returns errors. +func (configs *Configs) Validate() []error { + err := initValidator(customValidations) + if err != nil { + return []error{err} + } + valErrs := govalidator.Struct(configs) + errs := formatErrors(valErrs, "") + ctx := context.WithValue(context.Background(), configsKey, configs) + + // Each task is validated separately so that task name can be added in error messages + for taskName, tasks := range configs.Tasks { + taskValErrs := govalidator.VarCtx(ctx, tasks, "dive") + errs = append(errs, formatErrors(taskValErrs, taskName)...) + } + return errs +} + +func formatErrors(valErrs error, taskName string) []error { + var errs []error + if valErrs != nil { + if _, ok := valErrs.(*validator.InvalidValidationError); ok { + errs = append(errs, valErrs) + } else { + for _, e := range valErrs.(validator.ValidationErrors) { + if taskName == "" { + errs = append(errs, fmt.Errorf(e.Translate(trans))) + } else { + errs = append(errs, fmt.Errorf("task '%s': %s", taskName, e.Translate(trans))) + } + } + } + } + return errs +} + +func initValidator(customValidations []customValidation) error { + govalidator = validator.New() + govalidator.RegisterTagNameFunc(func(fld reflect.StructField) string { + name := strings.SplitN(fld.Tag.Get("yaml"), ",", 2)[0] + if name == "-" { + return "" + } + return name + }) + + // Register default translators + translator := en.New() + uni = ut.New(translator, translator) + var translatorFound bool + trans, translatorFound = uni.GetTranslator("en") + if !translatorFound { + return fmt.Errorf("failed to initialize validator with translator") + } + en_translations.RegisterDefaultTranslations(govalidator, trans) + + // Register Custom validators and translations + for _, t := range customValidations { + err := govalidator.RegisterValidationCtx(t.tag, t.validationFn) + if err != nil { + return fmt.Errorf("failed to register validation: %s", err.Error()) + } + err = govalidator.RegisterTranslation(t.tag, trans, registrationFunc(t.tag, t.translation), translateFunc) + if err != nil { + return fmt.Errorf("failed to register translations: %s", err.Error()) + } + } + return nil +} + +// ValidateMountDir verifies that mount values are in proper format :: +// Format should match, is optional which is `readOnly` by default and `src` directory exists in host machine +func ValidateMountDir(ctx context.Context, fl validator.FieldLevel) bool { + value := fl.Field().String() + f := func(c rune) bool { return c == ':' } + mountValues := strings.FieldsFunc(value, f) + if len(mountValues) != 3 { + mountValues = append(mountValues, defaultPermissionMode) + } + if len(mountValues) != 3 { + return false + } + validPerm := false + for _, perm := range validDirPermissionModes { + if mountValues[2] == perm { + validPerm = true + } + } + if !validPerm { + return false + } + return util.DirExists(mountValues[0]) } // GetConfigs reads and parses tasks from the dunner file @@ -126,21 +249,10 @@ func DecodeMount(mounts []string, step *docker.Step) error { strings.Trim(strings.Trim(m, `'`), `"`), ":", ) - if len(arr) != 3 && len(arr) != 2 { - return fmt.Errorf( - `config: invalid format for mount %s`, - m, - ) - } var readOnly = true if len(arr) == 3 { if arr[2] == "wr" || arr[2] == "w" { readOnly = false - } else if arr[2] != "r" { - return fmt.Errorf( - `config: invalid format of read-write mode for mount '%s'`, - m, - ) } } src, err := filepath.Abs(joinPathRelToHome(arr[0])) @@ -161,7 +273,24 @@ func DecodeMount(mounts []string, step *docker.Step) error { func joinPathRelToHome(p string) string { if p[0] == '~' { - return path.Join(os.Getenv("HOME"), strings.Trim(p, "~")) + return path.Join(util.HomeDir, strings.Trim(p, "~")) } return p } + +func registrationFunc(tag string, translation string) validator.RegisterTranslationsFunc { + return func(ut ut.Translator) (err error) { + if err = ut.Add(tag, translation, true); err != nil { + return + } + return + } +} + +func translateFunc(ut ut.Translator, fe validator.FieldError) string { + t, err := ut.T(fe.Tag(), reflect.ValueOf(fe.Value()).String(), fe.Param()) + if err != nil { + return fe.(error).Error() + } + return t +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 0058078..83ffb04 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "io/ioutil" "os" "reflect" @@ -13,7 +14,9 @@ func TestGetConfigs(t *testing.T) { var content = []byte(` test: - image: node - command: ["node", "--version"] + commands: + - ["node", "--version"] + - ["npm", "--version"] envs: - MYVAR=MYVAL`) @@ -38,10 +41,10 @@ test: } var task = Task{ - Name: "", - Image: "node", - Command: []string{"node", "--version"}, - Envs: []string{"MYVAR=MYVAL"}, + Name: "", + Image: "node", + Commands: [][]string{{"node", "--version"}, {"npm", "--version"}}, + Envs: []string{"MYVAR=MYVAL"}, } var tasks = make(map[string][]Task) tasks["test"] = []Task{task} @@ -54,3 +57,154 @@ test: } } + +func TestConfigs_Validate(t *testing.T) { + tasks := make(map[string][]Task, 0) + tasks["stats"] = []Task{getSampleTask()} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 0 { + t.Fatalf("Configs Validation failed, expected to pass. got: %s", errs) + } +} + +func TestConfigs_ValidateWithNoTasks(t *testing.T) { + tasks := make(map[string][]Task, 0) + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 1 { + t.Fatalf("Configs validation failed, expected 1 error, got %s", errs) + } + expected := "Tasks must contain at least 1 item" + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + +func TestConfigs_ValidateWithEmptyImageAndCommand(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := Task{Image: "", Command: []string{""}} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 2 { + t.Fatalf("expected 2 errors, got %d : %s", len(errs), errs) + } + + expected1 := "task 'stats': image is a required field" + expected2 := "task 'stats': command[0] is a required field" + if errs[0].Error() != expected1 { + t.Fatalf("expected: %s, got: %s", expected1, errs[0].Error()) + } + if errs[1].Error() != expected2 { + t.Fatalf("expected: %s, got: %s", expected2, errs[1].Error()) + } +} + +func TestConfigs_ValidateWithInvalidMountFormat(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + task.Mounts = []string{"invalid_dir"} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d : %s", len(errs), errs) + } + + expected := "task 'stats': mount directory 'invalid_dir' is invalid. Check format is '::' and has right permission level" + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + +func TestConfigs_ValidateWithValidMountDirectory(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + wd, _ := os.Getwd() + task.Mounts = []string{fmt.Sprintf("%s:%s:w", wd, wd)} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if errs != nil { + t.Fatalf("expected no errors, got %s", errs) + } +} + +func TestConfigs_ValidateWithNoModeGiven(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + wd, _ := os.Getwd() + task.Mounts = []string{fmt.Sprintf("%s:%s", wd, wd)} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if errs != nil { + t.Fatalf("expected no errors, got %s", errs) + } +} + +func TestConfigs_ValidateWithInvalidMode(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + wd, _ := os.Getwd() + task.Mounts = []string{fmt.Sprintf("%s:%s:ab", wd, wd)} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + expected := fmt.Sprintf("task 'stats': mount directory '%s' is invalid. Check format is '::' and has right permission level", task.Mounts[0]) + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + +func TestConfigs_ValidateWithInvalidMountDirectory(t *testing.T) { + tasks := make(map[string][]Task, 0) + task := getSampleTask() + task.Mounts = []string{"blah:foo:w"} + tasks["stats"] = []Task{task} + configs := &Configs{Tasks: tasks} + + errs := configs.Validate() + + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d : %s", len(errs), errs) + } + + expected := "task 'stats': mount directory 'blah:foo:w' is invalid. Check format is '::' and has right permission level" + if errs[0].Error() != expected { + t.Fatalf("expected: %s, got: %s", expected, errs[0].Error()) + } +} + +func getSampleTask() Task { + return Task{Image: "image_name", Command: []string{"node", "--version"}} +} + +func TestInitValidatorForNilTranslation(t *testing.T) { + vals := []customValidation{{tag: "foo", translation: "", validationFn: nil}} + + err := initValidator(vals) + + expected := "failed to register validation: Function cannot be empty" + if err == nil { + t.Fatalf("expected %s, got %s", expected, err) + } + if err.Error() != expected { + t.Fatalf("expected %s, got %s", expected, err.Error()) + } +} diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index a7dae36..027ac56 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -1,17 +1,22 @@ package docker import ( + "bytes" "context" + "fmt" "io" "io/ioutil" "os" "path/filepath" + "strings" + "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" "github.com/docker/docker/pkg/jsonmessage" + "github.com/docker/docker/pkg/stdcopy" "github.com/docker/docker/pkg/term" "github.com/leopardslab/dunner/internal/logger" "github.com/spf13/viper" @@ -25,30 +30,39 @@ type Step struct { Name string Image string Command []string + Commands [][]string Env []string WorkDir string Volumes map[string]string ExtMounts []mount.Mount + Follow string Args []string } +// Result stores the output of commands run using docker exec +type Result struct { + Command string + Output string + Error string +} + // Exec method is used to execute the task described in the corresponding step -func (step Step) Exec() (*io.ReadCloser, error) { +func (step Step) Exec() (*[]Result, error) { var ( hostMountFilepath = "./" containerDefaultWorkingDir = "/dunner" hostMountTarget = "/dunner" + defaultCommand = []string{"tail", "-f", "/dev/null"} + multipleCommands = false ) ctx := context.Background() - cli, err := client.NewClientWithOpts( - client.FromEnv, - client.WithVersion(viper.GetString("DockerAPIVersion")), - ) + cli, err := client.NewClientWithOpts(client.FromEnv) if err != nil { log.Fatal(err) } + cli.NegotiateAPIVersion(ctx) path, err := filepath.Abs(hostMountFilepath) if err != nil { @@ -56,7 +70,6 @@ func (step Step) Exec() (*io.ReadCloser, error) { } log.Infof("Pulling an image: '%s'", step.Image) - out, err := cli.ImagePull(ctx, step.Image, types.ImagePullOptions{}) if err != nil { log.Fatal(err) @@ -83,11 +96,15 @@ func (step Step) Exec() (*io.ReadCloser, error) { containerWorkingDir = filepath.Join(hostMountTarget, step.WorkDir) } + multipleCommands = len(step.Commands) > 0 + if !multipleCommands { + defaultCommand = step.Command + } resp, err := cli.ContainerCreate( ctx, &container.Config{ Image: step.Image, - Cmd: step.Command, + Cmd: defaultCommand, Env: step.Env, WorkingDir: containerWorkingDir, }, @@ -103,27 +120,94 @@ func (step Step) Exec() (*io.ReadCloser, error) { log.Fatal(err) } + if len(resp.Warnings) > 0 { + for warning := range resp.Warnings { + log.Warn(warning) + } + } + if err = cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { log.Fatal(err) } - statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) - select { - case err = <-errCh: + defer func() { + dur, err := time.ParseDuration("-1ns") // Negative duration means no force termination if err != nil { log.Fatal(err) } - case <-statusCh: + if err = cli.ContainerStop(ctx, resp.ID, &dur); err != nil { + log.Fatal(err) + } + }() + + var results []Result + if dryRun := viper.GetBool("Dry-run"); !dryRun { + if multipleCommands { + for _, cmd := range step.Commands { + r, err := runCmd(ctx, cli, resp.ID, cmd) + if err != nil { + log.Fatal(err) + } + results = append(results, *r) + } + } else { + statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) + select { + case err = <-errCh: + if err != nil { + log.Fatal(err) + } + case <-statusCh: + } + + out, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + if err != nil { + log.Fatal(err) + } + + results = []Result{*extractResult(out, step.Command)} + } + return &results, nil } + return nil, nil +} - out, err = cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ - ShowStdout: true, - ShowStderr: true, +func runCmd(ctx context.Context, cli *client.Client, containerID string, command []string) (*Result, error) { + if len(command) == 0 { + return nil, fmt.Errorf(`config: Command cannot be empty`) + } + + exec, err := cli.ContainerExecCreate(ctx, containerID, types.ExecConfig{ + Cmd: command, + AttachStdout: true, + AttachStderr: true, }) if err != nil { log.Fatal(err) } - return &out, nil + resp, err := cli.ContainerExecAttach(ctx, exec.ID, types.ExecStartCheck{}) + if err != nil { + log.Fatal(err) + } + defer resp.Close() + return extractResult(resp.Reader, command), nil +} + +func extractResult(reader io.Reader, command []string) *Result { + + var out, errOut bytes.Buffer + if _, err := stdcopy.StdCopy(&out, &errOut, reader); err != nil { + log.Fatal(err) + } + var result = Result{ + Command: strings.Join(command, " "), + Output: out.String(), + Error: errOut.String(), + } + return &result } diff --git a/pkg/docker/docker_test.go b/pkg/docker/docker_test.go index a8fd7d9..886b4bb 100644 --- a/pkg/docker/docker_test.go +++ b/pkg/docker/docker_test.go @@ -1,35 +1,30 @@ package docker import ( - "bytes" "strings" "testing" + + "github.com/leopardslab/dunner/internal/settings" ) func TestStep_Do(t *testing.T) { - + settings.Init() var testNodeVersion = "10.15.0" - step := &Step{ - Task: "test", - Name: "node", - Image: "node:" + testNodeVersion, - Command: []string{"node", "--version"}, - Env: nil, - Volumes: nil, + Task: "test", + Name: "node", + Image: "node:" + testNodeVersion, + Commands: [][]string{{"node", "--version"}}, + Env: nil, + Volumes: nil, } - pout, err := step.Exec() - if err != nil { - t.Error(err) - } - buffer := new(bytes.Buffer) - _, err = buffer.ReadFrom(*pout) + results, err := step.Exec() if err != nil { t.Error(err) } - strOut := buffer.String() + strOut := (*results)[0].Output var result = strings.Trim(strings.Split(strOut, "v")[1], "\n") if result != testNodeVersion { t.Fatalf("Detected version of node container: '%s'; Expected output: '%s'", result, testNodeVersion) diff --git a/pkg/dunner/dunner.go b/pkg/dunner/dunner.go index c30e2ce..150c461 100644 --- a/pkg/dunner/dunner.go +++ b/pkg/dunner/dunner.go @@ -1,13 +1,13 @@ package dunner import ( + "fmt" "os" "regexp" "strconv" "strings" "sync" - "github.com/docker/docker/pkg/stdcopy" "github.com/leopardslab/dunner/internal/logger" "github.com/leopardslab/dunner/pkg/config" "github.com/leopardslab/dunner/pkg/docker" @@ -32,6 +32,14 @@ func Do(_ *cobra.Command, args []string) { if err != nil { log.Fatal(err) } + errs := configs.Validate() + if len(errs) != 0 { + fmt.Println("Validation failed with following errors:") + for _, err := range errs { + fmt.Println(err.Error()) + } + os.Exit(1) + } execTask(configs, args[0], args[1:]) } @@ -44,13 +52,15 @@ func execTask(configs *config.Configs, taskName string, args []string) { wg.Add(1) } step := docker.Step{ - Task: taskName, - Name: stepDefinition.Name, - Image: stepDefinition.Image, - Command: stepDefinition.Command, - Env: stepDefinition.Envs, - WorkDir: stepDefinition.SubDir, - Args: stepDefinition.Args, + Task: taskName, + Name: stepDefinition.Name, + Image: stepDefinition.Image, + Command: stepDefinition.Command, + Commands: stepDefinition.Commands, + Env: stepDefinition.Envs, + WorkDir: stepDefinition.SubDir, + Follow: stepDefinition.Follow, + Args: stepDefinition.Args, } if err := config.DecodeMount(stepDefinition.Mounts, &step); err != nil { @@ -73,16 +83,15 @@ func process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args [ defer wg.Done() } - if newTask := regexp.MustCompile(`^@\w+$`).FindString(s.Name); newTask != "" { - newTask = strings.Trim(newTask, "@") + if s.Follow != "" { if async { wg.Add(1) go func(wg *sync.WaitGroup) { - execTask(configs, newTask, s.Args) + execTask(configs, s.Follow, s.Args) wg.Done() }(wg) } else { - execTask(configs, newTask, s.Args) + execTask(configs, s.Follow, s.Args) } return } @@ -95,38 +104,44 @@ func process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args [ log.Fatalf(`dunner: image repository name cannot be empty`) } - pout, err := (*s).Exec() + results, err := (*s).Exec() if err != nil { log.Fatal(err) } - log.Infof( - "Running task '%+v' on '%+v' Docker with command '%+v'", - s.Task, - s.Image, - strings.Join(s.Command, " "), - ) - - if _, err = stdcopy.StdCopy(os.Stdout, os.Stderr, *pout); err != nil { - log.Fatal(err) + if results == nil { + return } - if err = (*pout).Close(); err != nil { - log.Fatal(err) + for _, res := range *results { + log.Infof( + "Running task '%+v' on '%+v' Docker with command '%+v'", + s.Task, + s.Image, + res.Command, + ) + if res.Output != "" { + fmt.Printf(`OUT: %s`, res.Output) + } + if res.Error != "" { + fmt.Printf(`ERR: %s`, res.Error) + } } } func passArgs(s *docker.Step, args *[]string) error { - for i, subStr := range s.Command { - regex := regexp.MustCompile(`\$[1-9][0-9]*`) - subStr = regex.ReplaceAllStringFunc(subStr, func(str string) string { - j, err := strconv.Atoi(strings.Trim(str, "$")) - if err != nil { - log.Fatal(err) - } - return (*args)[j-1] - }) - s.Command[i] = subStr + for i, cmd := range s.Commands { + for j, subStr := range cmd { + regex := regexp.MustCompile(`\$[1-9][0-9]*`) + subStr = regex.ReplaceAllStringFunc(subStr, func(str string) string { + j, err := strconv.Atoi(strings.Trim(str, "$")) + if err != nil { + log.Fatal(err) + } + return (*args)[j-1] + }) + s.Commands[i][j] = subStr + } } return nil } diff --git a/snap.login.enc b/snap.login.enc new file mode 100644 index 0000000..ec5c6b7 Binary files /dev/null and b/snap.login.enc differ diff --git a/snapcraft.yaml b/snapcraft.yaml index 2cbd30a..9ca7070 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -4,7 +4,8 @@ summary: A Docker based task runner tool description: | Dunner is a task runner tool like Grunt but used Docker images like CircleCI do. | You can define tasks and steps of the tasks in your `.dunner.yaml` file and then run these steps with `Dunner do taskname` -confinement: devmode +grade: stable +confinement: strict base: core18 parts: @@ -18,4 +19,4 @@ parts: apps: dunner: - command: Dunner + command: bin/dunner