From b67b32ad292c90fe512e9ca808b733b7350376ad Mon Sep 17 00:00:00 2001 From: Ayush Jain Date: Thu, 27 Jun 2019 14:56:38 +0530 Subject: [PATCH] Add suitable comments for GoDoc --- pkg/config/config.go | 89 ++++++++++++++++++++++++++++++++------- pkg/config/config_test.go | 2 +- pkg/docker/docker.go | 45 ++++++++++++-------- pkg/dunner/dunner.go | 24 +++++++---- 4 files changed, 117 insertions(+), 43 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 9fbaa65..233055d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,3 +1,28 @@ +/* +Package config is the YAML parser of the task file for Dunner. + +For more information on how to write a task file for Dunner, please refer to the +following link of an article on Dunner repository's Wiki: +https://github.com/leopardslab/dunner/dunner/wiki/User-Guide#how-to-write-a-dunner-file + +Usage + +You can use the library by creating a dunner task file. For example, + # .dunner.yaml + prepare: + - image: node + commands: + - ["node", "--version"] + - image: node + commands: + - ["npm", "install"] + - image: mvn + commands: + - ["mvn", "package"] + +Use `GetConfigs` method to parse the dunner task file, and `ParseEnv` method to parse environment variables file, or +the host environment variables. The environment variables are used by invoking in the task file using backticks(`$var`). +*/ package config import ( @@ -65,18 +90,36 @@ var customValidations = []customValidation{ // Task describes a single task to be run in a docker container type Task struct { - Name string `yaml:"name"` - Image string `yaml:"image" validate:"required"` - SubDir string `yaml:"dir"` - Command []string `yaml:"command" validate:"omitempty,dive,required"` + // Name given as string to identify the task + Name string `yaml:"name"` + + // Image is the repo name on which Docker containers are built + Image string `yaml:"image" validate:"required"` + + // SubDir is the primary directory on which task is to be run + SubDir string `yaml:"dir"` + + // The command which runs on the container and exits + Command []string `yaml:"command" validate:"omitempty,dive,required"` + + // The list of commands that are to be run in sequence Commands [][]string `yaml:"commands" validate:"omitempty,dive,omitempty,dive,required"` - Envs []string `yaml:"envs"` - Mounts []string `yaml:"mounts" validate:"omitempty,dive,min=1,mountdir,parsedir"` - Follow string `yaml:"follow" validate:"omitempty,follow_exist"` - Args []string `yaml:"args"` + + // The list of environment variables to be exported inside the container + Envs []string `yaml:"envs"` + + // The directories to be mounted on the container as bind volumes + Mounts []string `yaml:"mounts" validate:"omitempty,dive,min=1,mountdir,parsedir"` + + // The next task that must be executed if this does go successfully + Follow string `yaml:"follow" validate:"omitempty,follow_exist"` + + // The list of arguments that are to be passed + Args []string `yaml:"args"` } -// Configs describes the parsed information from the dunner file +// Configs describes the parsed information from the dunner file. It is a map of task name as keys and the list of tasks +// associated with it. type Configs struct { Tasks map[string][]Task `validate:"required,min=1,dive,keys,required,endkeys,required,min=1,required"` } @@ -151,8 +194,9 @@ func initValidator(customValidations []customValidation) error { return nil } -// ValidateMountDir verifies that mount values are in proper format :: -// Format should match, is optional which is `readOnly` by default +// ValidateMountDir verifies that mount values are in proper format +// :: +// Format should match, is optional which is `readOnly` by default and `src` directory exists in host machine func ValidateMountDir(ctx context.Context, fl validator.FieldLevel) bool { value := fl.Field().String() f := func(c rune) bool { return c == ':' } @@ -199,7 +243,10 @@ func ParseMountDir(ctx context.Context, fl validator.FieldLevel) bool { return util.DirExists(parsedDir) } -// GetConfigs reads and parses tasks from the dunner file +// GetConfigs reads and parses tasks from the dunner task file. +// The task file is unmarshalled to an object of struct `Config` +// The default filename that is being read by Dunner during the time of execution is `dunner.yaml`, +// but it can be changed using `--task-file` flag in the CLI. func GetConfigs(filename string) (*Configs, error) { fileContents, err := ioutil.ReadFile(filename) if err != nil { @@ -212,7 +259,7 @@ func GetConfigs(filename string) (*Configs, error) { } loadDotEnv() - if err := parseEnv(&configs); err != nil { + if err := ParseEnv(&configs); err != nil { log.Fatal(err) } @@ -228,7 +275,12 @@ func loadDotEnv() { } } -func parseEnv(configs *Configs) error { +// ParseEnv parses the `.env` file as well as the host environment variables. +// If the same variable is defined in both the `.env` file and in the host environment, +// priority is given to the .env file. +// +// Note: You can change the filename of environment file (default: `.env`) using `--env-file/-e` flag in the CLI. +func ParseEnv(configs *Configs) error { for k, tasks := range (*configs).Tasks { for j, task := range tasks { for i, envVar := range task.Envs { @@ -257,6 +309,8 @@ func parseEnv(configs *Configs) error { 1, ) var val string + // Value of variable defined in environment file (default '.env') overrides + // the value defined in host's environment variables. if v, isSet := os.LookupEnv(key); isSet { val = v } @@ -280,7 +334,10 @@ func parseEnv(configs *Configs) error { return nil } -// DecodeMount parses mount format for directories to be mounted as bind volumes +// DecodeMount parses mount format for directories to be mounted as bind volumes. +// The format to configure a mount is +// :: +// By _mode_, the file permission level is defined in two ways, viz., _read-only_ mode(`r`) and _read-write_ mode(`wr` or `w`) func DecodeMount(mounts []string, step *docker.Step) error { for _, m := range mounts { arr := strings.Split( @@ -333,7 +390,7 @@ func lookupDirectory(dir string) (string, error) { val = v } if val == "" { - return dir, fmt.Errorf(`Could not find environment variable '%v'`, envKey) + return dir, fmt.Errorf(`could not find environment variable '%v'`, envKey) } parsedDir = strings.Replace(parsedDir, fmt.Sprintf("`$%s`", envKey), val, -1) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index a25982d..4bd7b12 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -296,7 +296,7 @@ var lookupEnvtests = []struct { {"`$HOME`", util.HomeDir, nil}, {"`$HOME`/foo", util.HomeDir + "/foo", nil}, {"`$HOME`/foo/`$HOME`", util.HomeDir + "/foo/" + util.HomeDir, nil}, - {"`$INVALID_TEST`/foo", "`$INVALID_TEST`/foo", fmt.Errorf("Could not find environment variable 'INVALID_TEST'")}, + {"`$INVALID_TEST`/foo", "`$INVALID_TEST`/foo", fmt.Errorf("could not find environment variable 'INVALID_TEST'")}, } func TestLookUpDirectory(t *testing.T) { diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index 027ac56..c6e0aef 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -1,3 +1,7 @@ +/* +Package docker is the interface of dunner to communicate with the Docker Engine through +methods wrapping over Docker client library. +*/ package docker import ( @@ -24,29 +28,34 @@ import ( var log = logger.Log -// Step describes the information required to run one task in docker container +// Step describes the information required to run one task in docker container. It is very similar to the concept +// of docker build of a 'Dockerfile' and then a sequence of commands to be executed in `docker run`. type Step struct { - Task string - Name string - Image string - Command []string - Commands [][]string - Env []string - WorkDir string - Volumes map[string]string - ExtMounts []mount.Mount - Follow string - Args []string + Task string // The name of the task that the step corresponds to + Name string // Name given to this step for identification purpose + Image string // Image is the repo name on which Docker containers are built + Command []string // The command which runs on the container and exits + Commands [][]string // The list of commands that are to be run in sequence + Env []string // The list of environment variables to be exported inside the container + WorkDir string // The primary directory on which task is to be run + Volumes map[string]string // Volumes that are to be attached to the container + ExtMounts []mount.Mount // The directories to be mounted on the container as bind volumes + Follow string // The next task that must be executed if this does go successfully + Args []string // The list of arguments that are to be passed } -// Result stores the output of commands run using docker exec +// Result stores the output of commands run using `docker exec` type Result struct { Command string Output string Error string } -// Exec method is used to execute the task described in the corresponding step +// Exec method is used to execute the task described in the corresponding step. It returns an object of the +// struct `Result` with the corresponding output and/or error. +// +// Note: A working internet connection is mandatory for the Docker container to contact Docker Hub to find the image and/or +// corresponding updates. func (step Step) Exec() (*[]Result, error) { var ( @@ -168,7 +177,7 @@ func (step Step) Exec() (*[]Result, error) { log.Fatal(err) } - results = []Result{*extractResult(out, step.Command)} + results = []Result{*ExtractResult(out, step.Command)} } return &results, nil } @@ -195,10 +204,12 @@ func runCmd(ctx context.Context, cli *client.Client, containerID string, command } defer resp.Close() - return extractResult(resp.Reader, command), nil + return ExtractResult(resp.Reader, command), nil } -func extractResult(reader io.Reader, command []string) *Result { +// ExtractResult can parse output and/or error corresponding to the command passed as an argument, +// from an io.Reader and convert to an object of strings. +func ExtractResult(reader io.Reader, command []string) *Result { var out, errOut bytes.Buffer if _, err := stdcopy.StdCopy(&out, &errOut, reader); err != nil { diff --git a/pkg/dunner/dunner.go b/pkg/dunner/dunner.go index 150c461..222b518 100644 --- a/pkg/dunner/dunner.go +++ b/pkg/dunner/dunner.go @@ -1,3 +1,6 @@ +/* +Package dunner consists of the main executing functions for the Dunner application. +*/ package dunner import ( @@ -41,10 +44,11 @@ func Do(_ *cobra.Command, args []string) { os.Exit(1) } - execTask(configs, args[0], args[1:]) + ExecTask(configs, args[0], args[1:]) } -func execTask(configs *config.Configs, taskName string, args []string) { +// ExecTask processes the parsed tasks from the dunner task file +func ExecTask(configs *config.Configs, taskName string, args []string) { var async = viper.GetBool("Async") var wg sync.WaitGroup for _, stepDefinition := range configs.Tasks[taskName] { @@ -68,16 +72,17 @@ func execTask(configs *config.Configs, taskName string, args []string) { } if async { - go process(configs, &step, &wg, args) + go Process(configs, &step, &wg, args) } else { - process(configs, &step, &wg, args) + Process(configs, &step, &wg, args) } } wg.Wait() } -func process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args []string) { +// Process executes a single step of the task. +func Process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args []string) { var async = viper.GetBool("Async") if async { defer wg.Done() @@ -87,16 +92,16 @@ func process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args [ if async { wg.Add(1) go func(wg *sync.WaitGroup) { - execTask(configs, s.Follow, s.Args) + ExecTask(configs, s.Follow, s.Args) wg.Done() }(wg) } else { - execTask(configs, s.Follow, s.Args) + ExecTask(configs, s.Follow, s.Args) } return } - if err := passArgs(s, &args); err != nil { + if err := PassArgs(s, &args); err != nil { log.Fatal(err) } @@ -129,7 +134,8 @@ func process(configs *config.Configs, s *docker.Step, wg *sync.WaitGroup, args [ } } -func passArgs(s *docker.Step, args *[]string) error { +// PassArgs replaces argument variables,of the form '`$d`', where d is a number, with dth argument. +func PassArgs(s *docker.Step, args *[]string) error { for i, cmd := range s.Commands { for j, subStr := range cmd { regex := regexp.MustCompile(`\$[1-9][0-9]*`)