feat(config): add Validators to ExportedConfigs and "config verify" c…

…ommand. (#154)

* feat: add verification command

* test: pass tests

* test: fix failed tests

* feat: support validators

* test: add integration test

* feat: add default config validation

* fix: optimize test coverage

* fix: optimize test coverage

* fix: optimize test coverage

* fix: json init issue

* fix: json init issue

* fix: remove version in config

.gitignore

@@ -3,3 +3,4 @@ core.iml
 .idea/*
 /.idea/
 .env
+config/testdata/module_test_partial.json

config/config.go

@@ -16,6 +16,7 @@ import (
 // KoanfAdapter is a implementation of contract.Config based on Koanf (https://github.com/knadh/koanf).
 type KoanfAdapter struct {
 	layers     []ProviderSet
+	validators []Validator
 	watcher    contract.ConfigWatcher
 	dispatcher contract.Dispatcher
 	delimiter  string
@@ -62,6 +63,13 @@ func WithDispatcher(dispatcher contract.Dispatcher) Option {
 	}
 }
 
+// WithValidators changes the validators of Koanf.
+func WithValidators(validators ...Validator) Option {
+	return func(option *KoanfAdapter) {
+		option.validators = validators
+	}
+}
+
 // NewConfig creates a new *KoanfAdapter.
 func NewConfig(options ...Option) (*KoanfAdapter, error) {
 	adapter := KoanfAdapter{delimiter: "."}
@@ -83,19 +91,29 @@ func NewConfig(options ...Option) (*KoanfAdapter, error) {
 // an error occurred, Reload will return early and abort the rest of the
 // reloading.
 func (k *KoanfAdapter) Reload() error {
-	if k.dispatcher != nil {
-		defer k.dispatcher.Dispatch(context.Background(), events.Of(events.OnReload{NewConf: k}))
-	}
-
-	k.rwlock.Lock()
-	defer k.rwlock.Unlock()
+	var tmp = koanf.New(".")
 
 	for i := len(k.layers) - 1; i >= 0; i-- {
-		err := k.K.Load(k.layers[i].Provider, k.layers[i].Parser)
+		err := tmp.Load(k.layers[i].Provider, k.layers[i].Parser)
 		if err != nil {
 			return fmt.Errorf("unable to load config %w", err)
 		}
 	}
+
+	for _, f := range k.validators {
+		if err := f(tmp.Raw()); err != nil {
+			return fmt.Errorf("validation failed: %w", err)
+		}
+	}
+
+	k.rwlock.Lock()
+	k.K = tmp
+	k.rwlock.Unlock()
+
+	if k.dispatcher != nil {
+		k.dispatcher.Dispatch(context.Background(), events.Of(events.OnReload{NewConf: k}))
+	}
+
 	return nil
 }
 

config/config_test.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -219,6 +220,17 @@ func TestMapAdapter_Unmarshal(t *gotesting.T) {
 	}, target)
 }
 
+func TestKoanfAdapter_Reload(t *gotesting.T) {
+	t.Parallel()
+	conf, err := NewConfig(
+		WithValidators(func(data map[string]interface{}) error {
+			return errors.New("bad config")
+		}),
+	)
+	assert.Error(t, err)
+	assert.Nil(t, conf)
+}
+
 func prepareJSONTestSubject(t *gotesting.T) *KoanfAdapter {
 	k := koanf.New(".")
 	if err := k.Load(file.Provider("testdata/mock.json"), json.Parser()); err != nil {

config/exported_config.go

@@ -3,7 +3,12 @@ package config
 // ExportedConfig is a struct that outlines a set of configuration.
 // Each module is supposed to emit ExportedConfig into DI, and Package config should collect them.
 type ExportedConfig struct {
-	Owner   string
-	Data    map[string]interface{}
-	Comment string
+	Owner    string
+	Data     map[string]interface{}
+	Comment  string
+	Validate Validator
 }
+
+// Validator is a method to verify if config is valid. If it is not valid, the
+// returned error should contain a human readable description of why.
+type Validator func(data map[string]interface{}) error

config/integration_test.go

@@ -0,0 +1,40 @@
+package config_test
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/DoNewsCode/core"
+	"github.com/DoNewsCode/core/config"
+	"github.com/DoNewsCode/core/di"
+)
+
+func provideConfig() configOut {
+	return configOut{
+		Config: []config.ExportedConfig{{
+			Validate: func(data map[string]interface{}) error {
+				return errors.New("bad config")
+			},
+		}},
+	}
+}
+
+type configOut struct {
+	di.Out
+
+	Config []config.ExportedConfig `group:"config,flatten"`
+}
+
+func Test_integration(t *testing.T) {
+	defer func() {
+		if r := recover(); r != nil {
+			return
+		}
+		t.Errorf("test should panic. the config is not valid.")
+	}()
+	c := core.Default()
+	c.Provide(di.Deps{
+		provideConfig,
+	})
+	c.AddModuleFunc(config.New)
+}

config/module.go

@@ -44,6 +44,11 @@ func New(p ConfigIn) (Module, error) {
 	if adapter, ok = p.Conf.(*KoanfAdapter); !ok {
 		return Module{}, fmt.Errorf("expects a *config.KoanfAdapter instance, but %T given", p.Conf)
 	}
+
+	if err := loadValidators(adapter, p.ExportedConfigs); err != nil {
+		return Module{}, err
+	}
+
 	return Module{
 		dispatcher:      p.Dispatcher,
 		conf:            adapter,
@@ -65,8 +70,8 @@ func (m Module) ProvideRunGroup(group *run.Group) {
 // ProvideCommand provides the config related command.
 func (m Module) ProvideCommand(command *cobra.Command) {
 	var (
-		outputFile string
-		style      string
+		targetFilePath string
+		style          string
 	)
 	initCmd := &cobra.Command{
 		Use:   "init [module]",
@@ -99,8 +104,8 @@ func (m Module) ProvideCommand(command *cobra.Command) {
 				}
 				exportedConfigs = copy
 			}
-			os.MkdirAll(filepath.Dir(outputFile), os.ModePerm)
-			targetFile, err = os.OpenFile(outputFile,
+			os.MkdirAll(filepath.Dir(targetFilePath), os.ModePerm)
+			targetFile, err = os.OpenFile(targetFilePath,
 				handler.flags(), os.ModePerm)
 			if err != nil {
 				return errors.Wrap(err, "failed to open config file")
@@ -121,29 +126,111 @@ func (m Module) ProvideCommand(command *cobra.Command) {
 			return nil
 		},
 	}
-	initCmd.Flags().StringVarP(
-		&outputFile,
+
+	verifyCmd := &cobra.Command{
+		Use:   "verify [module]",
+		Short: "verify the config file is correct.",
+		Long:  "verify the config file is correct based on the methods exported by modules.",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var (
+				handler         handler
+				targetFile      *os.File
+				exportedConfigs []ExportedConfig
+				confMap         map[string]interface{}
+				err             error
+			)
+			handler, err = getHandler(style)
+			if err != nil {
+				return err
+			}
+			if len(args) == 0 {
+				exportedConfigs = m.exportedConfigs
+			}
+			if len(args) >= 1 {
+				var copy = make([]ExportedConfig, 0)
+				for i := range m.exportedConfigs {
+					for j := 0; j < len(args); j++ {
+						if args[j] == m.exportedConfigs[i].Owner {
+							copy = append(copy, m.exportedConfigs[i])
+							break
+						}
+					}
+				}
+				exportedConfigs = copy
+			}
+			os.MkdirAll(filepath.Dir(targetFilePath), os.ModePerm)
+			targetFile, err = os.OpenFile(targetFilePath,
+				handler.flags(), os.ModePerm)
+			if err != nil {
+				return errors.Wrap(err, "failed to open config file")
+			}
+			defer targetFile.Close()
+			bytes, err := ioutil.ReadAll(targetFile)
+			if err != nil {
+				return errors.Wrap(err, "failed to read config file")
+			}
+			err = handler.unmarshal(bytes, &confMap)
+			if err != nil {
+				return errors.Wrap(err, "failed to unmarshal config file")
+			}
+			for _, config := range exportedConfigs {
+				if config.Validate == nil {
+					continue
+				}
+				if err := config.Validate(confMap); err != nil {
+					return errors.Wrap(err, "invalid config")
+				}
+			}
+			return nil
+		},
+	}
+
+	configCmd := &cobra.Command{
+		Use:   "config",
+		Short: "manage configuration",
+		Long:  "manage configuration, such as export a copy of default config.",
+	}
+	configCmd.PersistentFlags().StringVarP(
+		&targetFilePath,
 		"outputFile",
 		"o",
 		"./config/config.yaml",
-		"The output file of exported config",
+		"The output file of exported config (alias of targetFile)",
 	)
-	initCmd.Flags().StringVarP(
+	configCmd.PersistentFlags().StringVarP(
+		&targetFilePath,
+		"targetFile",
+		"t",
+		"./config/config.yaml",
+		"The targeted config file",
+	)
+	configCmd.PersistentFlags().StringVarP(
 		&style,
 		"style",
 		"s",
 		"yaml",
 		"The output file style",
 	)
-	configCmd := &cobra.Command{
-		Use:   "config",
-		Short: "manage configuration",
-		Long:  "manage configuration, such as export a copy of default config.",
-	}
 	configCmd.AddCommand(initCmd)
+	configCmd.AddCommand(verifyCmd)
 	command.AddCommand(configCmd)
 }
 
+func loadValidators(k *KoanfAdapter, exportedConfigs []ExportedConfig) error {
+	for _, config := range exportedConfigs {
+		if config.Validate == nil {
+			continue
+		}
+		k.validators = append(k.validators, config.Validate)
+	}
+	for _, f := range k.validators {
+		if err := f(k.K.Raw()); err != nil {
+			return fmt.Errorf("invalid config: %w", err)
+		}
+	}
+	return nil
+}
+
 func getHandler(style string) (handler, error) {
 	switch style {
 	case "json":
@@ -221,10 +308,10 @@ func (y jsonHandler) write(file *os.File, configs []ExportedConfig, confMap map[
 		confMap = make(map[string]interface{})
 	}
 	for _, exportedConfig := range configs {
-		if _, ok := confMap[exportedConfig.Owner]; ok {
-			continue
-		}
 		for k := range exportedConfig.Data {
+			if _, ok := confMap[k]; ok {
+				continue
+			}
 			confMap[k] = exportedConfig.Data[k]
 		}
 	}