cmd/catalyst: implement YAML configuration stack parsing

cmd/catalyst/catalyst.go

@@ -1,13 +1,17 @@
 package main
 
 import (
+	"encoding/json"
+	"fmt"
 	"os"
 	"syscall"
 
+	"github.com/icza/dyno"
 	"github.com/livepeer/catalyst/cmd/downloader/cli"
 	"github.com/livepeer/catalyst/cmd/downloader/downloader"
 	"github.com/livepeer/catalyst/cmd/downloader/types"
 	glog "github.com/magicsong/color-glog"
+	"gopkg.in/yaml.v3"
 )
 
 var Version = "undefined"
@@ -34,6 +38,11 @@ func execNext(cliFlags types.CliFlags) {
 		// Nothing to do.
 		return
 	}
+	configStr, err := handleConfigFile("/home/iameli/code/catalyst/config/full-stack.yaml")
+	if err != nil {
+		panic(err)
+	}
+	panic(configStr)
 	glog.Infof("downloader complete, now we will exec %v", cliFlags.ExecCommand)
 	execErr := syscall.Exec(cliFlags.ExecCommand[0], cliFlags.ExecCommand, os.Environ())
 	if execErr != nil {

cmd/catalyst/config/config.go

@@ -0,0 +1,149 @@
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/icza/dyno"
+	"gopkg.in/yaml.v2"
+)
+
+// takes /path1:/path2:/path3 and returns JSON bytes
+func HandleConfigStack(configPaths string) ([]byte, error) {
+	var err error
+	merged := map[string]any{}
+	filePaths := strings.Split(configPaths, ":")
+	for _, filePath := range filePaths {
+		contents, err := readYAMLFile(filePath)
+		// todo: handle missing file case (allowed as long as we have some)
+		if err != nil {
+			return []byte{}, fmt.Errorf("error handling config file %s: %w", filePath, err)
+		}
+		merged = mergeMaps(merged, contents)
+	}
+	config, err := optionalMap(merged, "config")
+	if err != nil {
+		return nil, err
+	}
+	protocols, err := optionalMap(config, "protocols")
+	if err != nil {
+		return nil, err
+	}
+	protocolArray := []map[string]any{}
+	for k, v := range protocols {
+		if v == nil {
+			continue
+		}
+		vMap, ok := v.(map[string]any)
+		if !ok {
+			return nil, fmt.Errorf("unable to convert protocol '%s' to a string map", k)
+		}
+		protocolArray = append(protocolArray, vMap)
+	}
+	config["protocols"] = protocolArray
+	jsonBytes, err := json.MarshalIndent(merged, "", "  ")
+	if err != nil {
+		return nil, err
+	}
+	return jsonBytes, nil
+}
+
+// Returns a new map merging source into dest
+// Merges any map[string]any maps that are present
+// Overwrites everything else
+func mergeMaps(dest, source map[string]any) map[string]any {
+	merged := map[string]any{}
+	// Start with a shallow copy of `dest`
+	for k, v := range dest {
+		merged[k] = v
+	}
+	for newKey, newValue := range source {
+		oldValue, has := merged[newKey]
+		if !has {
+			merged[newKey] = newValue
+			continue
+		}
+		newMap, newOk := newValue.(map[string]any)
+		oldMap, oldOk := oldValue.(map[string]any)
+		if newOk && oldOk {
+			// Both maps. Merge em!
+			merged[newKey] = mergeMaps(oldMap, newMap)
+			continue
+		}
+		// One or both is not a map, just copy over the new value
+		merged[newKey] = newValue
+	}
+	return merged
+}
+
+func readYAMLFile(filePath string) (map[string]any, error) {
+	var conf map[any]any
+	dat, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+	err = yaml.Unmarshal(dat, &conf)
+	if err != nil {
+		return nil, err
+	}
+	jsonConf := dyno.ConvertMapI2MapS(conf)
+	jsonMap, ok := jsonConf.(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("unable to convert config to a string map")
+	}
+	return jsonMap, nil
+}
+
+func optionalMap(parent map[string]any, key string) (map[string]any, error) {
+	child, ok := parent[key]
+	if !ok {
+		child = map[string]any{}
+		parent[key] = child
+	}
+	childMap, ok := child.(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("unable to convert '%s' to a string map", key)
+	}
+	return childMap, nil
+}
+
+func handleConfigFile(configPath string) (string, error) {
+	var conf map[any]any
+	dat, err := os.ReadFile(configPath)
+	if err != nil {
+		return "", err
+	}
+	err = yaml.Unmarshal(dat, &conf)
+	if err != nil {
+		return "", err
+	}
+	jsonConf := dyno.ConvertMapI2MapS(conf)
+	jsonMap, ok := jsonConf.(map[string]any)
+	if !ok {
+		return "", fmt.Errorf("unable to convert config to a string map")
+	}
+	config, err := optionalMap(jsonMap, "config")
+	if err != nil {
+		return "", err
+	}
+	protocols, err := optionalMap(config, "protocols")
+	if err != nil {
+		return "", err
+	}
+	protocolArray := []map[string]any{}
+	for k, v := range protocols {
+		vMap, ok := v.(map[string]any)
+		if !ok {
+			return "", fmt.Errorf("unable to convert protocol '%s' to a string map", k)
+		}
+		protocolArray = append(protocolArray, vMap)
+	}
+	config["protocols"] = protocolArray
+	str, err := json.MarshalIndent(jsonConf, "", "  ")
+	if err != nil {
+		return "", err
+	}
+	return string(str), nil
+}

cmd/catalyst/config/config_test.go

@@ -0,0 +1,89 @@
+package config
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/icza/dyno"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
+)
+
+func randPath(t *testing.T) string {
+	randBytes := make([]byte, 16)
+	rand.Read(randBytes)
+	return filepath.Join(t.TempDir(), hex.EncodeToString(randBytes)+".yaml")
+}
+
+func toFiles(t *testing.T, strs ...string) string {
+	paths := []string{}
+	for _, content := range strs {
+		filepath := randPath(t)
+		os.WriteFile(filepath, []byte(content), 0644)
+		paths = append(paths, filepath)
+	}
+	return strings.Join(paths, ":")
+}
+
+func yamlToJson(t *testing.T, yamlStr string) string {
+	var yamlStruct map[any]any
+	err := yaml.Unmarshal([]byte(yamlStr), &yamlStruct)
+	require.NoError(t, err)
+	jsonStruct := dyno.ConvertMapI2MapS(yamlStruct)
+	jsonBytes, err := json.Marshal(jsonStruct)
+	require.NoError(t, err)
+	return string(jsonBytes)
+}
+
+func TestMerge(t *testing.T) {
+	confStack := toFiles(t, conf1, conf2, conf3)
+	jsonBytes, err := HandleConfigStack(confStack)
+	require.NoError(t, err)
+	require.JSONEq(t, yamlToJson(t, mergedConf), string(jsonBytes))
+}
+
+var conf1 = `
+foo: conf1
+some-map:
+  opt1: cool
+config:
+  protocols:
+    example-protocol:
+      protocol-number: 15
+      protocol-boolean: true
+      protocol-string: foobar
+    removed-protocol:
+      connector: asdf
+`
+
+var conf2 = `
+foo: conf2
+`
+
+var conf3 = `
+foo: conf3
+some-map:
+  opt2: lmao
+config:
+  protocols:
+    example-protocol:
+      protocol-string: override
+    removed-protocol: null
+`
+
+var mergedConf = `
+foo: conf3
+some-map:
+  opt1: cool
+  opt2: lmao
+config:
+  protocols:
+    - protocol-number: 15
+      protocol-boolean: true
+      protocol-string: override
+`