merge: branch '3613-dotenv' into 'main'

Support .env [#3613] Closes #3613 See merge request accumulatenetwork/accumulate!1081
AccumulateNetwork · Jul 9, 2024 · 3d33014 · 3d33014
2 parents 99e0e8c + e1762e3
commit 3d33014
Show file tree

Hide file tree

Showing 8 changed files with 376 additions and 4 deletions.
diff --git a/cmd/accumulated/run/marshal.go → cmd/accumulated/run/config.go b/cmd/accumulated/run/marshal.go → cmd/accumulated/run/config.go
@@ -9,13 +9,17 @@ package run
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"reflect"
 	"regexp"
 	"strings"
 
 	"github.com/BurntSushi/toml"
+	"github.com/joho/godotenv"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/errors"
 	"gopkg.in/yaml.v3"
 )
@@ -24,6 +28,10 @@ func (c *Config) FilePath() string     { return c.file }
 func (c *Config) SetFilePath(p string) { c.file = p }
 
 func (c *Config) LoadFrom(file string) error {
+	return c.LoadFromFS(os.DirFS("."), file)
+}
+
+func (c *Config) LoadFromFS(fs fs.FS, file string) error {
 	var format func([]byte, any) error
 	switch s := filepath.Ext(file); s {
 	case ".toml", ".tml", ".ini":
@@ -36,12 +44,19 @@ func (c *Config) LoadFrom(file string) error {
 		return errors.BadRequest.WithFormat("unknown file type %s", s)
 	}
 
-	b, err := os.ReadFile(file)
+	f, err := fs.Open(file)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = f.Close() }()
+
+	b, err := io.ReadAll(f)
 	if err != nil {
 		return err
 	}
 
 	c.file = file
+	c.fs = fs
 	return c.Load(b, format)
 }
 
@@ -58,13 +73,73 @@ func (c *Config) Load(b []byte, format func([]byte, any) error) error {
 		return err
 	}
 
-	return json.Unmarshal(b, c)
+	err = json.Unmarshal(b, c)
+	if err != nil {
+		return err
+	}
+
+	return c.applyDotEnv()
+}
+
+func (c *Config) applyDotEnv() error {
+	if !setDefaultPtr(&c.DotEnv, false) {
+		return nil
+	}
+
+	file := ".env"
+	if c.file != "" {
+		dir := filepath.Dir(c.file)
+		file = filepath.Join(dir, file)
+	}
+
+	var expand func(name string) string
+	var errs []error
+
+	f, err := c.fs.Open(file)
+	switch {
+	case err == nil:
+		defer func() { _ = f.Close() }()
+
+		// Parse
+		env, err := godotenv.Parse(f)
+		if err != nil {
+			return err
+		}
+
+		// And expand
+		expand = func(name string) string {
+			value, ok := env[name]
+			if ok {
+				return value
+			}
+			errs = append(errs, fmt.Errorf("%q is not defined", name))
+			return fmt.Sprintf("#!MISSING(%q)", name)
+		}
+
+	case errors.Is(err, fs.ErrNotExist):
+		// Only return an error if there is at least one ${ENV}
+		expand = func(name string) string {
+			if len(errs) == 0 {
+				errs = append(errs, err)
+			}
+			return fmt.Sprintf("#!MISSING(%q)", name)
+		}
+
+	default:
+		return err
+	}
+
+	expandEnv(reflect.ValueOf(c), expand)
+	return errors.Join(errs...)
 }
 
 func (c *Config) Save() error {
 	if c.file == "" {
 		return errors.BadRequest.With("not loaded from a file")
 	}
+	if c.fs != nil && c.fs != os.DirFS(".") {
+		return errors.BadRequest.With("loaded from an immutable filesystem")
+	}
 	return c.SaveTo(c.file)
 }
 
@@ -165,3 +240,35 @@ func float2int(v reflect.Value) any {
 		return v.Interface()
 	}
 }
+
+func expandEnv(v reflect.Value, expand func(string) string) {
+	switch v.Kind() {
+	case reflect.String:
+		s := v.String()
+		s = os.Expand(s, expand)
+		v.SetString(s)
+
+	case reflect.Pointer, reflect.Interface:
+		expandEnv(v.Elem(), expand)
+
+	case reflect.Slice, reflect.Array:
+		for i, n := 0, v.Len(); i < n; i++ {
+			expandEnv(v.Index(i), expand)
+		}
+
+	case reflect.Map:
+		it := v.MapRange()
+		for it.Next() {
+			expandEnv(it.Key(), expand)
+			expandEnv(it.Value(), expand)
+		}
+
+	case reflect.Struct:
+		typ := v.Type()
+		for i, n := 0, typ.NumField(); i < n; i++ {
+			if typ.Field(i).IsExported() {
+				expandEnv(v.Field(i), expand)
+			}
+		}
+	}
+}
diff --git a/cmd/accumulated/run/config.yml b/cmd/accumulated/run/config.yml
@@ -4,6 +4,12 @@ Config:
     - name: file
       type: string
       marshal-as: none
+    - name: fs
+      type: fs.FS
+      marshal-as: none
+    - name: DotEnv
+      type: bool
+      pointer: true
     - name: Network
       type: string
     - name: Logging

diff --git a/cmd/accumulated/run/dotenv_test.go b/cmd/accumulated/run/dotenv_test.go
@@ -0,0 +1,101 @@
+// Copyright 2024 The Accumulate Authors
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+package run
+
+import (
+	"io/fs"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	testutil "gitlab.com/accumulatenetwork/accumulate/test/util"
+)
+
+func TestDotenv(t *testing.T) {
+	// When dot-env is set, ${FOO} is resolved
+	t.Run("Set", func(t *testing.T) {
+		fs := mkfs(map[string]string{
+			".env": `
+				FOO=bar`,
+			"accumulate.toml": `
+				dot-env = true
+				network = "${FOO}"`,
+		})
+
+		cfg := new(Config)
+		require.NoError(t, cfg.LoadFromFS(fs, "accumulate.toml"))
+		require.Equal(t, "bar", cfg.Network)
+	})
+
+	// When dot-env is unset, ${FOO} is left as is
+	t.Run("Unset", func(t *testing.T) {
+		fs := mkfs(map[string]string{
+			".env": `
+				FOO=bar`,
+			"accumulate.toml": `
+				network = "${FOO}"`,
+		})
+
+		cfg := new(Config)
+		require.NoError(t, cfg.LoadFromFS(fs, "accumulate.toml"))
+		require.Equal(t, "${FOO}", cfg.Network)
+	})
+
+	// When dot-env is set, referencing an unset variable ${BAR} is an error
+	t.Run("Wrong var", func(t *testing.T) {
+		fs := mkfs(map[string]string{
+			".env": `
+				FOO=bar`,
+			"accumulate.toml": `
+				dot-env = true
+				network = "${BAR}"`,
+		})
+
+		cfg := new(Config)
+		err := cfg.LoadFromFS(fs, "accumulate.toml")
+		require.EqualError(t, err, `"BAR" is not defined`)
+	})
+
+	// Variable are resolved exclusively from .env, not from actual environment
+	// variables
+	t.Run("Ignore env", func(t *testing.T) {
+		fs := mkfs(map[string]string{
+			"accumulate.toml": `
+				dot-env = true
+				network = "${FOO}"`,
+		})
+		require.NoError(t, os.Setenv("FOO", "bar"))
+
+		cfg := new(Config)
+		err := cfg.LoadFromFS(fs, "accumulate.toml")
+		require.EqualError(t, err, `open .env: file does not exist`)
+	})
+}
+
+func TestSaveToFS(t *testing.T) {
+	fs := mkfs(map[string]string{
+		"accumulate.toml": `network = "foo"`,
+	})
+
+	cfg := new(Config)
+	require.NoError(t, cfg.LoadFromFS(fs, "accumulate.toml"))
+
+	// Ensure Save doesn't panic when loaded from [fs.FS]
+	require.Error(t, cfg.Save())
+}
+
+func mkfs(files map[string]string) fs.FS {
+	root := new(testutil.Dir)
+	for name, data := range files {
+		root.Files = append(root.Files, &testutil.File{
+			Name: name,
+			Data: strings.NewReader(data),
+		})
+	}
+	return root
+}
diff --git a/cmd/accumulated/run/types_gen.go b/cmd/accumulated/run/types_gen.go
@@ -16,6 +16,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"log/slog"
 	"strings"
 	"time"
@@ -50,6 +51,8 @@ type CometPrivValFile struct {
 
 type Config struct {
 	file            string
+	fs              fs.FS
+	DotEnv          *bool            `json:"dotEnv,omitempty" form:"dotEnv" query:"dotEnv" validate:"required"`
 	Network         string           `json:"network,omitempty" form:"network" query:"network" validate:"required"`
 	Logging         *Logging         `json:"logging,omitempty" form:"logging" query:"logging" validate:"required"`
 	Instrumentation *Instrumentation `json:"instrumentation,omitempty" form:"instrumentation" query:"instrumentation" validate:"required"`
@@ -359,6 +362,10 @@ func (v *CometPrivValFile) CopyAsInterface() interface{} { return v.Copy() }
 func (v *Config) Copy() *Config {
 	u := new(Config)
 
+	if v.DotEnv != nil {
+		u.DotEnv = new(bool)
+		*u.DotEnv = *v.DotEnv
+	}
 	u.Network = v.Network
 	if v.Logging != nil {
 		u.Logging = (v.Logging).Copy()
@@ -976,6 +983,14 @@ func (v *CometPrivValFile) Equal(u *CometPrivValFile) bool {
 }
 
 func (v *Config) Equal(u *Config) bool {
+	switch {
+	case v.DotEnv == u.DotEnv:
+		// equal
+	case v.DotEnv == nil || u.DotEnv == nil:
+		return false
+	case !(*v.DotEnv == *u.DotEnv):
+		return false
+	}
 	if !(v.Network == u.Network) {
 		return false
 	}
@@ -1948,13 +1963,17 @@ func (v *CometPrivValFile) MarshalJSON() ([]byte, error) {
 
 func (v *Config) MarshalJSON() ([]byte, error) {
 	u := struct {
+		DotEnv          *bool                                          `json:"dotEnv,omitempty"`
 		Network         string                                         `json:"network,omitempty"`
 		Logging         *Logging                                       `json:"logging,omitempty"`
 		Instrumentation *Instrumentation                               `json:"instrumentation,omitempty"`
 		P2P             *P2P                                           `json:"p2P,omitempty"`
 		Configurations  *encoding.JsonUnmarshalListWith[Configuration] `json:"configurations,omitempty"`
 		Services        *encoding.JsonUnmarshalListWith[Service]       `json:"services,omitempty"`
 	}{}
+	if !(v.DotEnv == nil) {
+		u.DotEnv = v.DotEnv
+	}
 	if !(len(v.Network) == 0) {
 		u.Network = v.Network
 	}
@@ -2667,6 +2686,7 @@ func (v *CometPrivValFile) UnmarshalJSON(data []byte) error {
 
 func (v *Config) UnmarshalJSON(data []byte) error {
 	u := struct {
+		DotEnv          *bool                                          `json:"dotEnv,omitempty"`
 		Network         string                                         `json:"network,omitempty"`
 		Logging         *Logging                                       `json:"logging,omitempty"`
 		Instrumentation *Instrumentation                               `json:"instrumentation,omitempty"`
@@ -2675,6 +2695,7 @@ func (v *Config) UnmarshalJSON(data []byte) error {
 		Services        *encoding.JsonUnmarshalListWith[Service]       `json:"services,omitempty"`
 	}{}
 
+	u.DotEnv = v.DotEnv
 	u.Network = v.Network
 	u.Logging = v.Logging
 	u.Instrumentation = v.Instrumentation
@@ -2685,6 +2706,7 @@ func (v *Config) UnmarshalJSON(data []byte) error {
 	if err != nil {
 		return err
 	}
+	v.DotEnv = u.DotEnv
 	v.Network = u.Network
 	v.Logging = u.Logging
 	v.Instrumentation = u.Instrumentation

diff --git a/cmd/accumulated/run/utils.go b/cmd/accumulated/run/utils.go
@@ -38,11 +38,11 @@ var (
 
 func Ptr[T any](v T) *T { return &v }
 
-func setDefaultPtr[V any](ptr **V, def V) *V {
+func setDefaultPtr[V any](ptr **V, def V) V {
 	if *ptr == nil {
 		*ptr = &def
 	}
-	return *ptr
+	return **ptr
 }
 
 func setDefaultVal[V any](ptr *V, def V) V {

diff --git a/go.mod b/go.mod
@@ -297,6 +297,7 @@ require (
 	github.com/jingyugao/rowserrcheck v1.1.1 // indirect
 	github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
+	github.com/joho/godotenv v1.5.1
 	github.com/jonboulle/clockwork v0.3.0 // indirect
 	github.com/julz/importas v0.1.0 // indirect
 	github.com/kisielk/errcheck v1.7.0 // indirect

diff --git a/go.sum b/go.sum
@@ -645,6 +645,8 @@ github.com/jjti/go-spancheck v0.5.3 h1:vfq4s2IB8T3HvbpiwDTYgVPj1Ze/ZSXrTtaZRTc7C
 github.com/jjti/go-spancheck v0.5.3/go.mod h1:eQdOX1k3T+nAKvZDyLC3Eby0La4dZ+I19iOl5NzSPFE=
 github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U=
 github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQymUhg=
 github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=