nixcache: automatically configure nix daemon (#1996)

Author: gcurtis

internal/boxcli/cache.go

@@ -81,7 +81,7 @@ func cacheConfigureCmd() *cobra.Command {
 				u, _ := user.Current()
 				username = u.Username
 			}
-			return nixcache.Get().ConfigureAWS(cmd.Context(), username)
+			return nixcache.Get().Configure(cmd.Context(), username)
 		},
 	}
 	cmd.Flags().StringVar(&username, "user", "", "")

internal/devbox/packages.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"os/user"
 	"path/filepath"
 	"runtime/trace"
 	"slices"
@@ -22,6 +23,7 @@ import (
 	"go.jetpack.io/devbox/internal/devpkg"
 	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/lock"
+	"go.jetpack.io/devbox/internal/redact"
 	"go.jetpack.io/devbox/internal/shellgen"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
@@ -456,6 +458,25 @@ func (d *Devbox) installNixPackagesToStore(ctx context.Context, mode installMode
 			if err == nil {
 				args.Env = creds.Env()
 			}
+
+			u, err := user.Current()
+			if err != nil {
+				err = redact.Errorf("lookup current user: %v", err)
+				debug.Log("error configuring cache: %v", err)
+			}
+			err = d.providers.NixCache.Configure(ctx, u.Username)
+			if err != nil {
+				debug.Log("error configuring cache: %v", err)
+
+				var daemonErr *nix.DaemonError
+				if errors.As(err, &daemonErr) {
+					// Error here to give the user a chance to restart the daemon.
+					return usererr.New("Devbox configured Nix to use %q as a cache. Please restart the Nix daemon and re-run Devbox.", args.ExtraSubstituter)
+				}
+				// Other errors indicate we couldn't update nix.conf, so just warn and continue
+				// by building from source if necessary.
+				ux.Fwarning(d.stderr, "Devbox was unable to configure Nix to use your organization's private cache. Some packages might be built from source.")
+			}
 		}
 		for _, installable := range installables {
 			err = nix.Build(ctx, args, installable)

internal/devbox/providers/nixcache/nixcache.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/AlecAivazis/survey/v2"
 	"go.jetpack.io/devbox/internal/build"
+	"go.jetpack.io/devbox/internal/debug"
 	"go.jetpack.io/devbox/internal/devbox/providers/identity"
 	"go.jetpack.io/devbox/internal/envir"
 	"go.jetpack.io/devbox/internal/fileutil"
@@ -31,18 +32,30 @@ func Get() *Provider {
 	return singleton
 }
 
-func (p *Provider) ConfigureAWS(ctx context.Context, username string) error {
+func (p *Provider) Configure(ctx context.Context, username string) error {
+	debug.Log("checking if nix cache is configured for %s", username)
+
 	rootConfig, err := p.rootAWSConfigPath()
 	if err != nil {
 		return err
 	}
-	if fileutil.Exists(rootConfig) {
-		// Already configured.
+	debug.Log("root aws config path is: %s", rootConfig)
+	awsConfigExists := fileutil.Exists(rootConfig)
+
+	cfg, err := nix.CurrentConfig(ctx)
+	if err != nil {
+		return err
+	}
+	trusted, _ := cfg.IsUserTrusted(ctx, username)
+
+	configured := awsConfigExists && trusted
+	debug.Log("nix cache configured = %v (awsConfigExists == %v && trusted == %v)", configured, awsConfigExists, trusted)
+	if configured {
 		return nil
 	}
 
 	if os.Getuid() == 0 {
-		err := p.configureRoot(username)
+		err := p.configureRoot(ctx, username)
 		if err != nil {
 			return redact.Errorf("update ~root/.aws/config with devbox credentials: %s", err)
 		}
@@ -72,7 +85,7 @@ func (p *Provider) rootAWSConfigPath() (string, error) {
 	return filepath.Join(u.HomeDir, ".aws", "config"), nil
 }
 
-func (p *Provider) configureRoot(username string) error {
+func (p *Provider) configureRoot(ctx context.Context, username string) error {
 	exe := p.executable()
 	if exe == "" {
 		return redact.Errorf("get path to current devbox executable")
@@ -113,7 +126,14 @@ credential_process = %s -u %s -i %s cache credentials
 	if err != nil {
 		return err
 	}
-	return config.Close()
+	if err := config.Close(); err != nil {
+		return err
+	}
+
+	if err := nix.IncludeDevboxConfig(ctx, username); err != nil {
+		return redact.Errorf("modify nix config: %v", err)
+	}
+	return nil
 }
 
 func (p *Provider) sudoConfigureRoot(ctx context.Context, username string) error {
@@ -140,9 +160,14 @@ func (p *Provider) sudoConfigureRoot(ctx context.Context, username string) error
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 
+	debug.Log("running sudo: %s", cmd)
 	if err := cmd.Run(); err != nil {
 		return fmt.Errorf("failed to relaunch with sudo: %w", err)
 	}
+
+	// Print a warning if we were unable to automatically make the user
+	// trusted.
+	checkIfUserCanAddSubstituter(ctx)
 	return nil
 }
 
@@ -205,9 +230,6 @@ func (p *Provider) URI(ctx context.Context) (string, error) {
 	if err != nil {
 		return "", redact.Errorf("nixcache: get uri: %w", redact.Safe(err))
 	}
-	if uri != "" {
-		checkIfUserCanAddSubstituter(ctx)
-	}
 	return uri, nil
 }
 
@@ -227,7 +249,12 @@ func checkIfUserCanAddSubstituter(ctx context.Context) {
 	if err != nil {
 		return
 	}
-	trusted, _ := cfg.IsUserTrusted(ctx)
+
+	u, err := user.Current()
+	if err != nil {
+		return
+	}
+	trusted, _ := cfg.IsUserTrusted(ctx, u.Username)
 	if !trusted {
 		ux.Fwarning(
 			os.Stderr,

internal/nix/config.go

@@ -1,11 +1,16 @@
 package nix
 
 import (
+	"cmp"
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
+	"io"
+	"os"
 	"os/exec"
 	"os/user"
+	"path/filepath"
 	"slices"
 	"strings"
 
@@ -51,15 +56,15 @@ func CurrentConfig(ctx context.Context) (Config, error) {
 // IsUserTrusted reports if the current OS user is in the trusted-users list. If
 // there are any groups in the list, it also checks if the user belongs to any
 // of them.
-func (c Config) IsUserTrusted(ctx context.Context) (bool, error) {
+func (c Config) IsUserTrusted(ctx context.Context, username string) (bool, error) {
 	trusted := c.TrustedUsers.Value
 	if len(trusted) == 0 {
 		return false, nil
 	}
 
-	current, err := user.Current()
+	current, err := user.Lookup(username)
 	if err != nil {
-		return false, redact.Errorf("lookup current user: %v", err)
+		return false, redact.Errorf("lookup user: %v", err)
 	}
 	if slices.Contains(trusted, current.Username) {
 		return true, nil
@@ -99,3 +104,66 @@ func (c Config) IsUserTrusted(ctx context.Context) (bool, error) {
 	}
 	return false, nil
 }
+
+func IncludeDevboxConfig(ctx context.Context, username string) error {
+	info, _ := versionInfo()
+	path := cmp.Or(info.SystemConfig, "/etc/nix/nix.conf")
+	includePath := filepath.Join(filepath.Dir(path), "devbox-nix.conf")
+	b := fmt.Appendf(nil, "# This config was auto-generated by Devbox.\n\nextra-trusted-users = %s\n", username)
+	if err := os.WriteFile(includePath, b, 0o664); err != nil {
+		return redact.Errorf("write devbox nix.conf: %v", err)
+	}
+
+	appended, err := appendConfigInclude(path, includePath)
+	if err != nil {
+		return err
+	}
+	if appended {
+		return restartDaemon(ctx)
+	}
+	return nil
+}
+
+func appendConfigInclude(srcPath, includePath string) (appended bool, err error) {
+	nixConf, err := os.OpenFile(srcPath, os.O_RDWR, 0)
+	if err != nil {
+		return false, err
+	}
+	defer nixConf.Close()
+
+	confb, err := io.ReadAll(nixConf)
+	if err != nil {
+		return false, err
+	}
+	for _, line := range strings.Split(string(confb), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			// <whitespace>
+			continue
+		}
+		if strings.HasPrefix(line, "#") {
+			// # comment
+			continue
+		}
+
+		path := strings.TrimSpace(strings.TrimPrefix(line, "include"))
+		if path == includePath {
+			// include devbox-nix.conf
+			return false, nil
+		}
+		path = strings.TrimSpace(strings.TrimPrefix(line, "!include"))
+		if path == includePath {
+			// !include devbox-nix.conf
+			return false, nil
+		}
+	}
+
+	include := "\ninclude " + includePath + "\n"
+	if _, err := nixConf.WriteString(include); err != nil {
+		return false, redact.Errorf("append %q to %s: %v", redact.Safe(include), srcPath, err)
+	}
+	if err := nixConf.Close(); err != nil {
+		return false, redact.Errorf("append %q to %s: %v", redact.Safe(include), srcPath, err)
+	}
+	return true, nil
+}

internal/nix/config_test.go

@@ -8,20 +8,21 @@ import (
 
 //nolint:revive
 func TestConfigIsUserTrusted(t *testing.T) {
+	curUser, err := user.Current()
+	if err != nil {
+		t.Fatal("lookup current user:", err)
+	}
+
 	t.Run("UsernameInList", func(t *testing.T) {
-		u, err := user.Current()
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Setenv("NIX_CONFIG", "trusted-users = "+u.Username)
+		t.Setenv("NIX_CONFIG", "trusted-users = "+curUser.Username)
 
 		ctx := context.Background()
 		cfg, err := CurrentConfig(ctx)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		trusted, err := cfg.IsUserTrusted(ctx)
+		trusted, err := cfg.IsUserTrusted(ctx, curUser.Username)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -30,11 +31,7 @@ func TestConfigIsUserTrusted(t *testing.T) {
 		}
 	})
 	t.Run("UserGroupInList", func(t *testing.T) {
-		u, err := user.Current()
-		if err != nil {
-			t.Fatal(err)
-		}
-		g, err := user.LookupGroupId(u.Gid)
+		g, err := user.LookupGroupId(curUser.Gid)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -46,7 +43,7 @@ func TestConfigIsUserTrusted(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		trusted, err := cfg.IsUserTrusted(ctx)
+		trusted, err := cfg.IsUserTrusted(ctx, curUser.Username)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -63,7 +60,7 @@ func TestConfigIsUserTrusted(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		trusted, err := cfg.IsUserTrusted(ctx)
+		trusted, err := cfg.IsUserTrusted(ctx, curUser.Username)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -80,7 +77,7 @@ func TestConfigIsUserTrusted(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		trusted, err := cfg.IsUserTrusted(ctx)
+		trusted, err := cfg.IsUserTrusted(ctx, curUser.Username)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -97,7 +94,7 @@ func TestConfigIsUserTrusted(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		trusted, err := cfg.IsUserTrusted(ctx)
+		trusted, err := cfg.IsUserTrusted(ctx, curUser.Username)
 		if err != nil {
 			t.Fatal(err)
 		}

internal/nix/nix.go

@@ -12,6 +12,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"runtime/trace"
 	"strings"
 	"sync"
@@ -412,3 +413,35 @@ func parseInsecurePackagesFromExitError(errorMsg string) []string {
 
 	return insecurePackages
 }
+
+var ErrUnknownServiceManager = errors.New("unknown service manager")
+
+func restartDaemon(ctx context.Context) error {
+	if runtime.GOOS != "darwin" {
+		err := fmt.Errorf("don't know how to restart nix daemon: %w", ErrUnknownServiceManager)
+		return &DaemonError{err: err}
+	}
+
+	cmd := exec.CommandContext(ctx, "launchctl", "bootout", "system", "/Library/LaunchDaemons/org.nixos.nix-daemon.plist")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return &DaemonError{
+			cmd:    cmd.String(),
+			stderr: out,
+			err:    fmt.Errorf("stop nix daemon: %w", err),
+		}
+	}
+	cmd = exec.CommandContext(ctx, "launchctl", "bootstrap", "system", "/Library/LaunchDaemons/org.nixos.nix-daemon.plist")
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		return &DaemonError{
+			cmd:    cmd.String(),
+			stderr: out,
+			err:    fmt.Errorf("start nix daemon: %w", err),
+		}
+	}
+
+	// TODO(gcurtis): poll for daemon to come back instead.
+	time.Sleep(2 * time.Second)
+	return nil
+}

internal/nix/store.go

@@ -112,7 +112,10 @@ func (e *DaemonError) Unwrap() error {
 func (e *DaemonError) Redact() string {
 	// Don't include e.stderr in redacted messages because it can contain
 	// things like paths and usernames.
-	return fmt.Sprintf("command %s: %s", e.cmd, e.err)
+	if e.cmd != "" {
+		return fmt.Sprintf("command %s: %s", e.cmd, e.err)
+	}
+	return e.err.Error()
 }
 
 // DaemonVersion returns the version of the currently running Nix daemon.