RSDK-8886 Setup phase (#4457)

config/module.go

@@ -51,6 +51,7 @@ type Module struct {
 // JSONManifest contains meta.json fields that are used by both RDK and CLI.
 type JSONManifest struct {
 	Entrypoint string `json:"entrypoint"`
+	FirstRun   string `json:"first_run"`
 }
 
 // ModuleType indicates where a module comes from.
@@ -213,3 +214,85 @@ func (m Module) EvaluateExePath(packagesDir string) (string, error) {
 	}
 	return m.ExePath, nil
 }
+
+// FirstRunSuccessSuffix is the suffix of the file whose existence
+// denotes that the setup phase for a module ran successfully.
+//
+// Note that we create a new file instead of writing to `.status.json`,
+// which contains various package/module state tracking information.
+// Writing to `.status.json` introduces the risk of corrupting it, which
+// could break or uncoordinate package sync.
+const FirstRunSuccessSuffix = ".first_run_succeeded"
+
+// EvaluateFirstRunPath returns absolute FirstRunPath from one of two sources (in order of precedence):
+// 1. if there is a meta.json in the exe dir, use that, except in local non-tarball case.
+// 2. if this is a local tarball and there's a meta.json next to the tarball, use that.
+// Note: the working directory must be the unpacked tarball directory or local exec directory.
+//
+// On success (i.e. if the returned error is nil), this function also returns a function that creates
+// a marker file indicating that the setup phase has run successfully.
+func (m Module) EvaluateFirstRunPath(packagesDir string) (
+	string,
+	func() error,
+	error,
+) {
+	noop := func() error { return nil }
+	unpackedModDir, err := m.exeDir(packagesDir)
+	if err != nil {
+		return "", noop, err
+	}
+
+	firstRunSuccessPath := unpackedModDir + FirstRunSuccessSuffix
+	if _, err := os.Stat(firstRunSuccessPath); !errors.Is(err, os.ErrNotExist) {
+		return "", noop, errors.New("first run already ran")
+	}
+	markFirstRunSuccess := func() error {
+		//nolint:gosec // safe
+		_, err := os.Create(firstRunSuccessPath)
+		return err
+	}
+
+	// note: we don't look at internal meta.json in local non-tarball case because user has explicitly requested a binary.
+	localNonTarball := m.Type == ModuleTypeLocal && !m.NeedsSyntheticPackage()
+	if !localNonTarball {
+		// this is case 1, meta.json in exe folder.
+		metaPath, err := utils.SafeJoinDir(unpackedModDir, "meta.json")
+		if err != nil {
+			return "", noop, err
+		}
+		_, err = os.Stat(metaPath)
+		if err == nil {
+			// this is case 1, meta.json in exe dir
+			meta, err := parseJSONFile[JSONManifest](metaPath)
+			if err != nil {
+				return "", noop, err
+			}
+			firstRun, err := utils.SafeJoinDir(unpackedModDir, meta.FirstRun)
+			if err != nil {
+				return "", noop, err
+			}
+			firstRunPath, err := filepath.Abs(firstRun)
+			return firstRunPath, markFirstRunSuccess, err
+		}
+	}
+	if m.NeedsSyntheticPackage() {
+		// this is case 2, side-by-side
+		// TODO(RSDK-7848): remove this case once java sdk supports internal meta.json.
+		metaPath, err := utils.SafeJoinDir(filepath.Dir(m.ExePath), "meta.json")
+		if err != nil {
+			return "", noop, err
+		}
+		meta, err := parseJSONFile[JSONManifest](metaPath)
+		if err != nil {
+			// note: this error deprecates the side-by-side case because the side-by-side case is deprecated.
+			return "", noop, errors.Wrapf(err, "couldn't find meta.json inside tarball %s (or next to it)", m.ExePath)
+		}
+		firstRun, err := utils.SafeJoinDir(unpackedModDir, meta.FirstRun)
+		if err != nil {
+			return "", noop, err
+		}
+		firstRunPath, err := filepath.Abs(firstRun)
+		return firstRunPath, markFirstRunSuccess, err
+	}
+	return "", noop, errors.New("no first run script")
+}

module/modmanager/manager.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -40,6 +41,10 @@ import (
 	rutils "go.viam.com/rdk/utils"
 )
 
+const (
+	defaultFirstRunTimeout = 1 * time.Hour
+)
+
 var (
 	validateConfigTimeout       = 5 * time.Second
 	errMessageExitStatus143     = "exit status 143"
@@ -300,7 +305,7 @@ func (mgr *Manager) add(ctx context.Context, conf config.Module) error {
 	// only set the module data directory if the parent dir is present (which it might not be during tests)
 	if mgr.moduleDataParentDir != "" {
 		var err error
-		// todo: why isn't conf.Name being sanitized like PackageConfig.SanitizedName?
+		// TODO: why isn't conf.Name being sanitized like PackageConfig.SanitizedName?
 		moduleDataDir, err = rutils.SafeJoinDir(mgr.moduleDataParentDir, conf.Name)
 		if err != nil {
 			return err
@@ -1075,6 +1080,64 @@ func (m *module) checkReady(ctx context.Context, parentAddr string, logger loggi
 	}
 }
 
+// FirstRun is runs a module-specific setup script.
+func (mgr *Manager) FirstRun(ctx context.Context, conf config.Module) error {
+	logger := mgr.logger.AsZap().With("name", conf.Name)
+
+	// Evaluate the Module's FirstRun path. If there is an error we assume
+	// that the first run script does not exist and we debug log and exit quietly.
+	firstRunPath, markSuccess, err := conf.EvaluateFirstRunPath(packages.LocalPackagesDir(mgr.packagesDir))
+	if err != nil {
+		// TODO(RSDK-9067): some first run path evaluation errors should be promoted to WARN logs.
+		logger.Debug("no first run script detected, skipping setup phase", "error", err)
+		return nil
+	}
+
+	logger.Info("executing first run script")
+
+	// This value is normally set on a field on the [module] struct but it seems like we can safely get it on demand.
+	var dataDir string
+	if mgr.moduleDataParentDir != "" {
+		var err error
+		// TODO: why isn't conf.Name being sanitized like PackageConfig.SanitizedName?
+		dataDir, err = rutils.SafeJoinDir(mgr.moduleDataParentDir, conf.Name)
+		if err != nil {
+			return err
+		}
+	}
+
+	moduleEnvironment := getFullEnvironment(conf, dataDir, mgr.viamHomeDir)
+
+	// TODO(RSDK-9060): support a user-supplied timeout
+	cmdCtx, cancel := context.WithTimeout(ctx, defaultFirstRunTimeout)
+	defer cancel()
+
+	//nolint:gosec // Yes, we are deliberating executing arbitrary user code here.
+	cmd := exec.CommandContext(cmdCtx, firstRunPath)
+
+	cmd.Env = os.Environ()
+	for key, val := range moduleEnvironment {
+		cmd.Env = append(cmd.Env, key+"="+val)
+	}
+	cmdOut, err := cmd.CombinedOutput()
+
+	resultLogger := logger.With("path", firstRunPath, "output", string(cmdOut))
+	if err != nil {
+		resultLogger.Errorw("command failed", "error", err)
+		return err
+	}
+
+	resultLogger.Infow("command succeeded")
+
+	// Mark success by writing a marker file to disk. This is a best
+	// effort; if writing to disk fails the setup phase will run again
+	// for this module and version and we are okay with that.
+	if err := markSuccess(); err != nil {
+		logger.Errorw("failed to mark success", "error", err)
+	}
+	return nil
+}
+
 func (m *module) startProcess(
 	ctx context.Context,
 	parentAddr string,
@@ -1280,15 +1343,23 @@ func (m *module) cleanupAfterCrash(mgr *Manager) {
 }
 
 func (m *module) getFullEnvironment(viamHomeDir string) map[string]string {
+	return getFullEnvironment(m.cfg, m.dataDir, viamHomeDir)
+}
+
+func getFullEnvironment(
+	cfg config.Module,
+	dataDir string,
+	viamHomeDir string,
+) map[string]string {
 	environment := map[string]string{
 		"VIAM_HOME":        viamHomeDir,
-		"VIAM_MODULE_DATA": m.dataDir,
+		"VIAM_MODULE_DATA": dataDir,
 	}
-	if m.cfg.Type == config.ModuleTypeRegistry {
-		environment["VIAM_MODULE_ID"] = m.cfg.ModuleID
+	if cfg.Type == config.ModuleTypeRegistry {
+		environment["VIAM_MODULE_ID"] = cfg.ModuleID
 	}
 	// Overwrite the base environment variables with the module's environment variables (if specified)
-	for key, value := range m.cfg.Environment {
+	for key, value := range cfg.Environment {
 		environment[key] = value
 	}
 	return environment

module/modmaninterface/interface.go

@@ -27,5 +27,7 @@ type ModuleManager interface {
 	Provides(cfg resource.Config) bool
 	Handles() map[string]module.HandlerMap
 
+	FirstRun(ctx context.Context, conf config.Module) error
+
 	Close(ctx context.Context) error
 }

robot/impl/local_robot.go

@@ -1243,6 +1243,14 @@ func (r *localRobot) reconfigure(ctx context.Context, newConfig *config.Config,
 		return
 	}
 
+	// Run the setup phase for all modules in new config modules before proceeding with reconfiguration.
+	for _, mod := range newConfig.Modules {
+		if err := r.manager.moduleManager.FirstRun(ctx, mod); err != nil {
+			r.logger.CErrorw(ctx, "error executing setup phase", "module", mod.Name, "error", err)
+			return
+		}
+	}
+
 	if newConfig.Cloud != nil {
 		r.Logger().CDebug(ctx, "updating cached config")
 		if err := newConfig.StoreToCache(); err != nil {

robot/impl/resource_manager_modular_test.go

@@ -476,6 +476,10 @@ func (m *dummyModMan) Close(ctx context.Context) error {
 	return nil
 }
 
+func (m *dummyModMan) FirstRun(ctx context.Context, conf config.Module) error {
+	return nil
+}
+
 func TestTwoModulesSameName(t *testing.T) {
 	ctx := context.Background()
 	logger := logging.NewTestLogger(t)

robot/packages/utils.go

@@ -252,7 +252,8 @@ func commonCleanup(logger logging.Logger, expectedPackageDirectories map[string]
 			continue
 		}
 
-		// There should be no non-dir files in the packages/data dir except .status.json files. Delete any that exist
+		// There should be no non-dir files in the packages/data dir
+		// except .status.json and .first_run_succeeded. Delete any that exist
 		if packageTypeDir.Type()&os.ModeDir != os.ModeDir && !strings.HasSuffix(packageTypeDirName, statusFileExt) {
 			allErrors = multierr.Append(allErrors, os.Remove(packageTypeDirName))
 			continue
@@ -271,7 +272,8 @@ func commonCleanup(logger logging.Logger, expectedPackageDirectories map[string]
 			}
 			_, expectedToExist := expectedPackageDirectories[packageDirName]
 			_, expectedStatusFileToExist := expectedPackageDirectories[strings.TrimSuffix(packageDirName, statusFileExt)]
-			if !expectedToExist && !expectedStatusFileToExist {
+			_, expectedFirstRunSuccessFileToExist := expectedPackageDirectories[strings.TrimSuffix(packageDirName, config.FirstRunSuccessSuffix)]
+			if !expectedToExist && !expectedStatusFileToExist && !expectedFirstRunSuccessFileToExist {
 				logger.Debugf("Removing old package file(s) %s", packageDirName)
 				allErrors = multierr.Append(allErrors, os.RemoveAll(packageDirName))
 			}
@@ -294,6 +296,8 @@ type syncStatus string
 const (
 	syncStatusDownloading syncStatus = "downloading"
 	syncStatusDone        syncStatus = "done"
+
+	statusFileExt = ".status.json"
 )
 
 type packageSyncFile struct {
@@ -304,8 +308,6 @@ type packageSyncFile struct {
 	TarballChecksum string     `json:"tarball_checksum"`
 }
 
-var statusFileExt = ".status.json"
-
 func packageIsSynced(pkg config.PackageConfig, packagesDir string, logger logging.Logger) bool {
 	syncFile, err := readStatusFile(pkg, packagesDir)
 	switch {