POD-854 | Decouple pulling devcontainer spec from pulling project (lo…

…ft-sh#1244)

* Refactor runner Up method

* Init - pull project from within the container is separate devcontainer spec is provided

* Extract clone for workspace to agent pkg

cmd/agent/container/setup.go

@@ -159,6 +159,20 @@ func (cmd *SetupContainerCmd) Run(ctx context.Context) error {
 		}
 	}
 
+	if b, err := workspaceInfo.PullFromInsideContainer.Bool(); err == nil && b {
+		if err := agent.CloneRepositoryForWorkspace(ctx,
+			&workspaceInfo.Source,
+			&workspaceInfo.Agent,
+			workspaceInfo.ContentFolder,
+			"",
+			workspaceInfo.CLIOptions,
+			true,
+			logger,
+		); err != nil {
+			return err
+		}
+	}
+
 	// setup container
 	err = setup.SetupContainer(ctx, setupInfo, workspaceInfo.CLIOptions.WorkspaceEnv, cmd.ChownWorkspace, logger)
 	if err != nil {

cmd/agent/workspace/up.go

@@ -2,7 +2,6 @@ package workspace
 
 import (
 	"context"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -21,10 +20,9 @@ import (
 	"github.com/loft-sh/devpod/pkg/daemon"
 	"github.com/loft-sh/devpod/pkg/devcontainer"
 	config2 "github.com/loft-sh/devpod/pkg/devcontainer/config"
+	"github.com/loft-sh/devpod/pkg/devcontainer/crane"
 	"github.com/loft-sh/devpod/pkg/dockercredentials"
 	"github.com/loft-sh/devpod/pkg/extract"
-	"github.com/loft-sh/devpod/pkg/git"
-	"github.com/loft-sh/devpod/pkg/gitcredentials"
 	provider2 "github.com/loft-sh/devpod/pkg/provider"
 	"github.com/loft-sh/devpod/scripts"
 	"github.com/loft-sh/log"
@@ -271,7 +269,19 @@ func prepareWorkspace(ctx context.Context, workspaceInfo *provider2.AgentWorkspa
 			return nil
 		}
 
-		return cloneRepository(ctx, workspaceInfo, helper, log)
+		if workspaceInfo.CLIOptions.DevContainerSource != "" && crane.IsAvailable() {
+			log.Infof("Pulling devcontainer spec from %v", workspaceInfo.CLIOptions.DevContainerSource)
+			return nil
+		}
+		return agent.CloneRepositoryForWorkspace(ctx,
+			&workspaceInfo.Workspace.Source,
+			&workspaceInfo.Agent,
+			workspaceInfo.ContentFolder,
+			helper,
+			workspaceInfo.CLIOptions,
+			false,
+			log,
+		)
 	}
 
 	if workspaceInfo.Workspace.Source.LocalFolder != "" {
@@ -400,100 +410,6 @@ func prepareImage(workspaceDir, image string) error {
 	return nil
 }
 
-func cloneRepository(ctx context.Context, workspaceInfo *provider2.AgentWorkspaceInfo, helper string, log log.Logger) error {
-	s := workspaceInfo.Workspace.Source
-	log.Info("Clone repository")
-	log.Infof("URL: %s\n", s.GitRepository)
-	if s.GitBranch != "" {
-		log.Infof("Branch: %s\n", s.GitBranch)
-	}
-	if s.GitCommit != "" {
-		log.Infof("Commit: %s\n", s.GitCommit)
-	}
-	if s.GitSubPath != "" {
-		log.Infof("Subpath: %s\n", s.GitSubPath)
-	}
-	if s.GitPRReference != "" {
-		log.Infof("PR: %s\n", s.GitPRReference)
-	}
-
-	workspaceDir := workspaceInfo.ContentFolder
-	// remove the credential helper or otherwise we will receive strange errors within the container
-	defer func() {
-		if helper != "" {
-			if err := gitcredentials.RemoveHelperFromPath(gitcredentials.GetLocalGitConfigPath(workspaceDir)); err != nil {
-				log.Errorf("Remove git credential helper: %v", err)
-			}
-		}
-	}()
-
-	// check if command exists
-	if !command.Exists("git") {
-		local, _ := workspaceInfo.Agent.Local.Bool()
-		if local {
-			return fmt.Errorf("seems like git isn't installed on your system. Please make sure to install git and make it available in the PATH")
-		}
-		if err := git.InstallBinary(log); err != nil {
-			return err
-		}
-	}
-
-	// setup private ssh key if passed in
-	extraEnv := []string{}
-	if workspaceInfo.CLIOptions.SSHKey != "" {
-		sshExtraEnv, err := setupSSHKey(workspaceInfo.CLIOptions.SSHKey, workspaceInfo.Agent.Path)
-		if err != nil {
-			return err
-		}
-		extraEnv = append(extraEnv, sshExtraEnv...)
-	}
-
-	// run git command
-	cloner := git.NewCloner(workspaceInfo.CLIOptions.GitCloneStrategy)
-	gitInfo := git.NewGitInfo(s.GitRepository, s.GitBranch, s.GitCommit, s.GitPRReference, s.GitSubPath)
-	err := git.CloneRepositoryWithEnv(ctx, gitInfo, extraEnv, workspaceDir, helper, cloner, log)
-	if err != nil {
-		return fmt.Errorf("clone repository: %w", err)
-	}
-
-	log.Done("Successfully cloned repository")
-
-	return nil
-}
-
-func setupSSHKey(key string, agentPath string) ([]string, error) {
-	keyFile, err := os.CreateTemp("", "")
-	if err != nil {
-		return nil, err
-	}
-	defer os.Remove(keyFile.Name())
-	defer keyFile.Close()
-
-	if err := writeSSHKey(keyFile, key); err != nil {
-		return nil, err
-	}
-
-	if err := os.Chmod(keyFile.Name(), 0o400); err != nil {
-		return nil, err
-	}
-
-	env := []string{"GIT_TERMINAL_PROMPT=0"}
-	gitSSHCmd := []string{agentPath, "helper", "ssh-git-clone", "--key-file=" + keyFile.Name()}
-	env = append(env, "GIT_SSH_COMMAND="+command.Quote(gitSSHCmd))
-
-	return env, nil
-}
-
-func writeSSHKey(key *os.File, sshKey string) error {
-	data, err := base64.StdEncoding.DecodeString(sshKey)
-	if err != nil {
-		return err
-	}
-
-	_, err = key.WriteString(string(data))
-	return err
-}
-
 func installDocker(log log.Logger) error {
 	if !command.Exists("docker") {
 		writer := log.Writer(logrus.InfoLevel, false)

pkg/agent/workspace.go

@@ -1,6 +1,8 @@
 package agent
 
 import (
+	"context"
+	"encoding/base64"
 	"fmt"
 	"os"
 	"os/exec"
@@ -10,6 +12,10 @@ import (
 
 	"github.com/loft-sh/devpod/pkg/command"
 	"github.com/loft-sh/devpod/pkg/config"
+	"github.com/loft-sh/devpod/pkg/git"
+	"github.com/loft-sh/devpod/pkg/gitcredentials"
+	provider2 "github.com/loft-sh/devpod/pkg/provider"
+	"github.com/loft-sh/log"
 	"github.com/mitchellh/go-homedir"
 )
 
@@ -224,3 +230,131 @@ func CreateAgentWorkspaceDir(agentFolder, context, workspaceID string) (string,
 
 	return workspaceDir, nil
 }
+
+func CloneRepositoryForWorkspace(
+	ctx context.Context,
+	source *provider2.WorkspaceSource,
+	agentConfig *provider2.ProviderAgentConfig,
+	workspaceDir, helper string,
+	options provider2.CLIOptions,
+	overwriteContent bool,
+	log log.Logger,
+) error {
+	log.Info("Clone repository")
+	log.Infof("URL: %s\n", source.GitRepository)
+	if source.GitBranch != "" {
+		log.Infof("Branch: %s\n", source.GitBranch)
+	}
+	if source.GitCommit != "" {
+		log.Infof("Commit: %s\n", source.GitCommit)
+	}
+	if source.GitSubPath != "" {
+		log.Infof("Subpath: %s\n", source.GitSubPath)
+	}
+	if source.GitPRReference != "" {
+		log.Infof("PR: %s\n", source.GitPRReference)
+	}
+
+	// remove the credential helper or otherwise we will receive strange errors within the container
+	defer func() {
+		if helper != "" {
+			if err := gitcredentials.RemoveHelperFromPath(gitcredentials.GetLocalGitConfigPath(workspaceDir)); err != nil {
+				log.Errorf("Remove git credential helper: %v", err)
+			}
+		}
+	}()
+
+	// check if command exists
+	if !command.Exists("git") {
+		local, _ := agentConfig.Local.Bool()
+		if local {
+			return fmt.Errorf("seems like git isn't installed on your system. Please make sure to install git and make it available in the PATH")
+		}
+		if err := git.InstallBinary(log); err != nil {
+			return err
+		}
+	}
+
+	if overwriteContent {
+		if err := removeDirContents(workspaceDir); err != nil {
+			log.Infof("Failed cleanup")
+			return err
+		}
+	}
+
+	// setup private ssh key if passed in
+	extraEnv := []string{}
+	if options.SSHKey != "" {
+		sshExtraEnv, err := setupSSHKey(options.SSHKey, agentConfig.Path)
+		if err != nil {
+			return err
+		}
+		extraEnv = append(extraEnv, sshExtraEnv...)
+	}
+
+	// run git command
+	cloner := git.NewCloner(options.GitCloneStrategy)
+	gitInfo := git.NewGitInfo(source.GitRepository, source.GitBranch, source.GitCommit, source.GitPRReference, source.GitSubPath)
+	err := git.CloneRepositoryWithEnv(ctx, gitInfo, extraEnv, workspaceDir, helper, cloner, log)
+	if err != nil {
+		return fmt.Errorf("clone repository: %w", err)
+	}
+
+	log.Done("Successfully cloned repository")
+
+	return nil
+}
+
+func setupSSHKey(key string, agentPath string) ([]string, error) {
+	keyFile, err := os.CreateTemp("", "")
+	if err != nil {
+		return nil, err
+	}
+	defer os.Remove(keyFile.Name())
+	defer keyFile.Close()
+
+	if err := writeSSHKey(keyFile, key); err != nil {
+		return nil, err
+	}
+
+	if err := os.Chmod(keyFile.Name(), 0o400); err != nil {
+		return nil, err
+	}
+
+	env := []string{"GIT_TERMINAL_PROMPT=0"}
+	gitSSHCmd := []string{agentPath, "helper", "ssh-git-clone", "--key-file=" + keyFile.Name()}
+	env = append(env, "GIT_SSH_COMMAND="+command.Quote(gitSSHCmd))
+
+	return env, nil
+}
+
+func writeSSHKey(key *os.File, sshKey string) error {
+	data, err := base64.StdEncoding.DecodeString(sshKey)
+	if err != nil {
+		return err
+	}
+
+	_, err = key.WriteString(string(data))
+	return err
+}
+
+func removeDirContents(dirPath string) error {
+	entries, err := os.ReadDir(dirPath)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range entries {
+		entryPath := filepath.Join(dirPath, entry.Name())
+		if entry.IsDir() {
+			err = os.RemoveAll(entryPath)
+		} else {
+			err = os.Remove(entryPath)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/devcontainer/config.go

@@ -34,7 +34,7 @@ func (r *runner) getRawConfig(options provider2.CLIOptions) (*config.DevContaine
 			Origin: "",
 		}, nil
 	} else if options.DevContainerSource != "" && crane.IsAvailable() {
-		localWorkspaceFolder, err := crane.PullConfigFromSource(options.DevContainerSource, r.Log)
+		localWorkspaceFolder, err := crane.PullConfigFromSource(r.WorkspaceConfig, r.Log)
 		if err != nil {
 			return nil, err
 		}

pkg/devcontainer/crane/run.go

@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"path/filepath"
 
+	provider2 "github.com/loft-sh/devpod/pkg/provider"
 	"github.com/loft-sh/log"
 )
 
@@ -51,8 +52,8 @@ func runCommand(command string, args ...string) (string, error) {
 }
 
 // PullConfigFromSource pulls devcontainer config from configSource using git crane and returns config path
-func PullConfigFromSource(configSource string, log log.Logger) (string, error) {
-	data, err := runCommand(PullCommand, GitCrane, configSource)
+func PullConfigFromSource(workspaceInfo *provider2.AgentWorkspaceInfo, log log.Logger) (string, error) {
+	data, err := runCommand(PullCommand, GitCrane, workspaceInfo.CLIOptions.DevContainerSource)
 	if err != nil {
 		return "", err
 	}
@@ -69,17 +70,32 @@ func PullConfigFromSource(configSource string, log log.Logger) (string, error) {
 		return "", err
 	}
 
-	return createContentDirectory(content)
+	return writeContentToDirectory(workspaceInfo, content, log)
 }
 
-func createContentDirectory(content *Content) (string, error) {
+func writeContentToDirectory(workspaceInfo *provider2.AgentWorkspaceInfo, content *Content, _ log.Logger) (string, error) {
+	path := workspaceInfo.ContentFolder
+	if path == "" {
+		path = createContentDirectory()
+		if path == "" {
+			return path, fmt.Errorf("failed to create temporary directory")
+		}
+	}
+	return storeFilesInDirectory(content, path)
+}
+
+func createContentDirectory() string {
 	tmpDir, err := os.MkdirTemp("", tmpDirTemplate)
 	if err != nil {
-		return "", fmt.Errorf("failed to create temporary directory: %w", err)
+		return ""
 	}
 
+	return tmpDir
+}
+
+func storeFilesInDirectory(content *Content, path string) (string, error) {
 	for filename, fileContent := range content.Files {
-		filePath := filepath.Join(tmpDir, filename)
+		filePath := filepath.Join(path, filename)
 
 		dir := filepath.Dir(filePath)
 		if err := os.MkdirAll(dir, os.ModePerm); err != nil {
@@ -88,10 +104,10 @@ func createContentDirectory(content *Content) (string, error) {
 
 		err := os.WriteFile(filePath, []byte(fileContent), os.ModePerm)
 		if err != nil {
-			os.RemoveAll(tmpDir)
+			os.RemoveAll(path)
 			return "", fmt.Errorf("failed to write file %s: %w", filename, err)
 		}
 	}
 
-	return tmpDir, nil
+	return path, nil
 }