POD-859 | Use Environment Templates from Pro (#1253)

* Bump github.com/loft-sh/api/v4 -> v4.1.0-alpha.10

* Add EnvironmentTemplate to CLIOptions

* Support crane environment source

* Allow overriding bin name

* Change how crane command is build

cmd/agent/workspace/up.go

@@ -286,7 +286,7 @@ func prepareWorkspace(ctx context.Context, workspaceInfo *provider2.AgentWorkspa
 			return nil
 		}
 
-		if workspaceInfo.CLIOptions.DevContainerSource != "" && crane.IsAvailable() {
+		if crane.ShouldUse(&workspaceInfo.CLIOptions) {
 			log.Infof("Pulling devcontainer spec from %v", workspaceInfo.CLIOptions.DevContainerSource)
 			return nil
 		}

cmd/up.go

@@ -153,6 +153,8 @@ func NewUpCmd(flags *flags.GlobalFlags) *cobra.Command {
 	upCmd.Flags().StringVar(&cmd.DevContainerImage, "devcontainer-image", "", "The container image to use, this will override the devcontainer.json value in the project")
 	upCmd.Flags().StringVar(&cmd.DevContainerPath, "devcontainer-path", "", "The path to the devcontainer.json relative to the project")
 	upCmd.Flags().StringVar(&cmd.DevContainerSource, "devcontainer-source", "", "External devcontainer.json source")
+	upCmd.Flags().StringVar(&cmd.EnvironmentTemplate, "environment-template", "", "Environment template to use")
+	_ = upCmd.Flags().MarkHidden("environment-template")
 	upCmd.Flags().StringArrayVar(&cmd.ProviderOptions, "provider-option", []string{}, "Provider option in the form KEY=VALUE")
 	upCmd.Flags().BoolVar(&cmd.Recreate, "recreate", false, "If true will remove any existing containers and recreate them")
 	upCmd.Flags().BoolVar(&cmd.Reset, "reset", false, "If true will remove any existing containers including sources, and recreate them")

go.mod

@@ -20,8 +20,8 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.5.1
 	github.com/joho/godotenv v1.5.1
-	github.com/loft-sh/agentapi/v4 v4.1.0-alpha.1
-	github.com/loft-sh/api/v4 v4.1.0-alpha.1
+	github.com/loft-sh/agentapi/v4 v4.1.0-alpha.10
+	github.com/loft-sh/api/v4 v4.1.0-alpha.10
 	github.com/loft-sh/log v0.0.0-20240219160058-26d83ffb46ac
 	github.com/loft-sh/programming-language-detection v0.0.5
 	github.com/loft-sh/ssh v0.0.4
@@ -50,14 +50,14 @@ require (
 	gopkg.in/square/go-jose.v2 v2.6.0
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools v2.2.0+incompatible
-	k8s.io/api v0.30.1
-	k8s.io/apimachinery v0.30.1
-	k8s.io/client-go v0.30.1
+	k8s.io/api v0.30.4
+	k8s.io/apimachinery v0.30.4
+	k8s.io/client-go v0.30.4
 	k8s.io/klog/v2 v2.120.1
 	k8s.io/kube-aggregator v0.30.1
 	k8s.io/kubectl v0.29.1
 	mvdan.cc/sh/v3 v3.6.0
-	sigs.k8s.io/controller-runtime v0.18.4
+	sigs.k8s.io/controller-runtime v0.18.5
 )
 
 require (
@@ -131,7 +131,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
-	github.com/loft-sh/admin-apis v0.0.0-20240203010124-3600c1c582a8 // indirect
+	github.com/loft-sh/admin-apis v0.0.0-20240814093917-dc663916b354 // indirect
 	github.com/loft-sh/apiserver v0.0.0-20240607231110-634aeeab2b36 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
@@ -176,10 +176,10 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
-	k8s.io/apiextensions-apiserver v0.30.1 // indirect
-	k8s.io/apiserver v0.30.1 // indirect
+	k8s.io/apiextensions-apiserver v0.30.4 // indirect
+	k8s.io/apiserver v0.30.4 // indirect
 	k8s.io/cli-runtime v0.29.1 // indirect
-	k8s.io/component-base v0.30.1 // indirect
+	k8s.io/component-base v0.30.4 // indirect
 	k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect

go.sum

@@ -676,10 +676,16 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9
 github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=
 github.com/loft-sh/admin-apis v0.0.0-20240203010124-3600c1c582a8 h1:nuY9Vgvabh2FlaTYp9yhzh3cBzY6jjFEFSsOvw9ZAJw=
 github.com/loft-sh/admin-apis v0.0.0-20240203010124-3600c1c582a8/go.mod h1:MWczNwKvWssHo1KaeZKaWDdRLYSNbWqQBGsTLoCNd7U=
+github.com/loft-sh/admin-apis v0.0.0-20240814093917-dc663916b354 h1:aqT1UeosCYB+3DEzfxW0evhcYHxhoBVAEWtkXLXFimI=
+github.com/loft-sh/admin-apis v0.0.0-20240814093917-dc663916b354/go.mod h1:MWczNwKvWssHo1KaeZKaWDdRLYSNbWqQBGsTLoCNd7U=
 github.com/loft-sh/agentapi/v4 v4.1.0-alpha.1 h1:exsfePvLXhVnuQHhFqXK5FxbPzBZpQiLhj8jQQ1zq8c=
 github.com/loft-sh/agentapi/v4 v4.1.0-alpha.1/go.mod h1:5U2JmT3BpfROQg1rX+DIpJ3/JuF62OYBveZ7GzOQA9o=
+github.com/loft-sh/agentapi/v4 v4.1.0-alpha.10 h1:XUtSAPorloQbXelYKX98L9TtxLbHTQVFjBhJVxpkYVA=
+github.com/loft-sh/agentapi/v4 v4.1.0-alpha.10/go.mod h1:fj/fZ17NpJo63Bpx75eSx+ZmonfTWr52WgIZVwevc48=
 github.com/loft-sh/api/v4 v4.1.0-alpha.1 h1:SLOhaRlJE3fkZfllaTeOiNnuOQMe/7u14wJXfqgkMio=
 github.com/loft-sh/api/v4 v4.1.0-alpha.1/go.mod h1:1p5qVavAJC/gVZZpJ/x1YQQt91fux7wZaqSixCX5QlI=
+github.com/loft-sh/api/v4 v4.1.0-alpha.10 h1:vUIQ7PgaLOA0wTOu1uPkHjgXvcO6myh2ersBsdxZo1c=
+github.com/loft-sh/api/v4 v4.1.0-alpha.10/go.mod h1:GTwXhRrjQGSaEhk9d/PkcEuoXl/jQwOZS6Tfjfa3ixQ=
 github.com/loft-sh/apiserver v0.0.0-20240607231110-634aeeab2b36 h1:1euJ7mNHMI2MMA+g+jEYYKtoMrHZTkUMapHnmjHb9lM=
 github.com/loft-sh/apiserver v0.0.0-20240607231110-634aeeab2b36/go.mod h1:XxI95azXiqXHiIDRiyDTpZcxdtXQlUqeU5VMMDz8INA=
 github.com/loft-sh/log v0.0.0-20240219160058-26d83ffb46ac h1:Gz/7Lb7WgdgIv+KJz87ORA1zvQW52tUqKPGyunlp4dQ=
@@ -1517,31 +1523,43 @@ k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
 k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8=
 k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY=
 k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM=
+k8s.io/api v0.30.4 h1:XASIELmW8w8q0i1Y4124LqPoWMycLjyQti/fdYHYjCs=
+k8s.io/api v0.30.4/go.mod h1:ZqniWRKu7WIeLijbbzetF4U9qZ03cg5IRwl8YVs8mX0=
 k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws=
 k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4=
+k8s.io/apiextensions-apiserver v0.30.4 h1:FwOMIk/rzZvM/Gx0IOz0+biZ+dlnlCeyfXW17uzV1qE=
+k8s.io/apiextensions-apiserver v0.30.4/go.mod h1:m8cAkJ9PVU8Olb4cPW4hrUDBZGvoSJ0kY0G0CfdGQac=
 k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
 k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
 k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
 k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U=
 k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
+k8s.io/apimachinery v0.30.4 h1:5QHQI2tInzr8LsT4kU/2+fSeibH1eIHswNx480cqIoY=
+k8s.io/apimachinery v0.30.4/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
 k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
 k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM=
 k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=
 k8s.io/apiserver v0.30.1 h1:BEWEe8bzS12nMtDKXzCF5Q5ovp6LjjYkSp8qOPk8LZ8=
 k8s.io/apiserver v0.30.1/go.mod h1:i87ZnQ+/PGAmSbD/iEKM68bm1D5reX8fO4Ito4B01mo=
+k8s.io/apiserver v0.30.4 h1:rHkGJhxd+m4jILrgkenwSmG4X0QXk6ecGuybzS/PQak=
+k8s.io/apiserver v0.30.4/go.mod h1:oyGAj9B9/0+I9huJyf4/8SMBF2mNh2bTMlu7703dkH8=
 k8s.io/cli-runtime v0.29.1 h1:By3WVOlEWYfyxhGko0f/IuAOLQcbBSMzwSaDren2JUs=
 k8s.io/cli-runtime v0.29.1/go.mod h1:vjEY9slFp8j8UoMhV5AlO8uulX9xk6ogfIesHobyBDU=
 k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
 k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k=
 k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0=
 k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q=
 k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
+k8s.io/client-go v0.30.4 h1:eculUe+HPQoPbixfwmaSZGsKcOf7D288tH6hDAdd+wY=
+k8s.io/client-go v0.30.4/go.mod h1:IBS0R/Mt0LHkNHF4E6n+SUDPG7+m2po6RZU7YHeOpzc=
 k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0=
 k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
 k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI=
 k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=
 k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ=
 k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI=
+k8s.io/component-base v0.30.4 h1:FlgKqazIkIIxpLA4wFXsiPiDllJn9fhsN3G4TeX7T7U=
+k8s.io/component-base v0.30.4/go.mod h1:Qd3h+OJxV/LrnriXG/E15ZK83dzd306qJHW9+87S5ls=
 k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
 k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
 k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
@@ -1577,6 +1595,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RCh
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4=
 sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw=
 sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
+sigs.k8s.io/controller-runtime v0.18.5 h1:nTHio/W+Q4aBlQMgbnC5hZb4IjIidyrizMai9P6n4Rk=
+sigs.k8s.io/controller-runtime v0.18.5/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=

pkg/devcontainer/config.go

@@ -33,8 +33,8 @@ func (r *runner) getRawConfig(options provider2.CLIOptions) (*config.DevContaine
 			},
 			Origin: "",
 		}, nil
-	} else if options.DevContainerSource != "" && crane.IsAvailable() {
-		localWorkspaceFolder, err := crane.PullConfigFromSource(r.WorkspaceConfig, r.Log)
+	} else if crane.ShouldUse(&options) {
+		localWorkspaceFolder, err := crane.PullConfigFromSource(r.WorkspaceConfig, &options, r.Log)
 		if err != nil {
 			return nil, err
 		}

pkg/devcontainer/crane/run.go

@@ -20,25 +20,39 @@ const (
 	PullCommand    = "pull"
 	DecryptCommand = "decrypt"
 
-	GitCrane = "git"
+	GitCrane         = "git"
+	EnvironmentCrane = "environment"
 
-	BinPath = "devpod-crane" // FIXME
-
-	tmpDirTemplate = "devpod-crane-*"
+	defaultBinName     = "devpod-crane"
+	envDevPodCraneName = "DEVPOD_CRANE_NAME"
+	tmpDirTemplate     = "devpod-crane-*"
 )
 
 type Content struct {
 	Files map[string]string `json:"files"`
 }
 
-// IsAvailable checks if devpod crane is installed in host system
-func IsAvailable() bool {
-	_, err := exec.LookPath(BinPath)
-	return err == nil
+type command struct {
+	cmd  string
+	args []string
 }
 
-func runCommand(command string, args ...string) (string, error) {
-	cmd := exec.Command(BinPath, append([]string{command}, args...)...)
+func New(cmd string) *command {
+	return &command{cmd: cmd}
+}
+
+func (c *command) WithFlag(flag, val string) *command {
+	c.args = append(c.args, flag, val)
+	return c
+}
+
+func (c *command) WithArg(arg string) *command {
+	c.args = append(c.args, arg)
+	return c
+}
+
+func (c *command) Run() (string, error) {
+	cmd := exec.Command(c.cmd, c.args...)
 
 	var outBuf, errBuf bytes.Buffer
 	cmd.Stdout = &outBuf
@@ -51,15 +65,37 @@ func runCommand(command string, args ...string) (string, error) {
 	return outBuf.String(), nil
 }
 
+// ShouldUse takes CLIOptions and returns true if crane should be used
+func ShouldUse(cliOptions *provider2.CLIOptions) bool {
+	return IsAvailable() && (cliOptions.DevContainerSource != "" ||
+		cliOptions.EnvironmentTemplate != "")
+}
+
+// IsAvailable checks if devpod crane is installed in host system
+func IsAvailable() bool {
+	_, err := exec.LookPath(getBinName())
+	return err == nil
+}
+
 // PullConfigFromSource pulls devcontainer config from configSource using git crane and returns config path
-func PullConfigFromSource(workspaceInfo *provider2.AgentWorkspaceInfo, log log.Logger) (string, error) {
-	data, err := runCommand(PullCommand, GitCrane, workspaceInfo.CLIOptions.DevContainerSource)
+func PullConfigFromSource(workspaceInfo *provider2.AgentWorkspaceInfo, options *provider2.CLIOptions, log log.Logger) (string, error) {
+	var data string
+	var err error
+
+	switch {
+	case options.EnvironmentTemplate != "":
+		data, err = New(PullCommand).WithArg(EnvironmentCrane).WithArg(options.EnvironmentTemplate).Run()
+	case options.DevContainerSource != "":
+		data, err = New(PullCommand).WithArg(GitCrane).WithArg(options.DevContainerSource).Run()
+	default:
+		err = fmt.Errorf("failed to pull config from source based on options")
+	}
 	if err != nil {
 		return "", err
 	}
 
 	if craneSigningKey != "" {
-		data, err = runCommand(DecryptCommand, data, "--key", craneSigningKey)
+		data, err = New(DecryptCommand).WithArg(data).WithFlag("--key", craneSigningKey).Run()
 		if err != nil {
 			return "", err
 		}
@@ -111,3 +147,10 @@ func storeFilesInDirectory(content *Content, path string) (string, error) {
 
 	return path, nil
 }
+
+func getBinName() string {
+	if name := os.Getenv(envDevPodCraneName); name != "" {
+		return name
+	}
+	return defaultBinName
+}

pkg/devcontainer/setup.go

@@ -79,7 +79,7 @@ func (r *runner) setupContainer(
 		Agent:            r.WorkspaceConfig.Agent,
 		ContentFolder:    r.WorkspaceConfig.ContentFolder,
 	}
-	if r.WorkspaceConfig.CLIOptions.DevContainerSource != "" && crane.IsAvailable() {
+	if crane.ShouldUse(&r.WorkspaceConfig.CLIOptions) {
 		workspaceConfig.PullFromInsideContainer = "true"
 	}
 	// compress container workspace info

pkg/provider/workspace.go

@@ -191,6 +191,7 @@ type CLIOptions struct {
 	DevContainerImage    string            `json:"devContainerImage,omitempty"`
 	DevContainerPath     string            `json:"devContainerPath,omitempty"`
 	DevContainerSource   string            `json:"devContainerSource,omitempty"`
+	EnvironmentTemplate  string            `json:"environmentTemplate,omitempty"`
 	WorkspaceEnv         []string          `json:"workspaceEnv,omitempty"`
 	WorkspaceEnvFile     []string          `json:"workspaceEnvFile,omitempty"`
 	InitEnv              []string          `json:"initEnv,omitempty"`