[#3397] Airy CLI Installation with Terraform (#3404)

* [#3360] Bug with external Go dependency build

* Also add disable proto globally file to this PR and fix gazelle import order

* [#3397] Change CLI for AWS to use Terraform

* Decouple status from kubernetes

* Update output message and aws docs

* Update aws docs

* Clarify aws docs

Co-authored-by: Christoph Pröschel <[email protected]>
Co-authored-by: ljupcovangelski <[email protected]>

Author: 3 people

.gitignore

@@ -30,6 +30,7 @@ package-lock.json
 *.tfstate.*
 .terraform.tfstate.lock.info
 .terraform.lock.hcl
+/terraform
 
 # Kubernetes
 .kubeconfig

BUILD

@@ -181,6 +181,7 @@ exports_files(
 # gazelle:proto disable_global
 # gazelle:build_file_name BUILD
 # gazelle:prefix
+# gazelle:exclude infrastructure/terraform
 gazelle(name = "gazelle")
 
 nogo(

cli/go.mod

@@ -28,8 +28,10 @@ require (
 )
 
 require (
-	cloud.google.com/go v0.100.2 // indirect
-	cloud.google.com/go/compute v1.6.1 // indirect
+	cloud.google.com/go v0.102.0 // indirect
+	cloud.google.com/go/compute v1.7.0 // indirect
+	cloud.google.com/go/iam v0.3.0 // indirect
+	cloud.google.com/go/storage v1.22.1 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
@@ -44,6 +46,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.11.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.16.6 // indirect
 	github.com/aws/smithy-go v1.11.2 // indirect
+	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful v2.15.0+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
@@ -52,21 +55,29 @@ require (
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-getter v1.6.2 // indirect
+	github.com/hashicorp/go-safetemp v1.0.0 // indirect
+	github.com/hashicorp/go-version v1.1.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.11.2 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -81,6 +92,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.3.0 // indirect
+	github.com/ulikunitz/xz v0.5.8 // indirect
 	github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
 	golang.org/x/net v0.0.0-20220630215102-69896b714898 // indirect
@@ -89,7 +101,19 @@ require (
 	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 // indirect
+	golang.org/x/net v0.0.0-20220630215102-69896b714898 // indirect
+	golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 // indirect
+	golang.org/x/sys v0.0.0-20220624220833-87e55d714810 // indirect
+	golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
+	google.golang.org/api v0.86.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect
+	google.golang.org/grpc v1.47.0 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect

cli/go.sum

@@ -65,11 +65,21 @@ func create(cmd *cobra.Command, args []string) {
 	overrides.Version = version
 	overrides.Namespace = namespace
 	overrides.TrackingDisabled = disableTracking
+	if err := provider.CheckEnvironment(); err != nil {
+		console.Exit("please set up the required environment for the installation", err)
+	}
+
 	dir, err := workspace.Create(workspacePath, overrides)
 	if err != nil {
 		console.Exit("could not initialize Airy workspace directory", err)
 	}
 	fmt.Println("📁 Initialized Airy workspace directory at", dir.GetPath("."))
+
+	installDir, err := provider.PreInstallation(dir.GetPath("."))
+	if err != nil {
+		console.Exit("could not set up files for install in Airy workspace directory", err)
+	}
+	fmt.Println("📁 Set up installation directory in the Airy workspace at", installDir)
 	if initOnly {
 		os.Exit(0)
 	}
@@ -78,69 +88,69 @@ func create(cmd *cobra.Command, args []string) {
 	fmt.Fprintln(w)
 	fmt.Fprintln(w, providerName, "provider output:")
 	fmt.Fprintln(w)
-	context, err := provider.Provision(providerConfig, dir)
+	context, err := provider.Provision(providerConfig, workspace.ConfigDir{Path: installDir})
 	fmt.Fprintln(w)
 	if err != nil {
-		console.Exit("could not provision cluster: ", err)
-	}
-
-	fmt.Println("✅ Cluster provisioned")
-
-	clientset, err := context.GetClientSet()
-	if err != nil {
-		console.Exit("could not get clientset: ", err)
+		console.Exit("could not install Airy: ", err)
 	}
+	if providerName == "minikube" { // TEMP fix to keep minikube working
+		clientset, err := context.GetClientSet()
+		if err != nil {
+			console.Exit("could not get clientset: ", err)
+		}
 
-	if err = context.Store(); err != nil {
-		console.Exit("could not store the kube context: ", err)
-	}
+		if err = context.Store(); err != nil {
+			console.Exit("could not store the kube context: ", err)
+		}
 
-	helm := helm.New(clientset, version, namespace, dir.GetAiryYaml())
-	if err := helm.Setup(); err != nil {
-		console.Exit("setting up Helm failed with err: ", err)
-	}
+		helm := helm.New(clientset, version, namespace, dir.GetAiryYaml())
+		if err := helm.Setup(); err != nil {
+			console.Exit("setting up Helm failed with err: ", err)
+		}
 
-	fmt.Println("🚀 Starting airy with default components")
+		fmt.Println("🚀 Starting core with default components")
 
-	if err := helm.InstallCharts(); err != nil {
-		console.Exit("installing Helm charts failed with err: ", err)
-	}
+		if err := helm.InstallCharts(); err != nil {
+			console.Exit("installing Helm charts failed with err: ", err)
+		}
 
-	if err = provider.PostInstallation(providerConfig, namespace, dir); err != nil {
-		console.Exit("failed to run post installation hook: ", err)
-	}
+		if err = provider.PostInstallation(providerConfig, namespace, dir); err != nil {
+			console.Exit("failed to run post installation hook: ", err)
+		}
 
-	fmt.Println("🎉 Your Airy Core is ready")
+		fmt.Println("🎉 Your Airy Core is ready")
 
-	coreConfig, err := k8s.GetCmData("core-config", namespace, clientset)
-	if err != nil {
-		console.Exit("failed to get hosts from installation")
-	}
+		coreConfig, err := k8s.GetCmData("core-config", namespace, clientset)
+		if err != nil {
+			console.Exit("failed to get hosts from installation")
+		}
 
-	fmt.Println("\t 👩‍🍳 Available hosts:")
-	for hostName, host := range coreConfig {
-		switch hostName {
-		case "HOST":
-			fmt.Printf("\t\t %s:\t %s", "Host", host)
-			fmt.Println()
-		case "API_HOST":
-			fmt.Printf("\t\t %s:\t %s", "API", host)
-			fmt.Println()
-		case "NGROK":
-			fmt.Printf("\t\t %s:\t %s", "NGROK", host)
-			fmt.Println()
+		fmt.Println("\t 👩‍🍳 Available hosts:")
+		for hostName, host := range coreConfig {
+			switch hostName {
+			case "HOST":
+				fmt.Printf("\t\t %s:\t %s", "Host", host)
+				fmt.Println()
+			case "API_HOST":
+				fmt.Printf("\t\t %s:\t %s", "API", host)
+				fmt.Println()
+			case "NGROK":
+				fmt.Printf("\t\t %s:\t %s", "NGROK", host)
+				fmt.Println()
+			}
 		}
-	}
 
-	fmt.Println()
+		fmt.Println("✅ Airy Installed")
+		fmt.Println()
 
-	viper.Set("provider", provider)
-	viper.Set("namespace", namespace)
-	viper.WriteConfig()
+		viper.Set("provider", provider)
+		viper.Set("namespace", namespace)
+		viper.WriteConfig()
 
-	airyAnalytics.Track(analytics.Track{
-		UserId: coreConfig["CORE_ID"],
-		Event:  "installation_succesful"})
-	fmt.Printf("📚 For more information about the %s provider visit https://airy.co/docs/core/getting-started/installation/%s", providerName, providerName)
-	fmt.Println()
+		airyAnalytics.Track(analytics.Track{
+			UserId: coreConfig["CORE_ID"],
+			Event:  "installation_succesful"})
+		fmt.Printf("📚 For more information about the %s provider visit https://airy.co/docs/core/getting-started/installation/%s", providerName, providerName)
+		fmt.Println()
+	}
 }

cli/pkg/cmd/create/create.go

@@ -8,7 +8,6 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//cli/pkg/console",
-        "//cli/pkg/kube",
         "//lib/go/httpclient",
         "@com_github_spf13_cobra//:cobra",
         "@com_github_spf13_viper//:viper",

cli/pkg/cmd/status/BUILD

@@ -21,6 +21,7 @@ var StatusCmd = &cobra.Command{
 
 func status(cmd *cobra.Command, args []string) {
 	c := httpclient.NewClient(viper.GetString("apihost"), viper.GetString("authToken"))
+
 	res, err := c.Config()
 
 	if err != nil {

cli/pkg/cmd/status/status.go

@@ -17,15 +17,7 @@ go_library(
         "//cli/pkg/kube",
         "//cli/pkg/workspace",
         "//cli/pkg/workspace/template",
-        "@com_github_aws_aws_sdk_go//aws",
-        "@com_github_aws_aws_sdk_go_v2_config//:config",
-        "@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2",
-        "@com_github_aws_aws_sdk_go_v2_service_ec2//types",
-        "@com_github_aws_aws_sdk_go_v2_service_eks//:eks",
-        "@com_github_aws_aws_sdk_go_v2_service_eks//types",
-        "@com_github_aws_aws_sdk_go_v2_service_iam//:iam",
-        "@com_github_aws_aws_sdk_go_v2_service_iam//types",
-        "@com_github_twinproduction_go_color//:go-color",
+        "@com_github_hashicorp_go_getter//:go-getter",
         "@in_gopkg_segmentio_analytics_go_v3//:analytics-go_v3",
     ],
 )