[Local Environment Setup] Automatically Aggregate App CRDs Under Graf…

…ana API Server (#446)

As part of the `grafana-app-sdk project local generate`, create a new
script to aggregate app CRDs under a grafana API server. Additionally,
add a tilt step in the new-project generated tilt file to do this
aggregation automatically.

Resolves #271

cmd/grafana-app-sdk/project_local.go

@@ -219,14 +219,22 @@ func projectLocalEnvGenerate(cmd *cobra.Command, _ []string) error {
 		}
 	}
 
-	k8sYAML, err := generateKubernetesYAML(parseFunc, pluginID, *config)
+	k8sYAML, genProps, err := generateKubernetesYAML(parseFunc, pluginID, *config)
 	if err != nil {
 		return err
 	}
 	err = writeFile(filepath.Join(localGenPath, "dev-bundle.yaml"), k8sYAML)
 	if err != nil {
 		return err
 	}
+	aggregateScript, err := generateAggregationScript(*config, genProps)
+	if err != nil {
+		return err
+	}
+	err = writeExecutableFile(filepath.Join(localGenPath, "aggregate-apiserver.sh"), aggregateScript)
+	if err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -292,6 +300,11 @@ func generateK3dConfig(projectRoot string, config localEnvConfig) ([]byte, error
 	return buf.Bytes(), err
 }
 
+type scriptGenProperties struct {
+	Port int
+	CRDs []yamlGenPropsCRD
+}
+
 type yamlGenProperties struct {
 	PluginID                  string
 	PluginIDKube              string
@@ -344,8 +357,8 @@ type crdYAML struct {
 
 var kubeReplaceRegexp = regexp.MustCompile(`[^a-z0-9\-]`)
 
-//nolint:funlen,errcheck,revive
-func generateKubernetesYAML(crdGenFunc func() (codejen.Files, error), pluginID string, config localEnvConfig) ([]byte, error) {
+//nolint:funlen,errcheck,revive,gocyclo
+func generateKubernetesYAML(crdGenFunc func() (codejen.Files, error), pluginID string, config localEnvConfig) ([]byte, yamlGenProperties, error) {
 	output := bytes.Buffer{}
 	props := yamlGenProperties{
 		PluginID:       pluginID,
@@ -393,7 +406,7 @@ func generateKubernetesYAML(crdGenFunc func() (codejen.Files, error), pluginID s
 		// Generate cert bundle
 		bundle, err := generateCerts(fmt.Sprintf("%s-operator.default.svc", props.PluginID))
 		if err != nil {
-			return nil, err
+			return nil, props, err
 		}
 		props.WebhookProperties.Base64Cert = base64.StdEncoding.EncodeToString(bundle.cert)
 		props.WebhookProperties.Base64Key = base64.StdEncoding.EncodeToString(bundle.key)
@@ -403,19 +416,19 @@ func generateKubernetesYAML(crdGenFunc func() (codejen.Files, error), pluginID s
 	// Generate CRD YAML files, add the CRD metadata to the props
 	crdFiles, err := crdGenFunc()
 	if err != nil {
-		return nil, err
+		return nil, props, err
 	}
 	for _, f := range crdFiles {
 		// If converting webhooks are enabled, upate the yaml
 		// TODO: this is a hack workaround for now, this should eventually be in the CRD generator
 		if props.WebhookProperties.Converting != "" {
 			rawCRD := make(map[string]any)
 			if err := yaml.Unmarshal(f.Data, &rawCRD); err != nil {
-				return nil, err
+				return nil, props, err
 			}
 			spec, ok := rawCRD["spec"].(map[string]any)
 			if !ok {
-				return nil, fmt.Errorf("could not parse CRD")
+				return nil, props, fmt.Errorf("could not parse CRD")
 			}
 			spec["conversion"] = map[string]any{
 				"strategy": "Webhook",
@@ -434,15 +447,15 @@ func generateKubernetesYAML(crdGenFunc func() (codejen.Files, error), pluginID s
 			rawCRD["spec"] = spec
 			f.Data, err = yaml.Marshal(rawCRD)
 			if err != nil {
-				return nil, fmt.Errorf("unable to re-marshal CRD YAML after added conversion strategy: %w", err)
+				return nil, props, fmt.Errorf("unable to re-marshal CRD YAML after added conversion strategy: %w", err)
 			}
 		}
 
 		output.Write(append(f.Data, []byte("\n---\n")...))
 		yml := crdYAML{}
 		err = yaml.Unmarshal(f.Data, &yml)
 		if err != nil {
-			return nil, err
+			return nil, props, err
 		}
 		versions := make([]string, 0)
 		for _, v := range yml.Spec.Versions {
@@ -461,22 +474,33 @@ func generateKubernetesYAML(crdGenFunc func() (codejen.Files, error), pluginID s
 	// RBAC for CRDs
 	tmplRoles, err := template.ParseFS(localEnvFiles, "templates/local/generated/crd_roles.yaml")
 	if err != nil {
-		return nil, err
+		return nil, props, err
 	}
 	for _, c := range props.CRDs {
 		err = tmplRoles.Execute(&output, c)
 		if err != nil {
-			return nil, err
+			return nil, props, err
 		}
 		output.Write([]byte("\n---\n"))
 	}
 
+	// RBAC for aggregator
+	tmplAggregatorAccess, err := template.ParseFS(localEnvFiles, "templates/local/generated/aggregator-access.yaml")
+	if err != nil {
+		return nil, props, err
+	}
+	err = tmplAggregatorAccess.Execute(&output, props)
+	if err != nil {
+		return nil, props, err
+	}
+	output.Write([]byte("\n---\n"))
+
 	// Datasources
 	addedDeps := make(map[string]struct{})
 	for i, ds := range config.Datasources {
 		err := localGenerateDatasourceYAML(ds, i == 0, &props, addedDeps, &output)
 		if err != nil {
-			return nil, err
+			return nil, props, err
 		}
 		output.WriteString("\n---\n")
 	}
@@ -492,14 +516,30 @@ func generateKubernetesYAML(crdGenFunc func() (codejen.Files, error), pluginID s
 		output.WriteString("---\n")
 		tmplOperator, err := template.ParseFS(localEnvFiles, "templates/local/generated/operator.yaml")
 		if err != nil {
-			return nil, err
+			return nil, props, err
 		}
 		err = tmplOperator.Execute(&output, props)
 		if err != nil {
-			return nil, err
+			return nil, props, err
 		}
 	}
-	return output.Bytes(), err
+	return output.Bytes(), props, err
+}
+
+func generateAggregationScript(config localEnvConfig, genProps yamlGenProperties) ([]byte, error) {
+	tmpl, err := template.ParseFS(localEnvFiles, "templates/local/generated/configure-grafana.sh")
+	if err != nil {
+		return nil, err
+	}
+	output := bytes.Buffer{}
+	err = tmpl.Execute(&output, scriptGenProperties{
+		Port: config.Port,
+		CRDs: genProps.CRDs,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return output.Bytes(), nil
 }
 
 //nolint:revive

cmd/grafana-app-sdk/templates/local/Tiltfile

@@ -74,3 +74,6 @@ services = [('%s' % name(r)) for r in yaml_objects if r['kind'] == 'Service']
 webhooks = [r for r in yaml_objects if (r['kind'] == 'ValidatingWebhookConfiguration' or r['kind'] == 'MutatingWebhookConfiguration')]
 if len(webhooks) > 0:
   k8s_resource(new_name='Webhooks', objects=[('%s' % name(r)) for r in webhooks], resource_deps=services)
+
+# Aggregate API server CRD's to the grafana deployment
+local_resource('API Aggregation', cmd='sh ./generated/aggregate-apiserver.sh in_cluster', resource_deps=['grafana'])

cmd/grafana-app-sdk/templates/local/config.yaml

@@ -36,4 +36,4 @@ datasourceConfigs:
 generateGrafanaDeployment: true
 
 # which grafana image to use
-grafanaImage: grafana/grafana-enterprise:11.2.2
+grafanaImage: grafana/grafana-enterprise:main

cmd/grafana-app-sdk/templates/local/generated/aggregator-access.yaml

@@ -0,0 +1,25 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aggregator-access
+rules:{{ range $.CRDs }}
+  - apiGroups: ["{{.Group}}"]
+    resources: ["*"]
+    verbs: ["*"]{{ end }}
+  - nonResourceURLs:
+      - "/openapi/*"{{ range $.CRDs}}
+      - "/apis/{{.Group}}/*"{{ end }}
+    verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: aggregator-unauth-access-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aggregator-access
+subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: Group
+    name: system:unauthenticated

cmd/grafana-app-sdk/templates/local/generated/configure-grafana.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -eufo pipefail
+
+GRAFANA_RUN_MODE="${1}"
+GRAFANA_HOST="grafana.k3d.localhost:{{.Port}}"
+
+if [ $GRAFANA_RUN_MODE == "in_cluster" ]; then
+  KUBEAPI=($(kubectl run bash-local-env -it --rm --image=bash --restart=Never --command -- "bash" "-c" 'echo "$KUBERNETES_SERVICE_HOST $KUBERNETES_SERVICE_PORT"'))
+elif [ $GRAFANA_RUN_MODE == "standalone" ]; then
+  CONTROLPLANE=($(kubectl cluster-info | grep "control plane"))
+  ENDPOINT=$(echo ${CONTROLPLANE[${#CONTROLPLANE[@]}-1]} | sed 's~http[s]*://~~g')
+  IFS=: read -r -a KUBEAPI <<< "$ENDPOINT"
+  read -p "Grafana API Server URL (host:port): " HOST
+fi
+HOST=${KUBEAPI[0]}
+PORT=${KUBEAPI[1]}
+
+# Create a service account token to use in our requests (anonymous auth isn't allowed for apiservices)
+sa=$(curl "http://${GRAFANA_HOST}/api/serviceaccounts/" \
+  --request POST \
+  --header "Content-Type: application/json" \
+  --header "Accept: application/json" \
+  --data '{"name":"Aggregation User","role":"Admin"}' | jq .id)
+echo "SA: ${sa}"
+token=$(curl "http://${GRAFANA_HOST}/api/serviceaccounts/${sa}/tokens" \
+  --request POST \
+  --header "Content-Type: application/json" \
+  --header "Accept: application/json" \
+  --data '{"name":"sa-1-aggregation-user-e2a6a474-bdcd-4bac-8449-0deb7274484a"}'| jq -r '.key')
+echo "TOKEN: ${token}"
+
+{{ range .CRDs }}{{$crd:=.}}{{range .Versions}}curl \
+  "http://${GRAFANA_HOST}/apis/apiregistration.k8s.io/v1/apiservices?fieldManager=kubectl-create&fieldValidation=Strict" \
+  --request POST \
+  --header "Content-Type: application/json" \
+  --header "Authorization: Bearer ${token}" \
+  --data @- << EOF
+{
+  "apiVersion": "apiregistration.k8s.io/v1",
+  "kind": "APIService",
+  "metadata": {
+    "name": "{{.}}.{{$crd.Group}}"
+  },
+  "spec": {
+    "version": "{{.}}",
+    "insecureSkipTLSVerify": true,
+    "group": "{{$crd.Group}}",
+    "groupPriorityMinimum": 1000,
+    "versionPriority": 15,
+    "service": {
+      "name": "example-apiserver",
+      "namespace": "default",
+      "port": ${PORT}
+    }
+  }
+}
+EOF
+{{ end }}{{end}}
+
+curl \
+  "http://${GRAFANA_HOST}/apis/service.grafana.app/v0alpha1/namespaces/default/externalnames?fieldManager=kubectl-create&fieldValidation=Strict" \
+  --request POST \
+  --header "Content-Type: application/json" \
+  --header "Authorization: Bearer ${token}" \
+  --data @- << EOF
+{
+  "apiVersion": "service.grafana.app/v0alpha1",
+  "kind": "ExternalName",
+  "metadata": {
+    "name": "example-apiserver",
+    "namespace": "default"
+  },
+  "spec": {
+    "host": "${HOST}"
+  }
+}
+EOF

cmd/grafana-app-sdk/templates/local/generated/grafana.yaml

@@ -48,7 +48,8 @@ metadata:
   labels:
     name: grafana
 data:
-  grafana.ini: | 
+  grafana.ini: |
+    app_mode = development
     [analytics]
     reporting_enabled = false
     [tracing.opentelemetry.otlp]
@@ -70,6 +71,11 @@ data:
     enabled = true
     [users]
     default_theme = dark
+    [feature_toggles]
+    grafanaAPIServerEnsureKubectlAccess = true
+    ; disable the experimental APIs flag to disable bundling of the example service locally
+    grafanaAPIServerWithExperimentalAPIs = false
+    kubernetesAggregator = true
 ---
 apiVersion: v1
 kind: Service

cmd/grafana-app-sdk/templates/local/generated/k3d-config.json

@@ -11,6 +11,9 @@
     "kubeconfig": {
       "switchCurrentContext": true,
       "updateDefaultKubeconfig": true
+    },
+    "k3s": {
+      "extraArgs": [{"arg":"--kube-apiserver-arg=anonymous-auth=true","nodeFilters":["server:*"]}]
     }
   },
   "ports": [