[codegen] Source Codegen from CUE App Manifest (#483)

## What this PR Does This PR changes the format of CUE codegen, moving from reading all top-level CUE selectors as kinds to reading a single `manifest` top-level field (or a different name if supplied). Other top-level fields are ignored, allowing kinds to still be defined as root-level keys, but they will only be parsed for codegen if listed in the `kinds` field in the manifest. This change is required to properly generate a manifest that * enforces that all kinds for an app have the same group (`group` is now a manifest-level attribute, and if it is specified in the kind, it must match the full computed API group for the manifest) * allows for specifying permissions required for the app operator * Ensures that all kinds for an app are now API resources (the `model` type has been fully removed) There are some slight changes around writing a kind, namely that the fields previous in `apiResource` are either moved into the main body of the kind, or into the manifest itself, as all kinds are now always API resources. * `apiResource.scope` is now just `scope` * `apiResource.validateion` is now just `validation` * `apiResource.mutation` is now just `mutation` * `apiResource.conversion` is now just `conversion` * `apiResource.groupOverride` is now in the manifest, as `manifest.groupOverride`, as it will apply to all kinds for the manifest The `grafana-app-sdk project init` command now generates a `kinds/manifest.cue` with a basic manifest, and `grafana-app-sdk project kind add` still creates a kind CUE file, but now also adds the kind to the list of kinds in `manifest.cue`, found under `manifest.kinds`. In the `codegen` package, jennies may use `codegen.Kind` or `codegen.AppManifest` as their input, to allow for not requiring too much change there. The `cuekind.Parser` now doesn't implement `codegen.Parser[codegen.Kind]` itself, and instead has methods to get a parser that implements either `codegen.Parser[codegen.Kind]` or `codegen.Parser[codegen.AppManifest]`. This is likely an intermediary step as we continue to improve the codegen pipeline's structure. This PR aims mainly to change the input type of `--format=cue` to use the manifest, and define that manifest properly, with a low-impact changes as possible to the rest of the project. --------- Co-authored-by: Igor Suleymanov <[email protected]>
grafana · Dec 4, 2024 · a9d2b24 · a9d2b24
1 parent afa5519
commit a9d2b24
Show file tree

Hide file tree

Showing 82 changed files with 1,004 additions and 866 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -63,14 +63,15 @@ jobs:
         run: |
           mkdir -p codegen-tests && cd codegen-tests
           grafana-app-sdk project init "codegen-tests"
+          rm -rf kinds/manifest.cue
           cp ../grafana-app-sdk/codegen/cuekind/testing/*.cue kinds/
           mkdir -p cmp && cp -R ../grafana-app-sdk/codegen/testing/golden_generated/* cmp/
           find ./cmp -iname '*.txt' -exec bash -c 'mv -- "$1" "${1%.txt}"' bash {} \;
       - name: Generate code
         run: |
           cd codegen-tests
-          grafana-app-sdk generate --kindgrouping=kind --gogenpath=pkg/gen1 --tsgenpath=ts/gen1 --crdencoding=json --nomanifest
-          grafana-app-sdk generate --kindgrouping=group --gogenpath=pkg/gen2 --tsgenpath=ts/gen2 --crdencoding=yaml --nomanifest
+          grafana-app-sdk generate --kindgrouping=kind --gogenpath=pkg/gen1 --tsgenpath=ts/gen1 --crdencoding=json --nomanifest --selectors=customManifest,testManifest
+          grafana-app-sdk generate --kindgrouping=group --gogenpath=pkg/gen2 --tsgenpath=ts/gen2 --crdencoding=yaml --nomanifest --selectors=customManifest,testManifest
           diff pkg/gen1/resource/customkind cmp/go/groupbykind/customkind > diff.txt
           sed -i '/^Common subdirectories/d' diff.txt
           difflines=$(wc -l diff.txt | awk '{ print $1 }')
@@ -139,12 +140,12 @@ jobs:
         run: |
           mkdir -p test-project && cd test-project
           grafana-app-sdk project init "test-project"
+          rm -rf kinds/manifest.cue
           printf "\nreplace github.com/grafana/grafana-app-sdk => $(readlink -f ../grafana-app-sdk)\n" >> go.mod
           printf "\nreplace github.com/grafana/grafana-app-sdk/plugin => $(readlink -f ../grafana-app-sdk/plugin)\n" >> go.mod
           cp ../grafana-app-sdk/codegen/cuekind/testing/customkind.cue kinds/customkind.cue
-          cp ../grafana-app-sdk/codegen/cuekind/testing/customkind2.cue kinds/customkind2.cue
-          grafana-app-sdk generate
-          grafana-app-sdk project component add frontend backend operator --plugin-id=test-project
+          grafana-app-sdk generate --selectors=customManifest
+          grafana-app-sdk project component add frontend backend operator --plugin-id=test-project --selectors=customManifest
           go mod tidy
           go build cmd/operator/*.go
           go build plugin/pkg/*.go

diff --git a/app/manifest.cue b/app/manifest.cue
@@ -63,10 +63,21 @@ manifest: versions: v1alpha1: {
 			conversion: bool
 			versions: [...#ManifestKindVersion]
 		}
+		#KindPermission: {
+			group: string
+			resource: string
+			actions: [...string]
+		}
 		spec: {
 			appName: string
 			group: string
 			kinds: [...#ManifestKind]
+			// ExtraPermissions contains additional permissions needed for an app's backend component to operate.
+			// Apps implicitly have all permissions for kinds they managed (defined in `kinds`).
+			extraPermissions: {
+				// accessKinds is a list of KindPermission objects for accessing additional kinds provided by other apps
+				accessKinds: [...#KindPermission]
+			}
 		}
 	}
 }
diff --git a/app/manifest.go b/app/manifest.go
@@ -72,6 +72,9 @@ type ManifestData struct {
 	Group string `json:"group" yaml:"group"`
 	// Kinds is a list of all Kinds maintained by this App
 	Kinds []ManifestKind `json:"kinds,omitempty" yaml:"kinds,omitempty"`
+	// Permissions is the extra permissions for non-owned kinds this app needs to operate its backend.
+	// It may be nil if no extra permissions are required.
+	ExtraPermissions *Permissions `json:"extraPermissions,omitempty" yaml:"extraPermissions,omitempty"`
 }
 
 // ManifestKind is the manifest for a particular kind, including its Kind, Scope, and Versions
@@ -150,6 +153,18 @@ const (
 	AdmissionOperationConnect AdmissionOperation = "CONNECT"
 )
 
+type Permissions struct {
+	AccessKinds []KindPermission `json:"accessKinds,omitempty" yaml:"accessKinds,omitempty"`
+}
+
+type KindPermissionAction string
+
+type KindPermission struct {
+	Group    string                 `json:"group" yaml:"group"`
+	Resource string                 `json:"resource" yaml:"resource"`
+	Actions  []KindPermissionAction `json:"actions,omitempty" yaml:"actions,omitempty"`
+}
+
 func VersionSchemaFromMap(openAPISchema map[string]any) (*VersionSchema, error) {
 	vs := &VersionSchema{
 		raw: openAPISchema,

diff --git a/cmd/grafana-app-sdk/generate.go b/cmd/grafana-app-sdk/generate.go
@@ -185,14 +185,18 @@ func generateKindsCue(modFS fs.FS, cfg kindGenConfig, selectors ...string) (code
 	if err != nil {
 		return nil, err
 	}
-	generator, err := codegen.NewGenerator[codegen.Kind](parser, modFS)
+	// Slightly hacky multiple generators as an intermediary while we move to a better system.
+	// Both still source from a Manifest, but generatorForKinds supplies []Kind to jennies, vs AppManifest
+	generatorForKinds, err := codegen.NewGenerator[codegen.Kind](parser.KindParser(true), modFS)
+	if err != nil {
+		return nil, err
+	}
+	generatorForManifest, err := codegen.NewGenerator[codegen.AppManifest](parser.ManifestParser(), modFS)
 	if err != nil {
 		return nil, err
 	}
 	// Resource
-	resourceFiles, err := generator.FilteredGenerate(cuekind.ResourceGenerator(cfg.GroupKinds), func(kind codegen.Kind) bool {
-		return kind.Properties().APIResource != nil
-	}, selectors...)
+	resourceFiles, err := generatorForKinds.Generate(cuekind.ResourceGenerator(cfg.GroupKinds), selectors...)
 	if err != nil {
 		return nil, err
 	}
@@ -203,33 +207,7 @@ func generateKindsCue(modFS fs.FS, cfg kindGenConfig, selectors ...string) (code
 	for i, f := range resourceFiles {
 		resourceFiles[i].RelativePath = filepath.Join(relativePath, f.RelativePath)
 	}
-	// Model
-	modelFiles, err := generator.FilteredGenerate(cuekind.ModelsGenerator(true, cfg.GroupKinds), func(kind codegen.Kind) bool {
-		return kind.Properties().APIResource == nil
-	}, selectors...)
-	if err != nil {
-		return nil, err
-	}
-	for i, f := range modelFiles {
-		prefix := cfg.GoGenBasePath
-		if cfg.PrefixPathWithType {
-			prefix = filepath.Join(prefix, targetModel+"s")
-		}
-		modelFiles[i].RelativePath = filepath.Join(prefix, f.RelativePath)
-	}
-	// TypeScript
-	tsModelFiles, err := generator.FilteredGenerate(cuekind.TypeScriptModelsGenerator(true), func(kind codegen.Kind) bool {
-		return kind.Properties().APIResource == nil
-	}, selectors...)
-	if err != nil {
-		return nil, err
-	}
-	for i, f := range tsModelFiles {
-		tsModelFiles[i].RelativePath = filepath.Join(cfg.TSGenBasePath, f.RelativePath)
-	}
-	tsResourceFiles, err := generator.FilteredGenerate(cuekind.TypeScriptResourceGenerator(), func(kind codegen.Kind) bool {
-		return kind.Properties().APIResource != nil
-	}, selectors...)
+	tsResourceFiles, err := generatorForKinds.Generate(cuekind.TypeScriptResourceGenerator(), selectors...)
 	if err != nil {
 		return nil, err
 	}
@@ -243,9 +221,7 @@ func generateKindsCue(modFS fs.FS, cfg kindGenConfig, selectors ...string) (code
 		if cfg.CRDEncoding == "yaml" {
 			encFunc = yaml.Marshal
 		}
-		crdFiles, err = generator.FilteredGenerate(cuekind.CRDGenerator(encFunc, cfg.CRDEncoding), func(kind codegen.Kind) bool {
-			return kind.Properties().APIResource != nil
-		}, selectors...)
+		crdFiles, err = generatorForKinds.Generate(cuekind.CRDGenerator(encFunc, cfg.CRDEncoding), selectors...)
 		if err != nil {
 			return nil, err
 		}
@@ -265,9 +241,7 @@ func generateKindsCue(modFS fs.FS, cfg kindGenConfig, selectors ...string) (code
 			if cfg.CRDEncoding == "yaml" {
 				encFunc = yaml.Marshal
 			}
-			manifestFiles, err = generator.FilteredGenerate(cuekind.ManifestGenerator(encFunc, cfg.CRDEncoding, ""), func(kind codegen.Kind) bool {
-				return kind.Properties().APIResource != nil
-			}, selectors...)
+			manifestFiles, err = generatorForManifest.Generate(cuekind.ManifestGenerator(encFunc, cfg.CRDEncoding), selectors...)
 			if err != nil {
 				return nil, err
 			}
@@ -276,9 +250,7 @@ func generateKindsCue(modFS fs.FS, cfg kindGenConfig, selectors ...string) (code
 			}
 		}
 
-		goManifestFiles, err = generator.FilteredGenerate(cuekind.ManifestGoGenerator(filepath.Base(cfg.GoGenBasePath), ""), func(kind codegen.Kind) bool {
-			return kind.Properties().APIResource != nil
-		})
+		goManifestFiles, err = generatorForManifest.Generate(cuekind.ManifestGoGenerator(filepath.Base(cfg.GoGenBasePath)), selectors...)
 		if err != nil {
 			return nil, err
 		}
@@ -288,8 +260,6 @@ func generateKindsCue(modFS fs.FS, cfg kindGenConfig, selectors ...string) (code
 	}
 
 	allFiles := append(make(codejen.Files, 0), resourceFiles...)
-	allFiles = append(allFiles, modelFiles...)
-	allFiles = append(allFiles, tsModelFiles...)
 	allFiles = append(allFiles, tsResourceFiles...)
 	allFiles = append(allFiles, crdFiles...)
 	allFiles = append(allFiles, manifestFiles...)
@@ -307,15 +277,13 @@ func postGenerateFilesCue(modFS fs.FS, cfg kindGenConfig, selectors ...string) (
 	if err != nil {
 		return nil, err
 	}
-	generator, err := codegen.NewGenerator[codegen.Kind](parser, modFS)
+	generator, err := codegen.NewGenerator[codegen.Kind](parser.KindParser(true), modFS)
 	if err != nil {
 		return nil, err
 	}
 	relativePath := cfg.GoGenBasePath
 	if !cfg.GroupKinds {
 		relativePath = filepath.Join(relativePath, targetResource)
 	}
-	return generator.FilteredGenerate(cuekind.PostResourceGenerationGenerator(repo, relativePath, cfg.GroupKinds), func(kind codegen.Kind) bool {
-		return kind.Properties().APIResource != nil
-	}, selectors...)
+	return generator.Generate(cuekind.PostResourceGenerationGenerator(repo, relativePath, cfg.GroupKinds), selectors...)
 }
diff --git a/cmd/grafana-app-sdk/manifest.go b/cmd/grafana-app-sdk/manifest.go
@@ -0,0 +1,34 @@
+package main
+
+import (
+	"fmt"
+	"regexp"
+)
+
+func addKindToManifestBytesCUE(manifestBytes []byte, kindFieldName string) ([]byte, error) {
+	// Rather than attempt to load and modify in-CUE (as this is complex and will also change the CUE the user has written)
+	// We will just modify the file at <kindpath>/manifest.cue and stick kindFieldName at the beginning of the `kinds` array
+	// This is slightly brittle, but it keeps decent compatibility with the current `kind add` functionality.
+	contents := string(manifestBytes)
+	expr := regexp.MustCompile(`(?m)^(\s*kinds\s*:)(.*)$`)
+	matches := expr.FindStringSubmatch(contents)
+	if len(matches) < 3 {
+		return nil, fmt.Errorf("could not find kinds field in manifest.cue")
+	}
+	kindsStr := matches[2]
+	if regexp.MustCompile(`^\s*\[`).MatchString(kindsStr) {
+		// Direct array, we can prepend our field
+		// Check if there's anything in the array
+		if regexp.MustCompile(`^\s\[\s*]`).MatchString(kindsStr) {
+			// Empty, just replace with our field
+			contents = expr.ReplaceAllString(contents, matches[1]+" ["+kindFieldName+"]")
+		} else {
+			kindsStr = regexp.MustCompile(`^\s*\[`).ReplaceAllString(kindsStr, " ["+kindFieldName+", ")
+			contents = expr.ReplaceAllString(contents, matches[1]+kindsStr)
+		}
+	} else {
+		// Not a simple list, prepend `[<fieldname>] + `
+		contents = expr.ReplaceAllString(contents, matches[1]+" ["+kindFieldName+"] + "+matches[2])
+	}
+	return []byte(contents), nil
+}
diff --git a/cmd/grafana-app-sdk/project.go b/cmd/grafana-app-sdk/project.go
@@ -5,6 +5,7 @@ import (
 	"embed"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"os/exec"
@@ -141,6 +142,25 @@ func projectInit(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	// Init app manifest
+	mtmpl, err := template.ParseFS(templates, "templates/manifest.cue.tmpl")
+	if err != nil {
+		return err
+	}
+	mbuf := bytes.Buffer{}
+	appName := strings.Split(name, "/")[len(strings.Split(name, "/"))-1]
+	err = mtmpl.Execute(&mbuf, map[string]any{
+		"AppName": appName,
+		"Group":   appName,
+	})
+	if err != nil {
+		return err
+	}
+	err = writeFileWithOverwriteConfirm(filepath.Join(path, "kinds", "manifest.cue"), mbuf.Bytes())
+	if err != nil {
+		return err
+	}
+
 	// Initial empty project directory structure
 	err = checkAndMakePath(filepath.Join(path, "pkg"))
 	if err != nil {
@@ -241,10 +261,6 @@ func projectWriteGoModule(path, moduleName string, overwrite bool) (string, erro
 	return moduleName, nil
 }
 
-type simplePluginJSON struct {
-	ID string `json:"id"`
-}
-
 //nolint:revive,funlen
 func projectAddKind(cmd *cobra.Command, args []string) error {
 	if len(args) < 1 {
@@ -288,26 +304,14 @@ func projectAddKind(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("type must be one of 'resource' | 'model'")
 	}
 
-	pluginID, err := cmd.Flags().GetString("plugin-id")
+	file, err := os.DirFS(cuePath).Open("manifest.cue")
 	if err != nil {
 		return err
 	}
-	if pluginID == "" {
-		// Try to load the plugin ID from plugin/src/plugin.json
-		pluginJSONPath := filepath.Join(path, "plugin", "src", "plugin.json")
-		if _, err := os.Stat(pluginJSONPath); err != nil {
-			return fmt.Errorf("--plugin-id is required if plugin/src/plugin.json is not present")
-		}
-		contents, err := os.ReadFile(pluginJSONPath)
-		if err != nil {
-			return fmt.Errorf("could not read plugin/src/plugin.json: %w", err)
-		}
-		spj := simplePluginJSON{}
-		err = json.Unmarshal(contents, &spj)
-		if err != nil {
-			return fmt.Errorf("could not parse plugin.json: %w", err)
-		}
-		pluginID = spj.ID
+	defer file.Close()
+	manifestBytes, err := io.ReadAll(file)
+	if err != nil {
+		return err
 	}
 
 	for _, kindName := range args {
@@ -321,6 +325,13 @@ func projectAddKind(cmd *cobra.Command, args []string) error {
 			pkg = filepath.Base(cuePath)
 		}
 
+		fieldName := strings.ToLower(kindName[0:1]) + kindName[1:]
+
+		manifestBytes, err = addKindToManifestBytesCUE(manifestBytes, fieldName)
+		if err != nil {
+			return err
+		}
+
 		var templatePath string
 		switch format {
 		case FormatCUE:
@@ -336,11 +347,10 @@ func projectAddKind(cmd *cobra.Command, args []string) error {
 
 		buf := &bytes.Buffer{}
 		err = kindTmpl.Execute(buf, map[string]string{
-			"FieldName": strings.ToLower(kindName[0:1]) + kindName[1:],
+			"FieldName": fieldName,
 			"Name":      kindName,
 			"Target":    target,
 			"Package":   pkg,
-			"PluginID":  pluginID,
 		})
 		if err != nil {
 			return err
@@ -355,8 +365,7 @@ func projectAddKind(cmd *cobra.Command, args []string) error {
 			return err
 		}
 	}
-
-	return nil
+	return writeFile(filepath.Join(path, cuePath, "manifest.cue"), manifestBytes)
 }
 
 //nolint:revive,funlen,gocyclo
@@ -428,7 +437,7 @@ func projectAddComponent(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return err
 		}
-		generator, err = codegen.NewGenerator[codegen.Kind](parser, os.DirFS(cuePath))
+		generator, err = codegen.NewGenerator[codegen.Kind](parser.KindParser(true), os.DirFS(cuePath))
 		if err != nil {
 			return err
 		}

diff --git a/cmd/grafana-app-sdk/project_local.go b/cmd/grafana-app-sdk/project_local.go
@@ -201,7 +201,7 @@ func projectLocalEnvGenerate(cmd *cobra.Command, _ []string) error {
 			if err != nil {
 				return nil, err
 			}
-			generator, err := codegen.NewGenerator[codegen.Kind](parser, os.DirFS(cuePath))
+			generator, err := codegen.NewGenerator[codegen.Kind](parser.KindParser(true), os.DirFS(cuePath))
 			if err != nil {
 				return nil, err
 			}
@@ -764,13 +764,11 @@ func updateLocalConfigFromManifest(config *localEnvConfig, format string, cuePat
 		if err != nil {
 			return err
 		}
-		generator, err := codegen.NewGenerator[codegen.Kind](parser, os.DirFS(cuePath))
+		generator, err := codegen.NewGenerator[codegen.AppManifest](parser.ManifestParser(), os.DirFS(cuePath))
 		if err != nil {
 			return err
 		}
-		fs, err := generator.FilteredGenerate(cuekind.ManifestGenerator(json.Marshal, "json", "myapp"), func(kind codegen.Kind) bool {
-			return kind.Properties().APIResource != nil
-		})
+		fs, err := generator.Generate(cuekind.ManifestGenerator(json.Marshal, "json"))
 		if err != nil {
 			return err
 		}