Initial App Manifest (#395)

# What This PR Does / Why We Need It

This PR introduces the first step of the **App Manifest**, a concept to
help clarify and drive app development. The App Manifest is the listing
of the app's managed kinds and capabilities (admission, conversion) for
those kinds. This PR introduces admission and conversion capabilities
for kinds in the manifest, be leaves schemas unimplemented.

Added in this PR is the `app` package, which is where the App Manifest
(as `app.Manifest`) lives, and where additional app-centric logic will
reside for future features such as
#385

`app.Manifest` is decoupled from the actual manifest data by having an
`app.ManifestData` type which contains manifest data, and having the
`app.Manifest` contain a pointer to said data (which can be nil), and a
location for the data. This way, an `app.Manifest` can simply point to a
file on-disk, or an API server location, without having to have the data
loaded, and the consumer of the `app.Manifest` should understand how to
fetch the `app.ManifestData`. This allows an app to not need to know the
credentials to fetch the manifest data (such as kube config), but still
be able to tell other components which may have such knowledge where to
fetch the data (this is important in
#385, as the App
Provider will need to provide the manifest and a way of creating the
App, but will not know how to talk to the API server, as the runner will
be able to load those credentials).

Included in this PR is the generation of this in-progress initial
manifest as both an API server CR, and as in-code `app.ManifestData` to
use if and when the manifest type is not available in the API server. An
app author can now specify their app's capabilities for admission and
conversion for kinds via `admission`, `mutations`, and `conversion` in
the `apiResource` field in CUE, and can optionally override those
kind-wide defaults on a per-version basis with `admission` and
`mutation` fields in the version (`conversion` is always only kind-wide,
as all versions must be inter-convertable for `conversion` to be
allowed). The test data has been updated with these additional fields
for manifest generation testing. Two new files will now be generated on
each `grafana-app-sdk generate` call:
`definitions/<app-name>-manifest.(json|yaml)`, and
`pkg/generated/manifest.go`.

Relates to #353, which
will be completed once the manifest contains the CRDs (schemas) as well.

---------

Co-authored-by: Igor Suleymanov <[email protected]>

app/manifest.go

@@ -0,0 +1,141 @@
+package app
+
+// NewEmbeddedManifest returns a Manifest which has the ManifestData embedded in it
+func NewEmbeddedManifest(manifestData ManifestData) Manifest {
+	return Manifest{
+		Location: ManifestLocation{
+			Type: ManifestLocationEmbedded,
+		},
+		ManifestData: &manifestData,
+	}
+}
+
+// NewOnDiskManifest returns a Manifest which points to a path on-disk to load ManifestData from
+func NewOnDiskManifest(path string) Manifest {
+	return Manifest{
+		Location: ManifestLocation{
+			Type: ManifestLocationFilePath,
+			Path: path,
+		},
+	}
+}
+
+// NewAPIServerManifest returns a Manifest which points to a resource in an API server to load the ManifestData from
+func NewAPIServerManifest(resourceName string) Manifest {
+	return Manifest{
+		Location: ManifestLocation{
+			Type: ManifestLocationAPIServerResource,
+			Path: resourceName,
+		},
+	}
+}
+
+// Manifest is a type which represents the Location and Data in an App Manifest.
+type Manifest struct {
+	// ManifestData must be present if Location.Type == "embedded"
+	ManifestData *ManifestData
+	// Location indicates the place where the ManifestData should be loaded from
+	Location ManifestLocation
+}
+
+// ManifestLocation contains information of where a Manifest's ManifestData can be found.
+type ManifestLocation struct {
+	Type ManifestLocationType
+	// Path is the path to the manifest, based on location.
+	// For "filepath", it is the path on disk. For "apiserver", it is the NamespacedName. For "embedded", it is empty.
+	Path string
+}
+
+type ManifestLocationType string
+
+const (
+	ManifestLocationFilePath          = ManifestLocationType("filepath")
+	ManifestLocationAPIServerResource = ManifestLocationType("apiserver")
+	ManifestLocationEmbedded          = ManifestLocationType("embedded")
+)
+
+// ManifestData is the data in a Manifest, representing the Kinds and Capabilities of an App.
+// NOTE: ManifestData is still experimental and subject to change
+type ManifestData struct {
+	// AppName is the unique identifier for the App
+	AppName string `json:"appName" yaml:"appName"`
+	// Group is the group used for all kinds maintained by this app.
+	// This is usually "<AppName>.ext.grafana.com"
+	Group string `json:"group" yaml:"group"`
+	// Kinds is a list of all Kinds maintained by this App
+	Kinds []ManifestKind `json:"kinds,omitempty" yaml:"kinds,omitempty"`
+}
+
+// ManifestKind is the manifest for a particular kind, including its Kind, Scope, and Versions
+type ManifestKind struct {
+	// Kind is the name of the kind
+	Kind string `json:"kind" yaml:"kind"`
+	// Scope if the scope of the kind, typically restricted to "Namespaced" or "Cluster"
+	Scope string `json:"scope" yaml:"scope"`
+	// Versions is the set of versions for the kind. This list should be ordered as a series of progressively later versions.
+	Versions []ManifestKindVersion `json:"versions" yaml:"versions"`
+	// Conversion is true if the app has a conversion capability for this kind
+	Conversion bool `json:"conversion" yaml:"conversion"`
+}
+
+// ManifestKindVersion contains details for a version of a kind in a Manifest
+type ManifestKindVersion struct {
+	// Name is the version string name, such as "v1"
+	Name string `yaml:"name" json:"name"`
+	// Admission is the collection of admission capabilities for this version.
+	// If nil, no admission capabilities exist for the version.
+	Admission *AdmissionCapabilities `json:"admission,omitempty" yaml:"admission,omitempty"`
+	// Schema is the schema of this version, as an OpenAPI document.
+	// This is currently an `any` type as implementation is incomplete.
+	Schema any `json:"schema,omitempty" yaml:"schema,omitempty"` // TODO: actual schema
+}
+
+// AdmissionCapabilities is the collection of admission capabilities of a kind
+type AdmissionCapabilities struct {
+	// Validation contains the validation capability details. If nil, the kind does not have a validation capability.
+	Validation *ValidationCapability `json:"validation,omitempty" yaml:"validation,omitempty"`
+	// Mutation contains the mutation capability details. If nil, the kind does not have a mutation capability.
+	Mutation *MutationCapability `json:"mutation,omitempty" yaml:"mutation,omitempty"`
+}
+
+// SupportsAnyValidation returns true if the list of operations for validation is not empty.
+// This is a convenience method to avoid having to make several nil and length checks.
+func (c AdmissionCapabilities) SupportsAnyValidation() bool {
+	if c.Validation == nil {
+		return false
+	}
+	return len(c.Validation.Operations) > 0
+}
+
+// SupportsAnyMutation returns true if the list of operations for mutation is not empty.
+// This is a convenience method to avoid having to make several nil and length checks.
+func (c AdmissionCapabilities) SupportsAnyMutation() bool {
+	if c.Mutation == nil {
+		return false
+	}
+	return len(c.Mutation.Operations) > 0
+}
+
+// ValidationCapability is the details of a validation capability for a kind's admission control
+type ValidationCapability struct {
+	// Operations is the list of operations that the validation capability is used for.
+	// If this list if empty or nil, this is equivalent to the app having no validation capability.
+	Operations []AdmissionOperation `json:"operations,omitempty" yaml:"operations,omitempty"`
+}
+
+// MutationCapability is the details of a mutation capability for a kind's admission control
+type MutationCapability struct {
+	// Operations is the list of operations that the mutation capability is used for.
+	// If this list if empty or nil, this is equivalent to the app having no mutation capability.
+	Operations []AdmissionOperation `json:"operations,omitempty" yaml:"operations,omitempty"`
+}
+
+type AdmissionOperation string
+
+const (
+	AdmissionOperationAny     AdmissionOperation = "*"
+	AdmissionOperationCreate  AdmissionOperation = "CREATE"
+	AdmissionOperationUpdate  AdmissionOperation = "UPDATE"
+	AdmissionOperationDelete  AdmissionOperation = "DELETE"
+	AdmissionOperationConnect AdmissionOperation = "CONNECT"
+)

cmd/grafana-app-sdk/generate.go

@@ -177,6 +177,7 @@ type kindGenConfig struct {
 	GroupKinds    bool
 }
 
+//nolint:goconst
 func generateKindsThema(modFS fs.FS, cfg kindGenConfig, selectors ...string) (codejen.Files, error) {
 	parser, err := themagen.NewCustomKindParser(thema.NewRuntime(cuecontext.New()), modFS)
 	if err != nil {
@@ -269,6 +270,7 @@ func generateFrontendModelsThema(parser *themagen.CustomKindParser, genPath stri
 	return files, nil
 }
 
+//nolint:goconst
 func generateCRDsThema(parser *themagen.CustomKindParser, genPath string, encoding string, selectors []string) (codejen.Files, error) {
 	var ms themagen.Generator
 	if encoding == "yaml" {
@@ -289,7 +291,7 @@ func generateCRDsThema(parser *themagen.CustomKindParser, genPath string, encodi
 	return files, nil
 }
 
-//nolint:funlen
+//nolint:funlen,goconst
 func generateKindsCue(modFS fs.FS, cfg kindGenConfig, selectors ...string) (codejen.Files, error) {
 	parser, err := cuekind.NewParser()
 	if err != nil {
@@ -360,11 +362,43 @@ func generateKindsCue(modFS fs.FS, cfg kindGenConfig, selectors ...string) (code
 		}
 	}
 
+	// Manifest
+	var manifestFiles codejen.Files
+	if cfg.CRDEncoding != "none" {
+		encFunc := func(v any) ([]byte, error) {
+			return json.MarshalIndent(v, "", "    ")
+		}
+		if cfg.CRDEncoding == "yaml" {
+			encFunc = yaml.Marshal
+		}
+		manifestFiles, err = generator.FilteredGenerate(cuekind.ManifestGenerator(encFunc, cfg.CRDEncoding, ""), func(kind codegen.Kind) bool {
+			return kind.Properties().APIResource != nil
+		}, selectors...)
+		if err != nil {
+			return nil, err
+		}
+		for i, f := range manifestFiles {
+			manifestFiles[i].RelativePath = filepath.Join(cfg.CRDPath, f.RelativePath)
+		}
+	}
+
+	goManifestFiles, err := generator.FilteredGenerate(cuekind.ManifestGoGenerator(filepath.Base(cfg.GoGenBasePath), ""), func(kind codegen.Kind) bool {
+		return kind.Properties().APIResource != nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	for i, f := range goManifestFiles {
+		goManifestFiles[i].RelativePath = filepath.Join(cfg.GoGenBasePath, f.RelativePath)
+	}
+
 	allFiles := append(make(codejen.Files, 0), resourceFiles...)
 	allFiles = append(allFiles, modelFiles...)
 	allFiles = append(allFiles, tsModelFiles...)
 	allFiles = append(allFiles, tsResourceFiles...)
 	allFiles = append(allFiles, crdFiles...)
+	allFiles = append(allFiles, manifestFiles...)
+	allFiles = append(allFiles, goManifestFiles...)
 	return allFiles, nil
 }
 

codegen/cuekind/def.cue

@@ -70,6 +70,10 @@ Schema: {
 	_specIsNonEmpty: spec & struct.MinFields(0)
 }
 
+#AdmissionCapability: {
+	operations: [...string]
+}
+
 // Kind represents an arbitrary kind which can be used for code generation
 Kind: S={
 	kind: =~"^([A-Z][a-zA-Z0-9-]{0,61}[a-zA-Z0-9])$"
@@ -107,6 +111,16 @@ Kind: S={
 		// scope determines whether resources of this kind exist globally ("Cluster") or
 		// within Kubernetes namespaces.
 		scope: "Cluster" | *"Namespaced"
+		// validation determines whether there is code-based validation for this kind. Used for generating the manifest.
+		validation: #AdmissionCapability | *{
+			operations: []
+		}
+		// mutation determines whether there is code-based mutation for this kind. Used for generating the manifest.
+		mutation: #AdmissionCapability | *{
+			operations: []
+		}
+		// conversion determines whether there is code-based conversion for this kind. Used for generating the manifest.
+		conversion: bool | *false
 	}
 	// isCRD is true if the `crd` trait is present in the kind.
 	isAPIResource: apiResource != _|_
@@ -128,6 +142,8 @@ Kind: S={
 			// Fields must be from the root of the schema, i.e. 'spec.foo', and have a string type.
 			// Fields cannot include custom metadata (TODO: check if we can use annotations for field selectors)
 			selectableFields: [...string]
+			validation: #AdmissionCapability | *S.apiResource.validation
+			mutation: #AdmissionCapability | *S.apiResource.mutation
 		}
 	}
 	machineName: strings.ToLower(strings.Replace(S.kind, "-", "_", -1))

codegen/cuekind/generators.go

@@ -141,6 +141,25 @@ func PostResourceGenerationGenerator(projectRepo, goGenPath string, versioned bo
 	return g
 }
 
+func ManifestGenerator(encoder jennies.ManifestOutputEncoder, extension string, appName string) *codejen.JennyList[codegen.Kind] {
+	g := codejen.JennyListWithNamer[codegen.Kind](namerFunc)
+	g.Append(&jennies.ManifestGenerator{
+		AppName:       appName,
+		Encoder:       encoder,
+		FileExtension: extension,
+	})
+	return g
+}
+
+func ManifestGoGenerator(pkg string, appName string) *codejen.JennyList[codegen.Kind] {
+	g := codejen.JennyListWithNamer[codegen.Kind](namerFunc)
+	g.Append(&jennies.ManifestGoGenerator{
+		Package: pkg,
+		AppName: appName,
+	})
+	return g
+}
+
 func namerFunc(k codegen.Kind) string {
 	if k == nil {
 		return "nil"

codegen/cuekind/generators_test.go

@@ -170,6 +170,53 @@ func TestTypeScriptResourceGenerator(t *testing.T) {
 	})
 }
 
+func TestManifestGenerator(t *testing.T) {
+	parser, err := NewParser()
+	require.Nil(t, err)
+
+	t.Run("resource", func(t *testing.T) {
+		kinds, err := parser.Parse(os.DirFS(TestCUEDirectory), "testKind", "testKind2")
+		require.Nil(t, err)
+		files, err := ManifestGenerator(yaml.Marshal, "yaml", "test-app-test-kind").Generate(kinds...)
+		require.Nil(t, err)
+		// Check number of files generated
+		// 5 -> object, spec, metadata, status, schema
+		assert.Len(t, files, 1)
+		// Check content against the golden files
+		compareToGolden(t, files, "manifest")
+	})
+
+	t.Run("model", func(t *testing.T) {
+		kinds, err := parser.Parse(os.DirFS(TestCUEDirectory), "customKind2")
+		fmt.Println(err)
+		require.Nil(t, err)
+		files, err := ManifestGenerator(yaml.Marshal, "yaml", "test-app-custom-kind-2").Generate(kinds...)
+		require.Nil(t, err)
+		// Check number of files generated
+		// 5 -> object, spec, metadata, status, schema
+		assert.Len(t, files, 1)
+		// Check content against the golden files
+		compareToGolden(t, files, "manifest")
+	})
+}
+
+func TestManifestGoGenerator(t *testing.T) {
+	parser, err := NewParser()
+	require.Nil(t, err)
+
+	t.Run("resource", func(t *testing.T) {
+		kinds, err := parser.Parse(os.DirFS(TestCUEDirectory), "testKind", "testKind2")
+		require.Nil(t, err)
+		files, err := ManifestGoGenerator("generated", "test-app").Generate(kinds...)
+		require.Nil(t, err)
+		// Check number of files generated
+		// 5 -> object, spec, metadata, status, schema
+		assert.Len(t, files, 1)
+		// Check content against the golden files
+		compareToGolden(t, files, "manifest/go/testkinds")
+	})
+}
+
 func compareToGolden(t *testing.T, files codejen.Files, pathPrefix string) {
 	for _, f := range files {
 		// Check if there's a golden generated file to compare against

codegen/cuekind/testing/testkind.cue

@@ -6,7 +6,10 @@ testKind: {
 	kind: "TestKind"
 	plural: "testkinds"
 	group: "test"
-	apiResource: {}
+	apiResource: {
+		validation: operations: ["create","update"]
+		conversion: true
+	}
 	current: "v1"
 	codegen: frontend: false
 	versions: {
@@ -26,6 +29,7 @@ testKind: {
 					timeField: string & time.Time
 				}
 			}
+			mutation: operations: ["create","update"]
 		}
 	}
 }