From d889a68359a1f9de0462a53e06ca21c9aca62cc1 Mon Sep 17 00:00:00 2001 From: Lee Calcote Date: Mon, 30 Sep 2024 16:24:54 -0500 Subject: [PATCH] Revert "support generation of model from openapi schemas" --- encoding/convert.go | 15 --- generators/artifacthub/package.go | 8 -- generators/github/package.go | 59 +++------ generators/models/interfaces.go | 2 - utils/component/generator.go | 15 +-- utils/component/openapi_generator.go | 172 --------------------------- utils/component/utils.go | 16 ++- utils/helm/helm.go | 9 +- utils/kubernetes/crd.go | 19 +-- utils/manifests/utils.go | 15 ++- 10 files changed, 45 insertions(+), 285 deletions(-) delete mode 100644 encoding/convert.go delete mode 100644 utils/component/openapi_generator.go diff --git a/encoding/convert.go b/encoding/convert.go deleted file mode 100644 index 5dc810eb..00000000 --- a/encoding/convert.go +++ /dev/null @@ -1,15 +0,0 @@ -package encoding - -import ( - "gopkg.in/yaml.v3" -) - -func ToYaml(data []byte) ([]byte, error) { - var out map[string]interface{} - err := Unmarshal(data, &out) - if err != nil { - return nil, err - } - - return yaml.Marshal(out) -} diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index e40509d8..61b3d8be 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -38,14 +38,6 @@ func (pkg AhPackage) GetVersion() string { return pkg.Version } -func (pkg AhPackage) GetSourceURL() string { - return pkg.ChartUrl -} - -func (pkg AhPackage) GetName() string { - return pkg.Name -} - func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) // TODO: Move this to the configuration diff --git a/generators/github/package.go b/generators/github/package.go index e7efea6c..0c632ef9 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -6,7 +6,6 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" - "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" @@ -26,14 +25,6 @@ func (gp GitHubPackage) GetVersion() string { return gp.version } -func (gp GitHubPackage) GetSourceURL() string { - return gp.SourceURL -} - -func (gp GitHubPackage) GetName() string { - return gp.Name -} - func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) @@ -43,40 +34,28 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, } manifestBytes := bytes.Split(data, []byte("\n---\n")) - errs := []error{} - - for _, crd := range manifestBytes { - isCrd := kubernetes.IsCRD(string(crd)) - if !isCrd { + crds, errs := component.FilterCRDs(manifestBytes) - comps, err := component.GenerateFromOpenAPI(string(crd), gp) - if err != nil { - errs = append(errs, component.ErrGetSchema(err)) - continue - } - components = append(components, comps...) - } else { - comp, err := component.Generate(string(crd)) - if err != nil { - continue - } - if comp.Model.Metadata == nil { - comp.Model.Metadata = &model.ModelDefinition_Metadata{} - } - if comp.Model.Metadata.AdditionalProperties == nil { - comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) - } - - comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL - comp.Model.Version = gp.version - comp.Model.Name = gp.Name - comp.Model.Category = category.CategoryDefinition{ - Name: "", - } - comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) - components = append(components, comp) + for _, crd := range crds { + comp, err := component.Generate(crd) + if err != nil { + continue + } + if comp.Model.Metadata == nil { + comp.Model.Metadata = &model.ModelDefinition_Metadata{} + } + if comp.Model.Metadata.AdditionalProperties == nil { + comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) } + comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL + comp.Model.Version = gp.version + comp.Model.Name = gp.Name + comp.Model.Category = category.CategoryDefinition{ + Name: "", + } + comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) + components = append(components, comp) } return components, utils.CombineErrors(errs, "\n") diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index c6dcd3eb..19f61177 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -13,8 +13,6 @@ type Validator interface { type Package interface { GenerateComponents() ([]component.ComponentDefinition, error) GetVersion() string - GetSourceURL() string - GetName() string } // Supports pulling packages from Artifact Hub and other sources like Docker Hub. diff --git a/utils/component/generator.go b/utils/component/generator.go index b228b363..288f2780 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -45,28 +45,17 @@ var DefaultPathConfig2 = CuePathConfig{ SpecPath: "spec.validation.openAPIV3Schema", } -var OpenAPISpecPathConfig = CuePathConfig{ - NamePath: `x-kubernetes-group-version-kind"[0].kind`, - IdentifierPath: "spec.names.kind", - VersionPath: `"x-kubernetes-group-version-kind"[0].version`, - GroupPath: `"x-kubernetes-group-version-kind"[0].group`, - ScopePath: "spec.scope", - SpecPath: "spec.versions[0].schema.openAPIV3Schema", - PropertiesPath: "properties", -} - var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} -func Generate(resource string) (component.ComponentDefinition, error) { +func Generate(crd string) (component.ComponentDefinition, error) { cmp := component.ComponentDefinition{} cmp.SchemaVersion = v1beta1.ComponentSchemaVersion cmp.Metadata = component.ComponentDefinition_Metadata{} - crdCue, err := utils.YamlToCue(resource) + crdCue, err := utils.YamlToCue(crd) if err != nil { return cmp, err } - var schema string for _, cfg := range Configs { schema, err = getSchema(crdCue, cfg) diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go deleted file mode 100644 index f1acf67d..00000000 --- a/utils/component/openapi_generator.go +++ /dev/null @@ -1,172 +0,0 @@ -package component - -import ( - "encoding/json" - "fmt" - "strings" - - "cuelang.org/go/cue" - "cuelang.org/go/cue/cuecontext" - cueJson "cuelang.org/go/encoding/json" - "github.com/layer5io/meshkit/generators/models" - "github.com/layer5io/meshkit/utils" - "github.com/layer5io/meshkit/utils/manifests" - - "gopkg.in/yaml.v3" - - "github.com/meshery/schemas/models/v1beta1" - "github.com/meshery/schemas/models/v1beta1/category" - "github.com/meshery/schemas/models/v1beta1/component" - "github.com/meshery/schemas/models/v1beta1/model" -) - -func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.ComponentDefinition, error) { - if resource == "" { - return nil, nil - } - resource, err := getResolvedManifest(resource) - if err != nil { - return nil, err - } - cuectx := cuecontext.New() - cueParsedManExpr, err := cueJson.Extract("", []byte(resource)) - if err != nil { - return nil, err - } - - parsedManifest := cuectx.BuildExpr(cueParsedManExpr) - definitions, err := utils.Lookup(parsedManifest, "components.schemas") - - if err != nil { - return nil, err - } - - fields, err := definitions.Fields() - if err != nil { - fmt.Printf("%v\n", err) - return nil, err - } - components := make([]component.ComponentDefinition, 0) - - for fields.Next() { - fieldVal := fields.Value() - kindCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].kind`) - if err != nil { - continue - } - kind, err := kindCue.String() - kind = strings.ToLower(kind) - if err != nil { - fmt.Printf("%v", err) - continue - } - - crd, err := fieldVal.MarshalJSON() - if err != nil { - fmt.Printf("%v", err) - continue - } - versionCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].version`) - if err != nil { - continue - } - - groupCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].group`) - if err != nil { - continue - } - - apiVersion, _ := versionCue.String() - if g, _ := groupCue.String(); g != "" { - apiVersion = g + "/" + apiVersion - } - modified := make(map[string]interface{}) //Remove the given fields which is either not required by End user (like status) or is prefilled by system (like apiVersion, kind and metadata) - err = json.Unmarshal(crd, &modified) - if err != nil { - fmt.Printf("%v", err) - continue - } - - modifiedProps, err := UpdateProperties(fieldVal, cue.ParsePath("properties.spec"), apiVersion) - if err == nil { - modified = modifiedProps - } - - DeleteFields(modified) - crd, err = json.Marshal(modified) - if err != nil { - fmt.Printf("%v", err) - continue - } - - c := component.ComponentDefinition{ - SchemaVersion: v1beta1.ComponentSchemaVersion, - Version: "v1.0.0", - - Format: component.JSON, - Component: component.Component{ - Kind: kind, - Version: apiVersion, - Schema: string(crd), - }, - // Metadata: compMetadata, - DisplayName: manifests.FormatToReadableString(kind), - Model: model.ModelDefinition{ - SchemaVersion: v1beta1.ModelSchemaVersion, - Version: "v1.0.0", - - Model: model.Model{ - Version: pkg.GetVersion(), - }, - Name: pkg.GetName(), - DisplayName: manifests.FormatToReadableString(pkg.GetName()), - Category: category.CategoryDefinition{ - Name: "Orchestration & Management", - }, - Metadata: &model.ModelDefinition_Metadata{ - AdditionalProperties: map[string]interface{}{ - "source_uri": pkg.GetSourceURL(), - }, - }, - }, - } - - components = append(components, c) - } - return components, nil - -} - -func getResolvedManifest(manifest string) (string, error) { - var m map[string]interface{} - - err := yaml.Unmarshal([]byte(manifest), &m) - if err != nil { - return "", utils.ErrDecodeYaml(err) - } - - byt, err := json.Marshal(m) - if err != nil { - return "", utils.ErrMarshal(err) - } - - cuectx := cuecontext.New() - cueParsedManExpr, err := cueJson.Extract("", byt) - if err != nil { - return "", ErrGetSchema(err) - } - - parsedManifest := cuectx.BuildExpr(cueParsedManExpr) - definitions, err := utils.Lookup(parsedManifest, "components.schemas") - if err != nil { - return "", err - } - resol := manifests.ResolveOpenApiRefs{} - cache := make(map[string][]byte) - resolved, err := resol.ResolveReferences(byt, definitions, cache) - if err != nil { - return "", err - } - manifest = string(resolved) - return manifest, nil -} diff --git a/utils/component/utils.go b/utils/component/utils.go index f04ded45..f63d6068 100644 --- a/utils/component/utils.go +++ b/utils/component/utils.go @@ -7,6 +7,7 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" + "gopkg.in/yaml.v2" ) // Remove the fields which is either not required by end user (like status) or is prefilled by system (like apiVersion, kind and metadata) @@ -80,10 +81,19 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) { var errs []error var filteredManifests []string for _, m := range manifests { - isCrd := kubernetes.IsCRD(string(m)) - if isCrd { - filteredManifests = append(filteredManifests, string(m)) + + var crd map[string]interface{} + err := yaml.Unmarshal(m, &crd) + if err != nil { + errs = append(errs, err) + continue + } + + isCrd := kubernetes.IsCRD(crd) + if !isCrd { + continue } + filteredManifests = append(filteredManifests, string(m)) } return filteredManifests, errs } diff --git a/utils/helm/helm.go b/utils/helm/helm.go index 091e5902..4af65eed 100644 --- a/utils/helm/helm.go +++ b/utils/helm/helm.go @@ -9,7 +9,6 @@ import ( "regexp" "strings" - "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/utils" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -109,13 +108,7 @@ func writeToFile(w io.Writer, path string) error { if err != nil { return utils.ErrReadFile(err, path) } - - byt, err := encoding.ToYaml(data) - if err != nil { - return utils.ErrWriteFile(err, path) - } - - _, err = w.Write(byt) + _, err = w.Write(data) if err != nil { return utils.ErrWriteFile(err, path) } diff --git a/utils/kubernetes/crd.go b/utils/kubernetes/crd.go index 9bf8d991..0525a624 100644 --- a/utils/kubernetes/crd.go +++ b/utils/kubernetes/crd.go @@ -4,7 +4,6 @@ import ( "context" "github.com/layer5io/meshkit/encoding" - "github.com/layer5io/meshkit/utils" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ) @@ -54,19 +53,7 @@ func GetGVRForCustomResources(crd *CRDItem) *schema.GroupVersionResource { } } -func IsCRD(manifest string) bool { - cueValue, err := utils.YamlToCue(manifest) - if err != nil { - return false - } - kind, err := utils.Lookup(cueValue, "kind") - if err != nil { - return false - } - kindStr, err := kind.String() - if err != nil { - return false - } - - return kindStr == "CustomResourceDefinition" +func IsCRD(manifest map[string]interface{}) bool { + kind, ok := manifest["kind"].(string) + return ok && kind == "CustomResourceDefinition" } diff --git a/utils/manifests/utils.go b/utils/manifests/utils.go index 2935f1c0..fb3990f7 100644 --- a/utils/manifests/utils.go +++ b/utils/manifests/utils.go @@ -9,7 +9,6 @@ import ( "strings" "cuelang.org/go/cue" - "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/models/oam/core/v1alpha1" ) @@ -255,7 +254,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue cache = make(map[string][]byte) } var val map[string]interface{} - err := encoding.Unmarshal(manifest, &val) + err := json.Unmarshal(manifest, &val) if err != nil { return nil, err } @@ -267,7 +266,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if ro.isInsideJsonSchemaProps && (ref == JsonSchemaPropsRef) { // hack so that the UI doesn't crash val["$ref"] = "string" - marVal, errJson := encoding.Marshal(val) + marVal, errJson := json.Marshal(val) if errJson != nil { return manifest, nil } @@ -283,13 +282,13 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue newval = append(newval, v0) continue } - byt, _ := encoding.Marshal(v0) + byt, _ := json.Marshal(v0) byt, err = ro.ResolveReferences(byt, definitions, cache) if err != nil { return nil, err } var newvalmap map[string]interface{} - _ = encoding.Unmarshal(byt, &newvalmap) + _ = json.Unmarshal(byt, &newvalmap) newval = append(newval, newvalmap) } val[k] = newval @@ -334,7 +333,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if reflect.ValueOf(v).Kind() == reflect.Map { var marVal []byte var def []byte - marVal, err = encoding.Marshal(v) + marVal, err = json.Marshal(v) if err != nil { return nil, err } @@ -350,7 +349,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue } } } - res, err := encoding.Marshal(val) + res, err := json.Marshal(val) if err != nil { return nil, err } @@ -359,7 +358,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue func replaceRefWithVal(def []byte, val map[string]interface{}, k string) error { var defVal map[string]interface{} - err := encoding.Unmarshal([]byte(def), &defVal) + err := json.Unmarshal([]byte(def), &defVal) if err != nil { return err }