Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support generation of model from openapi schemas #582

Merged
merged 13 commits into from
Sep 30, 2024
15 changes: 15 additions & 0 deletions encoding/convert.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package encoding

import (
"gopkg.in/yaml.v3"
)

func ToYaml(data []byte) ([]byte, error) {
var out map[string]interface{}
err := Unmarshal(data, &out)
if err != nil {
return nil, err
}

return yaml.Marshal(out)
}
45 changes: 24 additions & 21 deletions generators/github/package.go
Original file line number Diff line number Diff line change
@@ -2,14 +2,12 @@ package github

import (
"bytes"
"fmt"
"os"

"github.com/layer5io/meshkit/utils"
"github.com/layer5io/meshkit/utils/component"
"github.com/layer5io/meshkit/utils/manifests"
"github.com/meshery/schemas/models/v1beta1/category"
_component "github.com/meshery/schemas/models/v1beta1/component"
"github.com/meshery/schemas/models/v1beta1/model"
)

type GitHubPackage struct {
@@ -35,28 +33,33 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition,

manifestBytes := bytes.Split(data, []byte("\n---\n"))
crds, errs := component.FilterCRDs(manifestBytes)

for _, crd := range crds {
comp, err := component.Generate(crd)
comps, err := component.GenerateFromOpenAPI(crd)
if err != nil {
continue
}
if comp.Model.Metadata == nil {
comp.Model.Metadata = &model.ModelDefinition_Metadata{}
}
if comp.Model.Metadata.AdditionalProperties == nil {
comp.Model.Metadata.AdditionalProperties = make(map[string]interface{})
fmt.Println("ERR line 42 : ", err)
}

comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL
comp.Model.Version = gp.version
comp.Model.Name = gp.Name
comp.Model.Category = category.CategoryDefinition{
Name: "",
}
comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name)
components = append(components, comp)
components = append(components, comps...)
}
// comp, err := component.Generate(crd)
// if err != nil {
// continue
// }
// if comp.Model.Metadata == nil {
// comp.Model.Metadata = &model.ModelDefinition_Metadata{}
// }
// if comp.Model.Metadata.AdditionalProperties == nil {
// comp.Model.Metadata.AdditionalProperties = make(map[string]interface{})
// }

// comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL
// comp.Model.Version = gp.version
// comp.Model.Name = gp.Name
// comp.Model.Category = category.CategoryDefinition{
// Name: "",
// }
// comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name)
// components = append(components, comp)
// }

return components, utils.CombineErrors(errs, "\n")
}
6 changes: 5 additions & 1 deletion generators/github/url.go
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ package github

import (
"bufio"
"fmt"
"io"

"net/url"
@@ -74,10 +75,13 @@ func ProcessContent(w io.Writer, downloadDirPath, downloadfilePath string) error

if err != nil {
return err
}
}

fmt.Println("TEST 80 inside github url.go")

err = utils.ProcessContent(downloadDirPath, func(path string) error {
err = helm.ConvertToK8sManifest(path, "", w)
fmt.Println("TEST 84 inside github url.go")
if err != nil {
return err
}
45 changes: 38 additions & 7 deletions utils/component/generator.go
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ import (

"cuelang.org/go/cue"
"github.com/layer5io/meshkit/utils"
"github.com/layer5io/meshkit/utils/kubernetes"
"github.com/layer5io/meshkit/utils/manifests"
"github.com/meshery/schemas/models/v1beta1"
"github.com/meshery/schemas/models/v1beta1/component"
@@ -45,42 +46,64 @@ var DefaultPathConfig2 = CuePathConfig{
SpecPath: "spec.validation.openAPIV3Schema",
}

var OpenAPISpecPathConfig = CuePathConfig{
NamePath: `x-kubernetes-group-version-kind"[0].kind`,
IdentifierPath: "spec.names.kind",
VersionPath: `"x-kubernetes-group-version-kind"[0].version`,
GroupPath: `"x-kubernetes-group-version-kind"[0].group`,
ScopePath: "spec.scope",
SpecPath: "spec.versions[0].schema.openAPIV3Schema",
PropertiesPath: "properties",
}

var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2}

func Generate(crd string) (component.ComponentDefinition, error) {
func Generate(resource string) (component.ComponentDefinition, error) {
cmp := component.ComponentDefinition{}
cmp.SchemaVersion = v1beta1.ComponentSchemaVersion

cmp.Metadata = component.ComponentDefinition_Metadata{}
crdCue, err := utils.YamlToCue(crd)
isCRD := kubernetes.IsCRD(resource)

var specPath string
if isCRD {
specPath = DefaultPathConfig.SpecPath
} else {
specPath = "components.schemas"
}

fmt.Println("SPEC PATH ", specPath)
cueValue, err := cueValueFromResource(resource, isCRD)
if err != nil {
return cmp, err
}

var schema string
for _, cfg := range Configs {
schema, err = getSchema(crdCue, cfg)
cfg.SpecPath = specPath
schema, err = getSchema(cueValue, cfg)
if err == nil {
break
}
}
cmp.Component.Schema = schema
name, err := extractCueValueFromPath(crdCue, DefaultPathConfig.NamePath)
name, err := extractCueValueFromPath(cueValue, DefaultPathConfig.NamePath)
if err != nil {
return cmp, err
}
version, err := extractCueValueFromPath(crdCue, DefaultPathConfig.VersionPath)
version, err := extractCueValueFromPath(cueValue, DefaultPathConfig.VersionPath)
if err != nil {
return cmp, err
}
group, err := extractCueValueFromPath(crdCue, DefaultPathConfig.GroupPath)
group, err := extractCueValueFromPath(cueValue, DefaultPathConfig.GroupPath)
if err != nil {
return cmp, err
}
// return component, err Ignore error if scope isn't found
if cmp.Metadata.AdditionalProperties == nil {
cmp.Metadata.AdditionalProperties = make(map[string]interface{})
}
scope, _ := extractCueValueFromPath(crdCue, DefaultPathConfig.ScopePath)
scope, _ := extractCueValueFromPath(cueValue, DefaultPathConfig.ScopePath)
if scope == "Cluster" {
cmp.Metadata.AdditionalProperties["isNamespaced"] = false
} else if scope == "Namespaced" {
@@ -98,6 +121,14 @@ func Generate(crd string) (component.ComponentDefinition, error) {
return cmp, nil
}

func cueValueFromResource(resource string, isCRD bool) (cue.Value, error) {
if isCRD {
return utils.YamlToCue(resource)
} else {
return utils.JsonToCue([]byte(resource))
}
}

/*
Find and modify specific schema properties.
1. Identify interesting properties by walking entire schema.
123 changes: 123 additions & 0 deletions utils/component/openapi_generator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package component

import (
"encoding/json"
"fmt"
"strings"

"cuelang.org/go/cue"
"cuelang.org/go/cue/cuecontext"
"cuelang.org/go/encoding/yaml"
"github.com/layer5io/meshkit/utils/manifests"
"github.com/meshery/schemas/models/v1beta1"
"github.com/meshery/schemas/models/v1beta1/category"
"github.com/meshery/schemas/models/v1beta1/component"
"github.com/meshery/schemas/models/v1beta1/model"
)

func GenerateFromOpenAPI(resource string) ([]component.ComponentDefinition, error) {
Copy link
Author

@MUzairS15 MUzairS15 Sep 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pass the "pkg" that contains info regarding model to generate.

cuectx := cuecontext.New()
fmt.Println("resource -----------: ", resource)
cueParsedManExpr, err := yaml.Extract("", []byte(resource))
parsedManifest := cuectx.BuildFile(cueParsedManExpr)

definitions := parsedManifest.LookupPath(cue.ParsePath("components.schemas"))
if err != nil {
return nil, nil
}

resol := manifests.ResolveOpenApiRefs{}
cache := make(map[string][]byte)
resolved, err := resol.ResolveReferences([]byte(resource), definitions, cache)
if err != nil {
return nil, err
}
fmt.Println("resource -----------: ", string(resolved))
cueParsedManExpr, err = yaml.Extract("", []byte(resolved))
parsedManifest = cuectx.BuildFile(cueParsedManExpr)
definitions = parsedManifest.LookupPath(cue.ParsePath("components.schemas"))
if err != nil {
return nil, nil
}

fields, err := definitions.Fields()
if err != nil {
fmt.Printf("%v\n", err)
return nil, err
}
components := make([]component.ComponentDefinition, 0)

for fields.Next() {
fieldVal := fields.Value()
kindCue := fieldVal.LookupPath(cue.ParsePath(`"x-kubernetes-group-version-kind"[0].kind`))
if kindCue.Err() != nil {
continue
}
kind, err := kindCue.String()
kind = strings.ToLower(kind)
if err != nil {
fmt.Printf("%v", err)
continue
}

crd, err := fieldVal.MarshalJSON()
if err != nil {
fmt.Printf("%v", err)
continue
}
versionCue := fieldVal.LookupPath(cue.ParsePath(`"x-kubernetes-group-version-kind"[0].version`))
groupCue := fieldVal.LookupPath(cue.ParsePath(`"x-kubernetes-group-version-kind"[0].group`))
apiVersion, _ := versionCue.String()
if g, _ := groupCue.String(); g != "" {
apiVersion = g + "/" + apiVersion
}
modified := make(map[string]interface{}) //Remove the given fields which is either not required by End user (like status) or is prefilled by system (like apiVersion, kind and metadata)
err = json.Unmarshal(crd, &modified)
if err != nil {
fmt.Printf("%v", err)
continue
}

modifiedProps, err := UpdateProperties(fieldVal, cue.ParsePath("properties.spec"), apiVersion)
if err == nil {
modified = modifiedProps
}

DeleteFields(modified)
crd, err = json.Marshal(modified)
if err != nil {
fmt.Printf("%v", err)
continue
}

c := component.ComponentDefinition{
SchemaVersion: v1beta1.ComponentSchemaVersion,
Version: "v1.0.0",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hardcoding appropriate?


Format: component.JSON,
Component: component.Component{
Kind: kind,
Version: apiVersion,
Schema: string(crd),
},
// Metadata: compMetadata,
DisplayName: manifests.FormatToReadableString(kind),
Model: model.ModelDefinition{
SchemaVersion: v1beta1.ModelSchemaVersion,
Version: "v1.0.0",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this still hardcoded?


Model: model.Model{
Version: "version",
},
Name: "kubernetes",
DisplayName: "Kubernetes",
Category: category.CategoryDefinition{
Name: "Orchestration & Management",
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Update

},
},
}
components = append(components, c)
}
return components, nil

}
18 changes: 5 additions & 13 deletions utils/component/utils.go
Original file line number Diff line number Diff line change
@@ -2,12 +2,11 @@ package component

import (
"encoding/json"
"fmt"

"cuelang.org/go/cue"
"github.com/layer5io/meshkit/utils"
"github.com/layer5io/meshkit/utils/kubernetes"
"github.com/layer5io/meshkit/utils/manifests"
"gopkg.in/yaml.v2"
)

// Remove the fields which is either not required by end user (like status) or is prefilled by system (like apiVersion, kind and metadata)
@@ -81,18 +80,11 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) {
var errs []error
var filteredManifests []string
for _, m := range manifests {
fmt.Println("TEST 84 : utils.go: ")
// isCrd := kubernetes.IsCRD(string(m))
// if !isCrd {

var crd map[string]interface{}
err := yaml.Unmarshal(m, &crd)
if err != nil {
errs = append(errs, err)
continue
}

isCrd := kubernetes.IsCRD(crd)
if !isCrd {
continue
}
// }
filteredManifests = append(filteredManifests, string(m))
}
return filteredManifests, errs
Loading
Loading