diff --git a/cyclops-ctrl/internal/template/git.go b/cyclops-ctrl/internal/template/git.go index 30abe800..4f14b0f2 100644 --- a/cyclops-ctrl/internal/template/git.go +++ b/cyclops-ctrl/internal/template/git.go @@ -18,6 +18,8 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" "helm.sh/helm/v3/pkg/chart" @@ -70,87 +72,119 @@ func (r Repo) LoadTemplate(repoURL, path, commit, resolvedVersion string) (*mode return nil, err } - // load helm chart metadata - chartMetadata, err := fs.Open(path2.Join(path, "Chart.yaml")) + // region read files + filesFs, err := fs.Chroot(path) if err != nil { - return nil, errors.Wrap(err, "could not read 'Chart.yaml' file; it should be placed in the repo/path you provided; make sure you provided the correct path") + return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists") } - var chartMetadataBuffer bytes.Buffer - _, err = io.Copy(bufio.NewWriter(&chartMetadataBuffer), chartMetadata) + files, err := readFiles("", filesFs) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to read template files") } + // endregion - var metadata *helm.Metadata - if err := yaml.Unmarshal(chartMetadataBuffer.Bytes(), &metadata); err != nil { - return nil, err + if len(files) == 0 { + return nil, errors.Errorf("no files found in repo %v on path %v; make sure the repo, path and version are correct", repoURL, path) } - // endregion - // region read templates - templatesPath := path2.Join(path, "templates") + // region map files to template + metadataBytes := []byte{} + schemaBytes := []byte{} + chartFiles := make([]*chart.File, 0) - _, err = fs.ReadDir(templatesPath) - if err != nil { - return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists") - } + templateFiles := make([]*chart.File, 0) + crdFiles := make([]*chart.File, 0) - manifests, err := concatenateTemplates(templatesPath, fs) - if err != nil { - return nil, errors.Wrap(err, "failed to load template files") - } - // endregion + dependenciesFromChartsDir := make(map[string]map[string][]byte, 0) - // region read files - filesFs, err := fs.Chroot(path) - if err != nil { - return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists") - } + for _, f := range files { + parts := strings.Split(f.Name, "/") - chartFiles, err := readFiles("", filesFs) - if err != nil { - return nil, errors.Wrap(err, "failed to read template files") - } - // endregion + if len(parts) == 1 && parts[0] == "Chart.yaml" { + metadataBytes = f.Data + continue + } - // region read schema - schemaFile, err := fs.Open(path2.Join(path, "values.schema.json")) - if err != nil { - return nil, errors.Wrap(err, "could not read 'values.schema.json' file; it should be placed in the repo/path you provided; make sure 'templates' directory exists") - } + if len(parts) == 1 && parts[0] == "values.schema.json" { + schemaBytes = f.Data + continue + } - var schemaChartBuffer bytes.Buffer - _, err = io.Copy(bufio.NewWriter(&schemaChartBuffer), schemaFile) - if err != nil { - return nil, err + if len(parts) > 1 && parts[0] == "templates" && + (parts[1] != "Notes.txt" && parts[1] != "NOTES.txt" && parts[1] != "tests") { + templateFiles = append(templateFiles, f) + continue + } + + if len(parts) > 1 && parts[0] == "crds" && + (parts[1] != "Notes.txt" && parts[1] != "NOTES.txt" && parts[1] != "tests") { + crdFiles = append(crdFiles, f) + continue + } + + if len(parts) > 2 && parts[0] == "charts" { + depName := parts[1] + if _, ok := dependenciesFromChartsDir[depName]; !ok { + dependenciesFromChartsDir[depName] = make(map[string][]byte) + } + + dependenciesFromChartsDir[depName][path2.Join(parts[1:]...)] = f.Data + continue + } + + chartFiles = append(chartFiles, f) } var schema helm.Property - if err := json.Unmarshal(schemaChartBuffer.Bytes(), &schema); err != nil { - return nil, err + // unmarshal values schema only if present + if len(schemaBytes) > 0 { + if err := json.Unmarshal(schemaBytes, &schema); err != nil { + fmt.Println("error on schema bytes", repoURL, path) + return &models.Template{}, err + } + } + + var metadata *helm.Metadata + if err := yaml.Unmarshal(metadataBytes, &metadata); err != nil { + fmt.Println("error on meta unmarshal", repoURL, path) + return &models.Template{}, err } - // endregion // region load dependencies dependencies, err := r.loadDependencies(metadata) if err != nil { - return nil, err + return &models.Template{}, err + } + + for depName, files := range dependenciesFromChartsDir { + if dependencyExists(depName, dependencies) { + continue + } + + dep, err := r.mapHelmChart(depName, files) + if err != nil { + return nil, err + } + + dependencies = append(dependencies, dep) } // endregion template := &models.Template{ - Name: "", - Manifest: strings.Join(manifests, "---\n"), - RootField: mapper.HelmSchemaToFields("", schema, schema.Definitions, dependencies), - Created: "", - Edited: "", - Version: commit, + Name: path, ResolvedVersion: commitSHA, + Version: commit, + RootField: mapper.HelmSchemaToFields("", schema, schema.Definitions, dependencies), Files: chartFiles, + Templates: templateFiles, + CRDs: crdFiles, Dependencies: dependencies, HelmChartMetadata: metadata, + RawSchema: schemaBytes, + IconURL: metadata.Icon, } + // endregion r.cache.SetTemplate(repoURL, path, commitSHA, string(cyclopsv1alpha1.TemplateSourceTypeGit), template) @@ -318,6 +352,12 @@ func readValuesFile(fs billy.Filesystem, path string) ([]byte, error) { func clone(repoURL, commit string, creds *auth.Credentials) (billy.Filesystem, error) { // region clone from git + if gitproviders.IsAzureRepo(repoURL) { + transport.UnsupportedCapabilities = []capability.Capability{ + capability.ThinPack, + } + } + repo, err := git.Clone(memory.NewStorage(), memfs.New(), &git.CloneOptions{ URL: repoURL, Tags: git.AllTags, diff --git a/cyclops-ctrl/internal/template/gitproviders/azure.go b/cyclops-ctrl/internal/template/gitproviders/azure.go new file mode 100644 index 00000000..3fb43821 --- /dev/null +++ b/cyclops-ctrl/internal/template/gitproviders/azure.go @@ -0,0 +1,12 @@ +package gitproviders + +import "net/url" + +func IsAzureRepo(repoURL string) bool { + host, err := url.Parse(repoURL) + if err != nil { + return false + } + + return host.Host == "dev.azure.com" +}