From 6ba8cff733045b3e931f7c6bd9ed66d40ff90059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petar=20Cvitanovi=C4=87?= <72022639+petar-cvit@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:35:31 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=8A=20support=20common=20git=20source?= =?UTF-8?q?=20(#567)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * support common git source * ⚙️ update cyclops to v0.11.1-rc.1 * ✨ copy labels from current module (#569) * copy labels current module * copy labels current module * fix imports * remove owner ref * remove unused import * ⬆️ bump v0.11.1 (#571) * 🕒 progressing deployment state (#570) * progressing deployment state * progressing workload status * add callback * remove module status streaming * remove unused import * remove watch resources * remove unused * format controller * add resource button icons * ✏️ civo x cyclops (#573) * 🪝 fetch resources by cyclops module * ⛓️ fetch module list periodicaly (#577) * fetch module list periodicaly * fetch module list periodicaly * single fetching func * 💧 immutable dropdown (#578) * 🖋️ immutable fields docsw (#579) * 🆙 bump to 0.12.0 (#580) * 💿 sts progressing status (#581) * sts progressing status * ready replicas * 🌊 stream all module resources (#582) * stream all module resources * stream all module resources * disable streaming flag * kind to resource * ⚙️ update cyclops to v0.13.0-rc.1 * get module workloads * get module workloads * ⚙️ update cyclops to v0.13.0-rc.2 --------- Co-authored-by: github-actions[bot] * ✨ default target namespace (#583) * default target namespace * default namespace * 🧮 pass commit sha to get template (#584) * 🤖 bump express from 4.19.2 to 4.21.0 in /cyclops-ui (#576) Bumps [express](https://github.com/expressjs/express) from 4.19.2 to 4.21.0. - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/4.21.0/History.md) - [Commits](https://github.com/expressjs/express/compare/4.19.2...4.21.0) --- updated-dependencies: - dependency-name: express dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * ✨ smaller default template image (#587) * optimize default module image * optimize default module image * optimize default module image * 🪗 advanced module section (#588) * ⚙️ update cyclops to v0.13.0 * 🫥 remove use navigate (#589) * ⚙️ update cyclops to v0.13.1 * ⚙️ docs to v0.13.1 (#590) * 🌊 support azure repos (#664) * support azure repos * return error if no files found --------- Signed-off-by: dependabot[bot] Co-authored-by: github-actions[bot] Co-authored-by: KaradzaJuraj <101581637+KaradzaJuraj@users.noreply.github.com> Co-authored-by: Nitin Tejuja <95347924+nitintecg@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- cyclops-ctrl/internal/template/git.go | 140 +++++++++++------- .../internal/template/gitproviders/azure.go | 12 ++ 2 files changed, 102 insertions(+), 50 deletions(-) create mode 100644 cyclops-ctrl/internal/template/gitproviders/azure.go diff --git a/cyclops-ctrl/internal/template/git.go b/cyclops-ctrl/internal/template/git.go index 30abe800..4f14b0f2 100644 --- a/cyclops-ctrl/internal/template/git.go +++ b/cyclops-ctrl/internal/template/git.go @@ -18,6 +18,8 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" "helm.sh/helm/v3/pkg/chart" @@ -70,87 +72,119 @@ func (r Repo) LoadTemplate(repoURL, path, commit, resolvedVersion string) (*mode return nil, err } - // load helm chart metadata - chartMetadata, err := fs.Open(path2.Join(path, "Chart.yaml")) + // region read files + filesFs, err := fs.Chroot(path) if err != nil { - return nil, errors.Wrap(err, "could not read 'Chart.yaml' file; it should be placed in the repo/path you provided; make sure you provided the correct path") + return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists") } - var chartMetadataBuffer bytes.Buffer - _, err = io.Copy(bufio.NewWriter(&chartMetadataBuffer), chartMetadata) + files, err := readFiles("", filesFs) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to read template files") } + // endregion - var metadata *helm.Metadata - if err := yaml.Unmarshal(chartMetadataBuffer.Bytes(), &metadata); err != nil { - return nil, err + if len(files) == 0 { + return nil, errors.Errorf("no files found in repo %v on path %v; make sure the repo, path and version are correct", repoURL, path) } - // endregion - // region read templates - templatesPath := path2.Join(path, "templates") + // region map files to template + metadataBytes := []byte{} + schemaBytes := []byte{} + chartFiles := make([]*chart.File, 0) - _, err = fs.ReadDir(templatesPath) - if err != nil { - return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists") - } + templateFiles := make([]*chart.File, 0) + crdFiles := make([]*chart.File, 0) - manifests, err := concatenateTemplates(templatesPath, fs) - if err != nil { - return nil, errors.Wrap(err, "failed to load template files") - } - // endregion + dependenciesFromChartsDir := make(map[string]map[string][]byte, 0) - // region read files - filesFs, err := fs.Chroot(path) - if err != nil { - return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists") - } + for _, f := range files { + parts := strings.Split(f.Name, "/") - chartFiles, err := readFiles("", filesFs) - if err != nil { - return nil, errors.Wrap(err, "failed to read template files") - } - // endregion + if len(parts) == 1 && parts[0] == "Chart.yaml" { + metadataBytes = f.Data + continue + } - // region read schema - schemaFile, err := fs.Open(path2.Join(path, "values.schema.json")) - if err != nil { - return nil, errors.Wrap(err, "could not read 'values.schema.json' file; it should be placed in the repo/path you provided; make sure 'templates' directory exists") - } + if len(parts) == 1 && parts[0] == "values.schema.json" { + schemaBytes = f.Data + continue + } - var schemaChartBuffer bytes.Buffer - _, err = io.Copy(bufio.NewWriter(&schemaChartBuffer), schemaFile) - if err != nil { - return nil, err + if len(parts) > 1 && parts[0] == "templates" && + (parts[1] != "Notes.txt" && parts[1] != "NOTES.txt" && parts[1] != "tests") { + templateFiles = append(templateFiles, f) + continue + } + + if len(parts) > 1 && parts[0] == "crds" && + (parts[1] != "Notes.txt" && parts[1] != "NOTES.txt" && parts[1] != "tests") { + crdFiles = append(crdFiles, f) + continue + } + + if len(parts) > 2 && parts[0] == "charts" { + depName := parts[1] + if _, ok := dependenciesFromChartsDir[depName]; !ok { + dependenciesFromChartsDir[depName] = make(map[string][]byte) + } + + dependenciesFromChartsDir[depName][path2.Join(parts[1:]...)] = f.Data + continue + } + + chartFiles = append(chartFiles, f) } var schema helm.Property - if err := json.Unmarshal(schemaChartBuffer.Bytes(), &schema); err != nil { - return nil, err + // unmarshal values schema only if present + if len(schemaBytes) > 0 { + if err := json.Unmarshal(schemaBytes, &schema); err != nil { + fmt.Println("error on schema bytes", repoURL, path) + return &models.Template{}, err + } + } + + var metadata *helm.Metadata + if err := yaml.Unmarshal(metadataBytes, &metadata); err != nil { + fmt.Println("error on meta unmarshal", repoURL, path) + return &models.Template{}, err } - // endregion // region load dependencies dependencies, err := r.loadDependencies(metadata) if err != nil { - return nil, err + return &models.Template{}, err + } + + for depName, files := range dependenciesFromChartsDir { + if dependencyExists(depName, dependencies) { + continue + } + + dep, err := r.mapHelmChart(depName, files) + if err != nil { + return nil, err + } + + dependencies = append(dependencies, dep) } // endregion template := &models.Template{ - Name: "", - Manifest: strings.Join(manifests, "---\n"), - RootField: mapper.HelmSchemaToFields("", schema, schema.Definitions, dependencies), - Created: "", - Edited: "", - Version: commit, + Name: path, ResolvedVersion: commitSHA, + Version: commit, + RootField: mapper.HelmSchemaToFields("", schema, schema.Definitions, dependencies), Files: chartFiles, + Templates: templateFiles, + CRDs: crdFiles, Dependencies: dependencies, HelmChartMetadata: metadata, + RawSchema: schemaBytes, + IconURL: metadata.Icon, } + // endregion r.cache.SetTemplate(repoURL, path, commitSHA, string(cyclopsv1alpha1.TemplateSourceTypeGit), template) @@ -318,6 +352,12 @@ func readValuesFile(fs billy.Filesystem, path string) ([]byte, error) { func clone(repoURL, commit string, creds *auth.Credentials) (billy.Filesystem, error) { // region clone from git + if gitproviders.IsAzureRepo(repoURL) { + transport.UnsupportedCapabilities = []capability.Capability{ + capability.ThinPack, + } + } + repo, err := git.Clone(memory.NewStorage(), memfs.New(), &git.CloneOptions{ URL: repoURL, Tags: git.AllTags, diff --git a/cyclops-ctrl/internal/template/gitproviders/azure.go b/cyclops-ctrl/internal/template/gitproviders/azure.go new file mode 100644 index 00000000..3fb43821 --- /dev/null +++ b/cyclops-ctrl/internal/template/gitproviders/azure.go @@ -0,0 +1,12 @@ +package gitproviders + +import "net/url" + +func IsAzureRepo(repoURL string) bool { + host, err := url.Parse(repoURL) + if err != nil { + return false + } + + return host.Host == "dev.azure.com" +}