🌊 support common git source (#567)

* support common git source

* ⚙️ update cyclops to v0.11.1-rc.1

* ✨ copy labels from current module (#569)

* copy labels current module

* copy labels current module

* fix imports

* remove owner ref

* remove unused import

* ⬆️ bump v0.11.1 (#571)

* 🕒 progressing deployment state (#570)

* progressing deployment state

* progressing workload status

* add callback

* remove module status streaming

* remove unused import

* remove watch resources

* remove unused

* format controller

* add resource button icons

* ✏️ civo x cyclops (#573)

* 🪝 fetch resources by cyclops module

* ⛓️ fetch module list periodicaly (#577)

* fetch module list periodicaly

* fetch module list periodicaly

* single fetching func

* 💧 immutable dropdown (#578)

* 🖋️ immutable fields docsw (#579)

* 🆙 bump to 0.12.0 (#580)

* 💿 sts progressing status (#581)

* sts progressing status

* ready replicas

* 🌊 stream all module resources (#582)

* stream all module resources

* stream all module resources

* disable streaming flag

* kind to resource

* ⚙️ update cyclops to v0.13.0-rc.1

* get module workloads

* get module workloads

* ⚙️ update cyclops to v0.13.0-rc.2

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

* ✨ default target namespace (#583)

* default target namespace

* default namespace

* 🧮 pass commit sha to get template (#584)

* 🤖 bump express from 4.19.2 to 4.21.0 in /cyclops-ui (#576)

Bumps [express](https://github.com/expressjs/express) from 4.19.2 to 4.21.0.
- [Release notes](https://github.com/expressjs/express/releases)
- [Changelog](https://github.com/expressjs/express/blob/4.21.0/History.md)
- [Commits](expressjs/express@4.19.2...4.21.0)

---
updated-dependencies:
- dependency-name: express
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* ✨ smaller default template image (#587)

* optimize default module image

* optimize default module image

* optimize default module image

* 🪗 advanced module section (#588)

* ⚙️ update cyclops to v0.13.0

* 🫥 remove use navigate (#589)

* ⚙️ update cyclops to v0.13.1

* ⚙️ docs to v0.13.1 (#590)

* 🌊 support azure repos (#664)

* support azure repos

* return error if no files found

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: KaradzaJuraj <[email protected]>
Co-authored-by: Nitin Tejuja <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

cyclops-ctrl/internal/template/git.go

@@ -18,6 +18,8 @@ import (
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/protocol/packp/capability"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/storage/memory"
 	"helm.sh/helm/v3/pkg/chart"
@@ -70,87 +72,119 @@ func (r Repo) LoadTemplate(repoURL, path, commit, resolvedVersion string) (*mode
 		return nil, err
 	}
 
-	// load helm chart metadata
-	chartMetadata, err := fs.Open(path2.Join(path, "Chart.yaml"))
+	// region read files
+	filesFs, err := fs.Chroot(path)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not read 'Chart.yaml' file; it should be placed in the repo/path you provided; make sure you provided the correct path")
+		return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists")
 	}
 
-	var chartMetadataBuffer bytes.Buffer
-	_, err = io.Copy(bufio.NewWriter(&chartMetadataBuffer), chartMetadata)
+	files, err := readFiles("", filesFs)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed to read template files")
 	}
+	// endregion
 
-	var metadata *helm.Metadata
-	if err := yaml.Unmarshal(chartMetadataBuffer.Bytes(), &metadata); err != nil {
-		return nil, err
+	if len(files) == 0 {
+		return nil, errors.Errorf("no files found in repo %v on path %v; make sure the repo, path and version are correct", repoURL, path)
 	}
-	// endregion
 
-	// region read templates
-	templatesPath := path2.Join(path, "templates")
+	// region map files to template
+	metadataBytes := []byte{}
+	schemaBytes := []byte{}
+	chartFiles := make([]*chart.File, 0)
 
-	_, err = fs.ReadDir(templatesPath)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists")
-	}
+	templateFiles := make([]*chart.File, 0)
+	crdFiles := make([]*chart.File, 0)
 
-	manifests, err := concatenateTemplates(templatesPath, fs)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to load template files")
-	}
-	// endregion
+	dependenciesFromChartsDir := make(map[string]map[string][]byte, 0)
 
-	// region read files
-	filesFs, err := fs.Chroot(path)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists")
-	}
+	for _, f := range files {
+		parts := strings.Split(f.Name, "/")
 
-	chartFiles, err := readFiles("", filesFs)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to read template files")
-	}
-	// endregion
+		if len(parts) == 1 && parts[0] == "Chart.yaml" {
+			metadataBytes = f.Data
+			continue
+		}
 
-	// region read schema
-	schemaFile, err := fs.Open(path2.Join(path, "values.schema.json"))
-	if err != nil {
-		return nil, errors.Wrap(err, "could not read 'values.schema.json' file; it should be placed in the repo/path you provided; make sure 'templates' directory exists")
-	}
+		if len(parts) == 1 && parts[0] == "values.schema.json" {
+			schemaBytes = f.Data
+			continue
+		}
 
-	var schemaChartBuffer bytes.Buffer
-	_, err = io.Copy(bufio.NewWriter(&schemaChartBuffer), schemaFile)
-	if err != nil {
-		return nil, err
+		if len(parts) > 1 && parts[0] == "templates" &&
+			(parts[1] != "Notes.txt" && parts[1] != "NOTES.txt" && parts[1] != "tests") {
+			templateFiles = append(templateFiles, f)
+			continue
+		}
+
+		if len(parts) > 1 && parts[0] == "crds" &&
+			(parts[1] != "Notes.txt" && parts[1] != "NOTES.txt" && parts[1] != "tests") {
+			crdFiles = append(crdFiles, f)
+			continue
+		}
+
+		if len(parts) > 2 && parts[0] == "charts" {
+			depName := parts[1]
+			if _, ok := dependenciesFromChartsDir[depName]; !ok {
+				dependenciesFromChartsDir[depName] = make(map[string][]byte)
+			}
+
+			dependenciesFromChartsDir[depName][path2.Join(parts[1:]...)] = f.Data
+			continue
+		}
+
+		chartFiles = append(chartFiles, f)
 	}
 
 	var schema helm.Property
-	if err := json.Unmarshal(schemaChartBuffer.Bytes(), &schema); err != nil {
-		return nil, err
+	// unmarshal values schema only if present
+	if len(schemaBytes) > 0 {
+		if err := json.Unmarshal(schemaBytes, &schema); err != nil {
+			fmt.Println("error on schema bytes", repoURL, path)
+			return &models.Template{}, err
+		}
+	}
+
+	var metadata *helm.Metadata
+	if err := yaml.Unmarshal(metadataBytes, &metadata); err != nil {
+		fmt.Println("error on meta unmarshal", repoURL, path)
+		return &models.Template{}, err
 	}
-	// endregion
 
 	// region load dependencies
 	dependencies, err := r.loadDependencies(metadata)
 	if err != nil {
-		return nil, err
+		return &models.Template{}, err
+	}
+
+	for depName, files := range dependenciesFromChartsDir {
+		if dependencyExists(depName, dependencies) {
+			continue
+		}
+
+		dep, err := r.mapHelmChart(depName, files)
+		if err != nil {
+			return nil, err
+		}
+
+		dependencies = append(dependencies, dep)
 	}
 	// endregion
 
 	template := &models.Template{
-		Name:              "",
-		Manifest:          strings.Join(manifests, "---\n"),
-		RootField:         mapper.HelmSchemaToFields("", schema, schema.Definitions, dependencies),
-		Created:           "",
-		Edited:            "",
-		Version:           commit,
+		Name:              path,
 		ResolvedVersion:   commitSHA,
+		Version:           commit,
+		RootField:         mapper.HelmSchemaToFields("", schema, schema.Definitions, dependencies),
 		Files:             chartFiles,
+		Templates:         templateFiles,
+		CRDs:              crdFiles,
 		Dependencies:      dependencies,
 		HelmChartMetadata: metadata,
+		RawSchema:         schemaBytes,
+		IconURL:           metadata.Icon,
 	}
+	// endregion
 
 	r.cache.SetTemplate(repoURL, path, commitSHA, string(cyclopsv1alpha1.TemplateSourceTypeGit), template)
 
@@ -318,6 +352,12 @@ func readValuesFile(fs billy.Filesystem, path string) ([]byte, error) {
 
 func clone(repoURL, commit string, creds *auth.Credentials) (billy.Filesystem, error) {
 	// region clone from git
+	if gitproviders.IsAzureRepo(repoURL) {
+		transport.UnsupportedCapabilities = []capability.Capability{
+			capability.ThinPack,
+		}
+	}
+
 	repo, err := git.Clone(memory.NewStorage(), memfs.New(), &git.CloneOptions{
 		URL:  repoURL,
 		Tags: git.AllTags,

cyclops-ctrl/internal/template/gitproviders/azure.go

@@ -0,0 +1,12 @@
+package gitproviders
+
+import "net/url"
+
+func IsAzureRepo(repoURL string) bool {
+	host, err := url.Parse(repoURL)
+	if err != nil {
+		return false
+	}
+
+	return host.Host == "dev.azure.com"
+}