allow componentsDir to be a glob (#193)

* allow componentsDir to be a glob

This adds support for multiple component directories expressed as a glob,
but does not introduce any namespace semantics. Components still have to
be unqiue across all directories.

* add tests

Author: gotwarlost

internal/model/app.go

@@ -466,71 +466,95 @@ func (a *App) DeclaredTopLevelVars() map[string]interface{} {
 	return ret
 }
 
-// loadComponents loads metadata for all components for the app.
-// The data is returned as a map keyed by component name. It does _not_ recurse
-// into subdirectories.
+// loadComponents loads metadata for all components for the app. It first expands the components directory
+// for glob patterns and loads components from all directories that match. It does _not_ recurse
+// into subdirectories. The data is returned as a map keyed by component name.
+// Note that component names must be unique across all directories. Support for multiple directories is just a
+// way to partition classes of components and does not introduce any namespace semantics.
 func (a *App) loadComponents() (map[string]Component, error) {
 	var list []Component
-	dir := strings.TrimSuffix(filepath.Clean(a.inner.Spec.ComponentsDir), "/")
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if path == dir {
-			return nil
-		}
-		if info.IsDir() {
-			files, err := filepath.Glob(filepath.Join(path, "*"))
+	loadDirComponents := func(dir string) error {
+		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
 				return err
 			}
-			var staticFiles []string
-			hasIndexJsonnet := false
-			hasIndexYAML := false
-			for _, f := range files {
-				stat, err := os.Stat(f)
+			if path == dir {
+				return nil
+			}
+			if info.IsDir() {
+				files, err := filepath.Glob(filepath.Join(path, "*"))
 				if err != nil {
 					return err
 				}
-				if stat.IsDir() {
-					continue
+				var staticFiles []string
+				hasIndexJsonnet := false
+				hasIndexYAML := false
+				for _, f := range files {
+					stat, err := os.Stat(f)
+					if err != nil {
+						return err
+					}
+					if stat.IsDir() {
+						continue
+					}
+					switch filepath.Base(f) {
+					case "index.jsonnet":
+						hasIndexJsonnet = true
+					case "index.yaml":
+						hasIndexYAML = true
+					}
+					if strings.HasSuffix(f, ".json") || strings.HasSuffix(f, ".yaml") {
+						staticFiles = append(staticFiles, f)
+					}
 				}
-				switch filepath.Base(f) {
-				case "index.jsonnet":
-					hasIndexJsonnet = true
-				case "index.yaml":
-					hasIndexYAML = true
-				}
-				if strings.HasSuffix(f, ".json") || strings.HasSuffix(f, ".yaml") {
-					staticFiles = append(staticFiles, f)
+				switch {
+				case hasIndexJsonnet:
+					list = append(list, Component{
+						Name:  filepath.Base(path),
+						Files: []string{filepath.Join(path, "index.jsonnet")},
+					})
+				case hasIndexYAML:
+					list = append(list, Component{
+						Name:  filepath.Base(path),
+						Files: staticFiles,
+					})
 				}
+				return filepath.SkipDir
 			}
-			switch {
-			case hasIndexJsonnet:
-				list = append(list, Component{
-					Name:  filepath.Base(path),
-					Files: []string{filepath.Join(path, "index.jsonnet")},
-				})
-			case hasIndexYAML:
+			extension := filepath.Ext(path)
+			if supportedExtensions[extension] {
 				list = append(list, Component{
-					Name:  filepath.Base(path),
-					Files: staticFiles,
+					Name:  strings.TrimSuffix(filepath.Base(path), extension),
+					Files: []string{path},
 				})
 			}
-			return filepath.SkipDir
-		}
-		extension := filepath.Ext(path)
-		if supportedExtensions[extension] {
-			list = append(list, Component{
-				Name:  strings.TrimSuffix(filepath.Base(path), extension),
-				Files: []string{path},
-			})
-		}
-		return nil
-	})
+			return nil
+		})
+		return err
+	}
+	ds, err := filepath.Glob(a.inner.Spec.ComponentsDir)
 	if err != nil {
 		return nil, err
 	}
+	var dirs []string
+	for _, d := range ds {
+		s, err := os.Stat(d)
+		if err != nil {
+			return nil, err
+		}
+		if s.IsDir() {
+			dirs = append(dirs, d)
+		}
+	}
+	if len(dirs) == 0 {
+		return nil, fmt.Errorf("no component directories found after expanding %s", a.inner.Spec.ComponentsDir)
+	}
+	for _, d := range dirs {
+		err := loadDirComponents(d)
+		if err != nil {
+			return nil, err
+		}
+	}
 	m := make(map[string]Component, len(list))
 	for _, c := range list {
 		if old, ok := m[c.Name]; ok {

internal/model/app_test.go

@@ -238,6 +238,7 @@ func TestAppSimple(t *testing.T) {
 
 	envs := app.Environments()
 	a.Equal(4, len(envs))
+	a.False(app.ClusterScopedLists())
 }
 
 func TestAppWarnings(t *testing.T) {
@@ -299,6 +300,33 @@ func TestAppComponentLoadSubdirs(t *testing.T) {
 	a.Contains(comp.Files, filepath.Join("components", "comp2", "index.yaml"))
 }
 
+func TestAppComponentLoadMultidirs(t *testing.T) {
+	reset := setPwd(t, "testdata/multi-dir-app")
+	defer reset()
+	app, err := NewApp("qbec.yaml", nil, "")
+	require.Nil(t, err)
+	comps, err := app.ComponentsForEnvironment("dev", nil, nil)
+	require.Nil(t, err)
+	a := assert.New(t)
+	a.Equal(2, len(comps))
+	comp := comps[0]
+	a.Equal("a", comp.Name)
+	a.Equal(1, len(comp.Files))
+	a.Contains(comp.Files, filepath.Join("components", "dir1", "a.jsonnet"))
+	comp = comps[1]
+	a.Equal("b", comp.Name)
+	a.Equal(1, len(comp.Files))
+	a.Contains(comp.Files, filepath.Join("components", "dir2", "b", "index.jsonnet"))
+}
+
+func TestAppComponentNoDirs(t *testing.T) {
+	reset := setPwd(t, "testdata/no-dirs-app")
+	defer reset()
+	_, err := NewApp("qbec.yaml", nil, "")
+	require.Error(t, err)
+	assert.Equal(t, "load components: no component directories found after expanding components/dir*", err.Error())
+}
+
 func TestAppComponentLoadNegative(t *testing.T) {
 	reset := setPwd(t, "../../examples/test-app")
 	defer reset()

internal/model/testdata/multi-dir-app/components/README.md

@@ -0,0 +1,4 @@
+### multi dirs
+
+A multi dir app is allowed to have files that are ignored.
+

internal/model/testdata/multi-dir-app/components/dir1/a.jsonnet

@@ -0,0 +1,10 @@
+{
+    apiVersion: 'v1',
+    kind: 'ConfigMap',
+    metadata: {
+        name: 'cm-a',
+    },
+    data: {
+      foo: 'bar'
+    }
+}

internal/model/testdata/multi-dir-app/components/dir2/b/index.jsonnet

@@ -0,0 +1,11 @@
+{
+    apiVersion: 'v1',
+    kind: 'ConfigMap',
+    metadata: {
+        name: 'cm-b',
+    },
+    data: {
+      bar: 'baz'
+    }
+}
+

internal/model/testdata/multi-dir-app/qbec.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: qbec.io/v1alpha1
+kind: App
+metadata:
+  name: multi-dir-app
+spec:
+  componentsDir: components/*
+  environments:
+    dev:
+      server: https://dev-server

internal/model/testdata/no-dirs-app/qbec.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: qbec.io/v1alpha1
+kind: App
+metadata:
+  name: no-dirs-app
+spec:
+  componentsDir: components/dir*
+  environments:
+    dev:
+      server: https://dev-server