d2ir: Globs

ci/release/changelogs/next.md

@@ -9,6 +9,7 @@ Layout capability also takes a subtle but important step forward: you can now cu
 - Scale renders and disable fit to screen with `--scale` flag [#1413](https://github.com/terrastruct/d2/pull/1413)
 - `null` keyword can be used to un-declare. See [docs](https://d2lang.com/tour/overrides#null) [#1446](https://github.com/terrastruct/d2/pull/1446)
 - Develop multi-board diagrams in watch mode (links to layers/scenarios/steps work in `--watch`) [#1503](https://github.com/terrastruct/d2/pull/1503)
+- Glob patterns have been implemented. See [docs](https://d2lang.com/tour/globs). [#1479](https://github.com/terrastruct/d2/pull/1479)
 
 #### Improvements 🧹
 

d2ast/d2ast.go

@@ -500,6 +500,8 @@ type Number struct {
 type UnquotedString struct {
 	Range Range              `json:"range"`
 	Value []InterpolationBox `json:"value"`
+	// Pattern holds the parsed glob pattern if in a key and the unquoted string represents a valid pattern.
+	Pattern []string `json:"pattern,omitempty"`
 }
 
 func (s *UnquotedString) Coalesce() {
@@ -738,6 +740,31 @@ func (kp *KeyPath) IDA() (ida []string) {
 	return ida
 }
 
+func (kp *KeyPath) Copy() *KeyPath {
+	kp2 := *kp
+	kp2.Path = nil
+	kp2.Path = append(kp2.Path, kp.Path...)
+	return &kp2
+}
+
+func (kp *KeyPath) HasDoubleGlob() bool {
+	for _, el := range kp.Path {
+		if el.UnquotedString != nil && el.ScalarString() == "**" {
+			return true
+		}
+	}
+	return false
+}
+
+func (kp *KeyPath) HasGlob() bool {
+	for _, el := range kp.Path {
+		if el.UnquotedString != nil && len(el.UnquotedString.Pattern) > 0 {
+			return true
+		}
+	}
+	return false
+}
+
 type Edge struct {
 	Range Range `json:"range"`
 
@@ -1056,6 +1083,10 @@ func (sb *StringBox) Unbox() String {
 	}
 }
 
+func (sb *StringBox) ScalarString() string {
+	return sb.Unbox().ScalarString()
+}
+
 // InterpolationBox is used to select between strings and substitutions in unquoted and
 // double quoted strings. There is no corresponding interface to avoid unnecessary
 // abstraction.

d2ir/compile.go

@@ -396,19 +396,26 @@ func (c *compiler) compileKey(refctx *RefContext) {
 }
 
 func (c *compiler) compileField(dst *Map, kp *d2ast.KeyPath, refctx *RefContext) {
-	if refctx.Key != nil && len(refctx.Key.Edges) == 0 && refctx.Key.Value.Null != nil {
+	fa, err := dst.EnsureField(kp, refctx, true)
+	if err != nil {
+		c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
+		return
+	}
+
+	for _, f := range fa {
+		c._compileField(f, refctx)
+	}
+}
+
+func (c *compiler) _compileField(f *Field, refctx *RefContext) {
+	if len(refctx.Key.Edges) == 0 && refctx.Key.Value.Null != nil {
 		// For vars, if we delete the field, it may just resolve to an outer scope var of the same name
 		// Instead we keep it around, so that resolveSubstitutions can find it
-		if !IsVar(dst) {
-			dst.DeleteField(kp.IDA()...)
+		if !IsVar(ParentMap(f)) {
+			ParentMap(f).DeleteField(f.Name)
 			return
 		}
 	}
-	f, err := dst.EnsureField(kp, refctx)
-	if err != nil {
-		c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
-		return
-	}
 
 	if refctx.Key.Primary.Unbox() != nil {
 		f.Primary_ = &Scalar{
@@ -602,12 +609,17 @@ func (c *compiler) compileLink(refctx *RefContext) {
 }
 
 func (c *compiler) compileEdges(refctx *RefContext) {
-	if refctx.Key.Key != nil {
-		f, err := refctx.ScopeMap.EnsureField(refctx.Key.Key, refctx)
-		if err != nil {
-			c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
-			return
-		}
+	if refctx.Key.Key == nil {
+		c._compileEdges(refctx)
+		return
+	}
+
+	fa, err := refctx.ScopeMap.EnsureField(refctx.Key.Key, refctx, true)
+	if err != nil {
+		c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
+		return
+	}
+	for _, f := range fa {
 		if _, ok := f.Composite.(*Array); ok {
 			c.errorf(refctx.Key.Key, "cannot index into array")
 			return
@@ -617,9 +629,13 @@ func (c *compiler) compileEdges(refctx *RefContext) {
 				parent: f,
 			}
 		}
-		refctx.ScopeMap = f.Map()
+		refctx2 := *refctx
+		refctx2.ScopeMap = f.Map()
+		c._compileEdges(&refctx2)
 	}
+}
 
+func (c *compiler) _compileEdges(refctx *RefContext) {
 	eida := NewEdgeIDs(refctx.Key)
 	for i, eid := range eida {
 		if refctx.Key != nil && refctx.Key.Value.Null != nil {
@@ -630,66 +646,59 @@ func (c *compiler) compileEdges(refctx *RefContext) {
 		refctx = refctx.Copy()
 		refctx.Edge = refctx.Key.Edges[i]
 
-		var e *Edge
-		if eid.Index != nil {
-			ea := refctx.ScopeMap.GetEdges(eid)
+		var ea []*Edge
+		if eid.Index != nil || eid.Glob {
+			ea = refctx.ScopeMap.GetEdges(eid, refctx)
 			if len(ea) == 0 {
 				c.errorf(refctx.Edge, "indexed edge does not exist")
 				continue
 			}
-			e = ea[0]
-			e.References = append(e.References, &EdgeReference{
-				Context: refctx,
-			})
-			refctx.ScopeMap.appendFieldReferences(0, refctx.Edge.Src, refctx)
-			refctx.ScopeMap.appendFieldReferences(0, refctx.Edge.Dst, refctx)
-		} else {
-			_, err := refctx.ScopeMap.EnsureField(refctx.Edge.Src, refctx)
-			if err != nil {
-				c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
-				continue
-			}
-			_, err = refctx.ScopeMap.EnsureField(refctx.Edge.Dst, refctx)
-			if err != nil {
-				c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
-				continue
+			for _, e := range ea {
+				e.References = append(e.References, &EdgeReference{
+					Context: refctx,
+				})
+				refctx.ScopeMap.appendFieldReferences(0, refctx.Edge.Src, refctx)
+				refctx.ScopeMap.appendFieldReferences(0, refctx.Edge.Dst, refctx)
 			}
-
-			e, err = refctx.ScopeMap.CreateEdge(eid, refctx)
+		} else {
+			var err error
+			ea, err = refctx.ScopeMap.CreateEdge(eid, refctx)
 			if err != nil {
 				c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
 				continue
 			}
 		}
 
-		if refctx.Key.EdgeKey != nil {
-			if e.Map_ == nil {
-				e.Map_ = &Map{
-					parent: e,
-				}
-			}
-			c.compileField(e.Map_, refctx.Key.EdgeKey, refctx)
-		} else {
-			if refctx.Key.Primary.Unbox() != nil {
-				e.Primary_ = &Scalar{
-					parent: e,
-					Value:  refctx.Key.Primary.Unbox(),
-				}
-			}
-			if refctx.Key.Value.Array != nil {
-				c.errorf(refctx.Key.Value.Unbox(), "edges cannot be assigned arrays")
-				continue
-			} else if refctx.Key.Value.Map != nil {
+		for _, e := range ea {
+			if refctx.Key.EdgeKey != nil {
 				if e.Map_ == nil {
 					e.Map_ = &Map{
 						parent: e,
 					}
 				}
-				c.compileMap(e.Map_, refctx.Key.Value.Map, refctx.ScopeAST)
-			} else if refctx.Key.Value.ScalarBox().Unbox() != nil {
-				e.Primary_ = &Scalar{
-					parent: e,
-					Value:  refctx.Key.Value.ScalarBox().Unbox(),
+				c.compileField(e.Map_, refctx.Key.EdgeKey, refctx)
+			} else {
+				if refctx.Key.Primary.Unbox() != nil {
+					e.Primary_ = &Scalar{
+						parent: e,
+						Value:  refctx.Key.Primary.Unbox(),
+					}
+				}
+				if refctx.Key.Value.Array != nil {
+					c.errorf(refctx.Key.Value.Unbox(), "edges cannot be assigned arrays")
+					continue
+				} else if refctx.Key.Value.Map != nil {
+					if e.Map_ == nil {
+						e.Map_ = &Map{
+							parent: e,
+						}
+					}
+					c.compileMap(e.Map_, refctx.Key.Value.Map, refctx.ScopeAST)
+				} else if refctx.Key.Value.ScalarBox().Unbox() != nil {
+					e.Primary_ = &Scalar{
+						parent: e,
+						Value:  refctx.Key.Value.ScalarBox().Unbox(),
+					}
 				}
 			}
 		}

d2ir/compile_test.go

@@ -26,6 +26,7 @@ func TestCompile(t *testing.T) {
 	t.Run("scenarios", testCompileScenarios)
 	t.Run("steps", testCompileSteps)
 	t.Run("imports", testCompileImports)
+	t.Run("patterns", testCompilePatterns)
 }
 
 type testCase struct {
@@ -84,23 +85,31 @@ func assertQuery(t testing.TB, n d2ir.Node, nfields, nedges int, primary interfa
 	m := n.Map()
 	p := n.Primary()
 
+	var na []d2ir.Node
 	if idStr != "" {
 		var err error
-		n, err = m.Query(idStr)
+		na, err = m.QueryAll(idStr)
 		assert.Success(t, err)
 		assert.NotEqual(t, n, nil)
+	} else {
+		na = append(na, n)
+	}
 
-		p = n.Primary()
+	for _, n := range na {
 		m = n.Map()
+		p = n.Primary()
+		assert.Equal(t, nfields, m.FieldCountRecursive())
+		assert.Equal(t, nedges, m.EdgeCountRecursive())
+		if !makeScalar(p).Equal(makeScalar(primary)) {
+			t.Fatalf("expected primary %#v but got %s", primary, p)
+		}
 	}
 
-	assert.Equal(t, nfields, m.FieldCountRecursive())
-	assert.Equal(t, nedges, m.EdgeCountRecursive())
-	if !makeScalar(p).Equal(makeScalar(primary)) {
-		t.Fatalf("expected primary %#v but got %s", primary, p)
+	if len(na) == 0 {
+		return nil
 	}
 
-	return n
+	return na[0]
 }
 
 func makeScalar(v interface{}) *d2ir.Scalar {