Cross-module support

This patch updates the controller-gen package loader to support
loading packages across Go module boundaries.

Author: akutz

pkg/loader/loader.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Kubernetes Authors.
+Copyright 2019-2022 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -25,9 +25,12 @@ import (
 	"go/types"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strings"
 	"sync"
 
 	"golang.org/x/tools/go/packages"
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 // Much of this is strongly inspired by the contents of go/packages,
@@ -329,7 +332,7 @@ func LoadRoots(roots ...string) ([]*Package, error) {
 //
 // This is generally only useful for use in testing when you need to modify
 // loading settings to load from a fake location.
-func LoadRootsWithConfig(cfg *packages.Config, roots ...string) ([]*Package, error) {
+func LoadRootsWithConfig(cfg *packages.Config, roots ...string) (pkgs []*Package, retErr error) {
 	l := &loader{
 		cfg:      cfg,
 		packages: make(map[*packages.Package]*Package),
@@ -341,13 +344,221 @@ func LoadRootsWithConfig(cfg *packages.Config, roots ...string) ([]*Package, err
 	// put our build flags first so that callers can override them
 	l.cfg.BuildFlags = append([]string{"-tags", "ignore_autogenerated"}, l.cfg.BuildFlags...)
 
-	rawPkgs, err := packages.Load(l.cfg, roots...)
-	if err != nil {
-		return nil, err
+	// ensure the cfg.Dir field is reset to its original value upon
+	// returning from this function. it should honestly be fine if it is
+	// not given most callers will not send in the cfg parameter directly,
+	// as it's largely for testing, but still, let's be good stewards.
+	defer func(d string) {
+		cfg.Dir = d
+	}(cfg.Dir)
+
+	// ensure cfg.Dir is absolute. it just makes the rest of the code easier
+	// to debug when there are any issues
+	if cfg.Dir != "" && !filepath.IsAbs(cfg.Dir) {
+		ap, err := filepath.Abs(cfg.Dir)
+		if err != nil {
+			return nil, err
+		}
+		cfg.Dir = ap
 	}
 
-	for _, rawPkg := range rawPkgs {
-		l.Roots = append(l.Roots, l.packageFor(rawPkg))
+	// loadRoots is a helper function used in three locations below.
+	loadRoots := func(uniqueRoots *sets.String, roots ...string) error {
+		// load the root and gets its raw packages
+		rawPkgs, err := packages.Load(l.cfg, roots...)
+		if err != nil {
+			return err
+		}
+
+		// get the package path for each raw package
+		for _, rawPkg := range rawPkgs {
+			pkg := l.packageFor(rawPkg)
+
+			// if a set was provided to prevent duplicate packages, use it,
+			// otherwise append to the package list
+			if uniqueRoots == nil {
+				l.Roots = append(l.Roots, pkg)
+			} else if !uniqueRoots.Has(pkg.ID) {
+				l.Roots = append(l.Roots, pkg)
+				uniqueRoots.Insert(pkg.ID)
+			}
+		}
+
+		return nil
+	}
+
+	// if no roots were provided then load the current package and return early
+	if len(roots) == 0 {
+		if err := loadRoots(nil); err != nil {
+			return nil, err
+		}
+		return l.Roots, nil
+	}
+
+	// store the value of cfg.Dir so we can use it later if it is non-empty.
+	// we need to store it now as the value of cfg.Dir will be updated by
+	// a loop below
+	cfgDir := cfg.Dir
+
+	// uniqueRoots is used to keep track of the discovered packages to be nice
+	// and try and prevent packages from showing up twice when nested module
+	// support is enabled. there is not harm that comes from this per se, but
+	// it makes testing easier when a known number of modules can be asserted
+	uniqueRoots := sets.String{}
+
+	// isFSRoots is used to keep track of whether a root is a filesystem path
+	// or a package/module name. the logic to determine this is based on the
+	// output from "go help packages". The "go" command determines if a path
+	// is a filesystem path based on whether it is:
+	//
+	//   * absolute
+	//   * begins with "."
+	//   * begins with ".."
+	//
+	// otherwise the path is considered to be a package. this set is a list of
+	// the indices from the roots slice that are filesystem paths
+	isFSRoots := sets.Int{}
+
+	// addNestedGoModulesToRoots is given to filepath.WalkDir
+	// and adds the directory part of p to the list of roots
+	// IFF p is the path to a file named "go.mod"
+	addNestedGoModulesToRoots := func(
+		p string,
+		d os.DirEntry,
+		e error) error {
+
+		if e != nil {
+			return e
+		}
+		if !d.IsDir() && filepath.Base(p) == "go.mod" {
+			roots = append(roots, filepath.Join(filepath.Dir(p), "..."))
+			isFSRoots.Insert(len(roots) - 1)
+		}
+		return nil
+	}
+
+	// the basic loader logic is as follows:
+	//
+	//    1. iterate over the list of provided roots
+	//
+	//    2. if a root uses the nested path syntax, ex. ..., then walk the
+	//       root's descendants to search for any any nested Go modules, and if
+	//       found, load them to the list of roots
+	//
+	//    3. iterate over the list of updated roots
+	//
+	//    4. update the loader config's Dir property to be the directory element
+	//       of the current root
+	//
+	//    5. execute the loader on the base element of the current root, which
+	//       will be either "./." or "./..."
+	//
+	// the following range operation is parts 1-2
+	for i := range roots {
+		r := roots[i]
+
+		//fmt.Printf("1.processing root=%s\n", r)
+
+		// based on the logic from "go help packages", a path is a package/mod
+		// name if it is absolute or begins with "." or "..". If it is none of
+		// those things then go ahead and skip to the next iteration of the loop
+		if !filepath.IsAbs(r) &&
+			!strings.HasPrefix(r, ".") &&
+			!strings.HasPrefix(r, "..") {
+
+			continue
+		}
+
+		//fmt.Printf("1.processing root as file=%s\n", r)
+
+		// this is a filesytem path
+		isFSRoots.Insert(i)
+
+		// clean up the root
+		r = filepath.Clean(r)
+
+		// get the absolute path of the root
+		if !filepath.IsAbs(r) {
+
+			// if the initial value of cfg.Dir was non-empty then use it when
+			// building the absolute path to this root. otherwise use the
+			// filepath.Abs function to get the absolute path of the root based
+			// on the working directory
+			if cfgDir != "" {
+				r = filepath.Join(cfgDir, r)
+			} else {
+				ar, err := filepath.Abs(r)
+				if err != nil {
+					return nil, err
+				}
+				r = ar
+			}
+		}
+
+		// update the root to be an absolute path
+		roots[i] = r
+
+		b, d := filepath.Base(r), filepath.Dir(r)
+
+		// if the base element is "..." then it means nested traversal is
+		// activated. this can be passed directly to the loader. however, if
+		// specified we also want to traverse the path manually to determine if
+		// there are any nested Go modules we want to add to the list of roots
+		// to process
+		if b == "..." {
+			if err := filepath.WalkDir(
+				d,
+				addNestedGoModulesToRoots); err != nil {
+
+				return nil, err
+			}
+		}
+	}
+
+	// this range operation is parts 3-5 from above.
+	for i := range roots {
+		r := roots[i]
+
+		//fmt.Printf("2.processing root=%s\n", r)
+
+		// if the root is not a filesystem path then just load it directly
+		// and skip to the next iteration of the loop
+		if !isFSRoots.Has(i) {
+			l.cfg.Dir = cfgDir
+			if err := loadRoots(&uniqueRoots, r); err != nil {
+				return nil, err
+			}
+			//fmt.Printf("2.skipping root=%s\n", r)
+			continue
+		}
+
+		b, d := filepath.Base(r), filepath.Dir(r)
+
+		// we want the base part of the path to be either "..." or ".", except
+		// Go's filepath utilities clean paths during manipulation, removing the
+		// ".". thus, if not "...", let's update the path components so that:
+		//
+		//   d = r
+		//   b = "."
+		if b != "..." {
+			d = r
+			b = "."
+		}
+
+		// update the loader configuration's Dir field to the directory part of
+		// the root
+		l.cfg.Dir = d
+
+		// update the root to be "./..." or "./."
+		// (with OS-specific filepath separator). please note filepath.Join
+		// would clean up the trailing "." character that we want preserved,
+		// hence the more manual path concatenation logic
+		r = fmt.Sprintf(".%s%s", string(filepath.Separator), b)
+
+		// load the root
+		if err := loadRoots(&uniqueRoots, r); err != nil {
+			return nil, err
+		}
 	}
 
 	return l.Roots, nil

pkg/loader/loader_suite_test.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package loader_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestLoader(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Loader Patching Suite")
+}