Nested plugin binaries are output to their respective directories (#1007

)

* do it

* fix lint issues

* simplify buildInfoJSON regex for test

build/common.go

@@ -6,9 +6,12 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
+	"sync"
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
@@ -24,8 +27,11 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/internal"
 )
 
-var defaultOutputBinaryPath = "dist"
-var defaultPluginJSONPath = "src"
+var (
+	defaultOutputBinaryPath     = "dist"
+	defaultPluginJSONPath       = "src"
+	defaultNestedDataSourcePath = "datasource"
+)
 
 // Callbacks give you a way to run custom behavior when things happen
 var beforeBuild = func(cfg Config) (Config, error) {
@@ -40,6 +46,10 @@ func SetBeforeBuildCallback(cb BeforeBuildCallback) error {
 
 var exname string
 
+// Deprecated: Use getExecutableNameForPlugin instead.
+// getExecutableName returns the name of the executable for the current platform.
+// It reads the plugin.json from the directory specified in `pluginJSONPath`. It uses internal.GetExecutableFromPluginJSON
+// which will also retrieve the executable from a nested datasource directory which may not be the desired behavior.
 func getExecutableName(os string, arch string, pluginJSONPath string) (string, error) {
 	if exname == "" {
 		exename, err := internal.GetExecutableFromPluginJSON(pluginJSONPath)
@@ -50,26 +60,73 @@ func getExecutableName(os string, arch string, pluginJSONPath string) (string, e
 		exname = exename
 	}
 
-	exeName := fmt.Sprintf("%s_%s_%s", exname, os, arch)
+	return asExecutableName(os, arch, pluginJSONPath), nil
+}
+
+// execNameCache is a cache for the executable name, so we don't have to read the plugin.json file multiple times.
+var executableNameCache sync.Map
+
+// getExecutableNameForPlugin returns the name of the executable from the plugin.json file in the provided directory.
+// The executable name is cached to avoid reading the same file multiple times.
+// If found, the executable name is returned in the format: <executable>_<os>_<arch>
+func getExecutableNameForPlugin(os string, arch string, pluginDir string) (string, error) {
+	if cached, ok := executableNameCache.Load(pluginDir); ok {
+		return asExecutableName(os, arch, cached.(string)), nil
+	}
+	exe, err := internal.GetStringValueFromJSON(path.Join(pluginDir, "plugin.json"), "executable")
+	if err != nil {
+		return "", err
+	}
+
+	executableNameCache.Store(pluginDir, exe)
+
+	return asExecutableName(os, arch, exe), nil
+}
+
+func asExecutableName(os string, arch string, exe string) string {
+	exeName := fmt.Sprintf("%s_%s_%s", exe, os, arch)
 	if os == "windows" {
 		exeName = fmt.Sprintf("%s.exe", exeName)
 	}
-	return exeName, nil
+	return exeName
 }
 
 func buildBackend(cfg Config) error {
-	cfg, err := beforeBuild(cfg)
+	cfg, args, err := getBuildBackendCmdInfo(cfg)
 	if err != nil {
 		return err
 	}
 
+	// TODO: Change to sh.RunWithV once available.
+	return sh.RunWith(cfg.Env, "go", args...)
+}
+
+func getBuildBackendCmdInfo(cfg Config) (Config, []string, error) {
+	cfg, err := beforeBuild(cfg)
+	if err != nil {
+		return cfg, []string{}, err
+	}
+
 	pluginJSONPath := defaultPluginJSONPath
 	if cfg.PluginJSONPath != "" {
 		pluginJSONPath = cfg.PluginJSONPath
 	}
-	exeName, err := getExecutableName(cfg.OS, cfg.Arch, pluginJSONPath)
+	exePath, err := getExecutableNameForPlugin(cfg.OS, cfg.Arch, pluginJSONPath)
 	if err != nil {
-		return err
+		// Look for a nested backend data source plugin
+		nestedPluginJSONPath := defaultNestedDataSourcePath
+		exe, err2 := getExecutableNameForPlugin(cfg.OS, cfg.Arch, filepath.Join(pluginJSONPath, nestedPluginJSONPath))
+		if err2 != nil {
+			// return the original error
+			return cfg, []string{}, err
+		}
+		// For backwards compatibility, if the executable is in the root directory, strip that information.
+		if strings.HasPrefix(exe, "../") {
+			exePath = exe[3:]
+		} else {
+			// Make sure the executable is in the relevant nested plugin directory.
+			exePath = filepath.Join(nestedPluginJSONPath, exe)
+		}
 	}
 
 	ldFlags := ""
@@ -92,7 +149,7 @@ func buildBackend(cfg Config) error {
 		outputPath = defaultOutputBinaryPath
 	}
 	args := []string{
-		"build", "-o", filepath.Join(outputPath, exeName),
+		"build", "-o", filepath.Join(outputPath, exePath),
 	}
 
 	info := getBuildInfoFromEnvironment()
@@ -114,8 +171,15 @@ func buildBackend(cfg Config) error {
 		}
 	}
 
-	for k, v := range flags {
-		ldFlags = fmt.Sprintf("%s -X '%s=%s'", ldFlags, k, v)
+	// Sort the flags to ensure a consistent build command
+	flagsKeys := make([]string, 0, len(flags))
+	for k := range flags {
+		flagsKeys = append(flagsKeys, k)
+	}
+	sort.Strings(flagsKeys)
+
+	for _, k := range flagsKeys {
+		ldFlags = fmt.Sprintf("%s -X '%s=%s'", ldFlags, k, flags[k])
 	}
 	args = append(args, "-ldflags", ldFlags)
 
@@ -133,9 +197,7 @@ func buildBackend(cfg Config) error {
 	if !cfg.EnableCGo {
 		cfg.Env["CGO_ENABLED"] = "0"
 	}
-
-	// TODO: Change to sh.RunWithV once available.
-	return sh.RunWith(cfg.Env, "go", args...)
+	return cfg, args, nil
 }
 
 func newBuildConfig(os string, arch string) Config {

build/common_test.go

@@ -0,0 +1,232 @@
+package build
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_getExecutableNameForPlugin(t *testing.T) {
+	rootDir := t.TempDir()
+	plugins := map[string]string{
+		"foo-datasource": "gpx_foo",
+		"bar-datasource": "gpx_bar",
+		"baz-datasource": "gpx_baz",
+	}
+
+	type args struct {
+		os        string
+		arch      string
+		pluginDir string
+	}
+	tcs := []struct {
+		name          string
+		args          args
+		expected      string
+		expectedCache map[string]string
+		wantErr       assert.ErrorAssertionFunc
+	}{
+		{
+			name: "Valid plugin with executable is found and cached",
+			args: args{
+				os:        "darwin",
+				arch:      "arm64",
+				pluginDir: filepath.Join(rootDir, "foo-datasource"),
+			},
+			expected: "gpx_foo_darwin_arm64",
+			expectedCache: map[string]string{
+				filepath.Join(rootDir, "foo-datasource"): "gpx_foo",
+			},
+			wantErr: assert.NoError,
+		},
+		{
+			name: "Another valid plugin with executable is found and cached",
+			args: args{
+				os:        "windows",
+				arch:      "amd64",
+				pluginDir: filepath.Join(rootDir, "baz-datasource"),
+			},
+			expected: "gpx_baz_windows_amd64.exe",
+			expectedCache: map[string]string{
+				filepath.Join(rootDir, "foo-datasource"): "gpx_foo",
+				filepath.Join(rootDir, "baz-datasource"): "gpx_baz",
+			},
+			wantErr: assert.NoError,
+		},
+		{
+			name: "Same plugin with executable is found in cache",
+			args: args{
+				os:        "windows",
+				arch:      "amd64",
+				pluginDir: filepath.Join(rootDir, "baz-datasource"),
+			},
+			expected: "gpx_baz_windows_amd64.exe",
+			expectedCache: map[string]string{
+				filepath.Join(rootDir, "foo-datasource"): "gpx_foo",
+				filepath.Join(rootDir, "baz-datasource"): "gpx_baz",
+			},
+			wantErr: assert.NoError,
+		},
+		{
+			name: "Non existing plugin returns an error",
+			args: args{
+				os:        "linux",
+				arch:      "amd64",
+				pluginDir: filepath.Join(rootDir, "foobarbaz-datasource"),
+			},
+			expected: "",
+			expectedCache: map[string]string{
+				filepath.Join(rootDir, "foo-datasource"): "gpx_foo",
+				filepath.Join(rootDir, "baz-datasource"): "gpx_baz",
+			},
+			wantErr: assert.Error,
+		},
+	}
+
+	for pluginID, executable := range plugins {
+		pluginRootDir := filepath.Join(rootDir, pluginID)
+		err := os.MkdirAll(pluginRootDir, os.ModePerm)
+		require.NoError(t, err)
+		f, err := os.Create(filepath.Join(pluginRootDir, "plugin.json"))
+		require.NoError(t, err)
+
+		_, err = f.WriteString(fmt.Sprintf(`{"executable": %q}`, executable))
+		require.NoError(t, err)
+		err = f.Close()
+		require.NoError(t, err)
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := getExecutableNameForPlugin(tc.args.os, tc.args.arch, tc.args.pluginDir)
+			if !tc.wantErr(t, err, fmt.Sprintf("getExecutableNameForPlugin(%v, %v, %v)", tc.args.os, tc.args.arch, tc.args.pluginDir)) {
+				return
+			}
+			assert.Equalf(t, tc.expected, got, "getExecutableNameForPlugin(%v, %v, %v)", tc.args.os, tc.args.arch, tc.args.pluginDir)
+
+			numCached := 0
+			executableNameCache.Range(func(_, _ any) bool {
+				numCached++
+				return true
+			})
+
+			assert.Equal(t, len(tc.expectedCache), numCached)
+			for s, s2 := range tc.expectedCache {
+				val, ok := executableNameCache.Load(s)
+				assert.True(t, ok)
+				assert.Equal(t, s2, val.(string))
+			}
+		})
+	}
+}
+
+func Test_getBuildBackendCmdInfo(t *testing.T) {
+	tmpDir := t.TempDir()
+	tcs := []struct {
+		name             string
+		pluginJSONCreate func(t *testing.T)
+		cfg              Config
+		expectedCfg      Config
+		expectedArgs     []string
+		wantErr          assert.ErrorAssertionFunc
+	}{
+		{
+			name: "Happy path",
+			cfg: Config{
+				OS:             "darwin",
+				Arch:           "arm64",
+				Env:            make(map[string]string),
+				PluginJSONPath: filepath.Join(tmpDir, "foobar-datasource"),
+			},
+			pluginJSONCreate: func(t *testing.T) {
+				t.Helper()
+				createPluginJSON(t, filepath.Join(tmpDir, "foobar-datasource"), "gpx_foo")
+			},
+			expectedCfg: Config{
+				OS:             "darwin",
+				Arch:           "arm64",
+				Env:            map[string]string{"CGO_ENABLED": "0", "GOARCH": "arm64", "GOOS": "darwin"},
+				PluginJSONPath: filepath.Join(tmpDir, "foobar-datasource"),
+			},
+			expectedArgs: []string{"build", "-o", filepath.Join(defaultOutputBinaryPath, "gpx_foo_darwin_arm64"), "-ldflags", "-w -s -extldflags \"-static\" -X 'github.com/grafana/grafana-plugin-sdk-go/build.buildInfoJSON={.*}' -X 'main.branch=.+' -X 'main.commit=[a-z0-9\\d]{40}'", "./pkg"},
+			wantErr:      assert.NoError,
+		},
+		{
+			name: "Happy path with nested datasource",
+			cfg: Config{
+				OS:             "darwin",
+				Arch:           "arm64",
+				Env:            make(map[string]string),
+				PluginJSONPath: filepath.Join(tmpDir, "foobar-app"),
+			},
+			pluginJSONCreate: func(t *testing.T) {
+				t.Helper()
+				createPluginJSON(t, filepath.Join(tmpDir, "foobar-app", defaultNestedDataSourcePath), "gpx_foo")
+			},
+			expectedCfg: Config{
+				OS:             "darwin",
+				Arch:           "arm64",
+				Env:            map[string]string{"CGO_ENABLED": "0", "GOARCH": "arm64", "GOOS": "darwin"},
+				PluginJSONPath: filepath.Join(tmpDir, "foobar-app"),
+			},
+			expectedArgs: []string{"build", "-o", filepath.Join(defaultOutputBinaryPath, defaultNestedDataSourcePath, "gpx_foo_darwin_arm64"), "-ldflags", "-w -s -extldflags \"-static\" -X 'github.com/grafana/grafana-plugin-sdk-go/build.buildInfoJSON={.*}' -X 'main.branch=.+' -X 'main.commit=[a-z0-9\\d]{40}'", "./pkg"},
+			wantErr:      assert.NoError,
+		},
+		{
+			name: "Happy path with nested datasource that has executable path in root directory",
+			cfg: Config{
+				OS:             "windows",
+				Arch:           "amd64",
+				Env:            make(map[string]string),
+				PluginJSONPath: filepath.Join(tmpDir, "foobarbaz-app"),
+			},
+			pluginJSONCreate: func(t *testing.T) {
+				t.Helper()
+				createPluginJSON(t, filepath.Join(tmpDir, "foobarbaz-app", defaultNestedDataSourcePath), "../gpx_foobarbaz")
+			},
+			expectedCfg: Config{
+				OS:             "windows",
+				Arch:           "amd64",
+				Env:            map[string]string{"CGO_ENABLED": "0", "GOARCH": "amd64", "GOOS": "windows"},
+				PluginJSONPath: filepath.Join(tmpDir, "foobarbaz-app"),
+			},
+			expectedArgs: []string{"build", "-o", filepath.Join(defaultOutputBinaryPath, "gpx_foobarbaz_windows_amd64.exe"), "-ldflags", "-w -s -extldflags \"-static\" -X 'github.com/grafana/grafana-plugin-sdk-go/build.buildInfoJSON={.*}' -X 'main.branch=.+' -X 'main.commit=[a-z0-9\\d]{40}'", "./pkg"},
+			wantErr:      assert.NoError,
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			tc.pluginJSONCreate(t)
+
+			cfg, args, err := getBuildBackendCmdInfo(tc.cfg)
+			if !tc.wantErr(t, err, fmt.Sprintf("getBuildBackendCmdInfo(%v)", tc.cfg)) {
+				return
+			}
+			assert.Equalf(t, tc.expectedCfg, cfg, "getBuildBackendCmdInfo(%v)", tc.cfg)
+
+			// check if expected build arg regex matches against actual build arg
+			buildArg := strings.Join(args, " ")
+			expectedBuildArg := strings.Join(tc.expectedArgs, " ")
+			assert.Regexp(t, expectedBuildArg, buildArg, "getBuildBackendCmdInfo(%v)", tc.cfg)
+		})
+	}
+}
+
+func createPluginJSON(t *testing.T, pluginDir string, executable string) {
+	t.Helper()
+	err := os.MkdirAll(pluginDir, os.ModePerm)
+	require.NoError(t, err)
+	f, err := os.Create(filepath.Join(pluginDir, "plugin.json"))
+	require.NoError(t, err)
+
+	_, err = f.WriteString(fmt.Sprintf(`{"executable": %q}`, executable))
+	require.NoError(t, err)
+	err = f.Close()
+	require.NoError(t, err)
+}