diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index 50118c8dc9ff..1601b5659d33 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -67,13 +67,13 @@ func restoreRepoLayout(fromDir string, dirEntries []string, scratchDirName strin // addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file. func addVersionToMod(version string) bool { - cmd := exec.Command("go", "mod", "edit", "-go="+version) + cmd := toolchain.GoCommand("mod", "edit", "-go="+version) return util.RunCmd(cmd) } // checkVendor tests to see whether a vendor directory is inconsistent according to the go frontend func checkVendor() bool { - vendorCheckCmd := exec.Command("go", "list", "-mod=vendor", "./...") + vendorCheckCmd := toolchain.GoCommand("list", "-mod=vendor", "./...") outp, err := vendorCheckCmd.CombinedOutput() if err != nil { badVendorRe := regexp.MustCompile(`(?m)^go: inconsistent vendoring in .*:$`) @@ -438,7 +438,7 @@ func installDependencies(workspace project.GoWorkspace) { util.RunCmd(vendor) } - install = exec.Command("go", "get", "-v", "./...") + install = toolchain.GoCommand("get", "-v", "./...") install.Dir = path log.Printf("Installing dependencies using `go get -v ./...` in `%s`.\n", path) util.RunCmd(install) diff --git a/go/extractor/toolchain/toolchain.go b/go/extractor/toolchain/toolchain.go index 8abe93ba0963..01b3ab813bd5 100644 --- a/go/extractor/toolchain/toolchain.go +++ b/go/extractor/toolchain/toolchain.go @@ -137,9 +137,16 @@ func SupportsWorkspaces() bool { return GetEnvGoSemVer().IsAtLeast(V1_18) } +// Constructs a `*exec.Cmd` for `go` with the specified arguments. +func GoCommand(arg ...string) *exec.Cmd { + cmd := exec.Command("go", arg...) + util.ApplyProxyEnvVars(cmd) + return cmd +} + // Run `go mod tidy -e` in the directory given by `path`. func TidyModule(path string) *exec.Cmd { - cmd := exec.Command("go", "mod", "tidy", "-e") + cmd := GoCommand("mod", "tidy", "-e") cmd.Dir = path return cmd } @@ -159,21 +166,21 @@ func InitModule(path string) *exec.Cmd { } } - modInit := exec.Command("go", "mod", "init", moduleName) + modInit := GoCommand("mod", "init", moduleName) modInit.Dir = path return modInit } // Constructs a command to run `go mod vendor -e` in the directory given by `path`. func VendorModule(path string) *exec.Cmd { - modVendor := exec.Command("go", "mod", "vendor", "-e") + modVendor := GoCommand("mod", "vendor", "-e") modVendor.Dir = path return modVendor } // Constructs a command to run `go version`. func Version() *exec.Cmd { - version := exec.Command("go", "version") + version := GoCommand("version") return version } @@ -209,7 +216,7 @@ func RunListWithEnv(format string, patterns []string, additionalEnv []string, fl func ListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) *exec.Cmd { args := append([]string{"list", "-e", "-f", format}, flags...) args = append(args, patterns...) - cmd := exec.Command("go", args...) + cmd := GoCommand(args...) cmd.Env = append(os.Environ(), additionalEnv...) return cmd } diff --git a/go/extractor/util/BUILD.bazel b/go/extractor/util/BUILD.bazel index b7a7783aa799..dd294a63a9c1 100644 --- a/go/extractor/util/BUILD.bazel +++ b/go/extractor/util/BUILD.bazel @@ -6,6 +6,7 @@ go_library( name = "util", srcs = [ "extractvendordirs.go", + "registryproxy.go", "semver.go", "util.go", ], @@ -17,6 +18,7 @@ go_library( go_test( name = "util_test", srcs = [ + "registryproxy_test.go", "semver_test.go", "util_test.go", ], diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go new file mode 100644 index 000000000000..43eaa461032a --- /dev/null +++ b/go/extractor/util/registryproxy.go @@ -0,0 +1,134 @@ +package util + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "os/exec" + "strings" +) + +const PROXY_HOST = "CODEQL_PROXY_HOST" +const PROXY_PORT = "CODEQL_PROXY_PORT" +const PROXY_CA_CERTIFICATE = "CODEQL_PROXY_CA_CERTIFICATE" +const PROXY_URLS = "CODEQL_PROXY_URLS" +const GOPROXY_SERVER = "goproxy_server" + +type RegistryConfig struct { + Type string `json:"type"` + URL string `json:"url"` +} + +// The address of the proxy including protocol and port (e.g. http://localhost:1234) +var proxy_address string + +// The path to the temporary file that stores the proxy certificate, if any. +var proxy_cert_file string + +// An array of registry configurations that are relevant to Go. +// This excludes other registry configurations that may be available, but are not relevant to Go. +var proxy_configs []RegistryConfig + +// Stores the environment variables that we wish to pass on to `go` commands. +var proxy_vars []string = nil + +// Keeps track of whether we have inspected the proxy environment variables. +// Needed since `proxy_vars` may be nil either way. +var proxy_vars_checked bool = false + +// Tries to parse the given string value into an array of RegistryConfig values. +func parseRegistryConfigs(str string) ([]RegistryConfig, error) { + var configs []RegistryConfig + if err := json.Unmarshal([]byte(str), &configs); err != nil { + return nil, err + } + + return configs, nil +} + +func getEnvVars() []string { + var result []string + + if proxy_host, proxy_host_set := os.LookupEnv(PROXY_HOST); proxy_host_set { + if proxy_port, proxy_port_set := os.LookupEnv(PROXY_PORT); proxy_port_set { + proxy_address = fmt.Sprintf("http://%s:%s", proxy_host, proxy_port) + result = append(result, fmt.Sprintf("HTTP_PROXY=%s", proxy_address), fmt.Sprintf("HTTPS_PROXY=%s", proxy_address)) + + slog.Info("Found private registry proxy", slog.String("proxy_address", proxy_address)) + } + } + + if proxy_cert, proxy_cert_set := os.LookupEnv(PROXY_CA_CERTIFICATE); proxy_cert_set { + // Write the certificate to a temporary file + slog.Info("Found certificate") + + f, err := os.CreateTemp("", "codeql-proxy.crt") + if err != nil { + slog.Error("Failed to create temporary file for the proxy certificate", slog.String("error", err.Error())) + } + + _, err = f.WriteString(proxy_cert) + if err != nil { + slog.Error("Failed to write to the temporary certificate file", slog.String("error", err.Error())) + } + + err = f.Close() + if err != nil { + slog.Error("Failed to close the temporary certificate file", slog.String("error", err.Error())) + } else { + proxy_cert_file = f.Name() + result = append(result, fmt.Sprintf("SSL_CERT_FILE=%s", proxy_cert_file)) + } + } + + if proxy_urls, proxy_urls_set := os.LookupEnv(PROXY_URLS); proxy_urls_set { + val, err := parseRegistryConfigs(proxy_urls) + if err != nil { + slog.Error("Unable to parse proxy configurations", slog.String("error", err.Error())) + } else { + // We only care about private registry configurations that are relevant to Go and + // filter others out at this point. + for _, cfg := range val { + if cfg.Type == GOPROXY_SERVER { + proxy_configs = append(proxy_configs, cfg) + slog.Info("Found GOPROXY server", slog.String("url", cfg.URL)) + } + } + + if len(proxy_configs) > 0 { + goproxy_val := "https://proxy.golang.org,direct" + + for _, cfg := range proxy_configs { + goproxy_val = cfg.URL + "," + goproxy_val + } + + result = append(result, fmt.Sprintf("GOPROXY=%s", goproxy_val), "GOPRIVATE=", "GONOPROXY=") + } + } + } + + return result +} + +// Applies private package proxy related environment variables to `cmd`. +func ApplyProxyEnvVars(cmd *exec.Cmd) { + slog.Debug( + "Applying private registry proxy environment variables", + slog.String("cmd_args", strings.Join(cmd.Args, " ")), + ) + + // If we haven't done so yet, check whether the proxy environment variables are set + // and extract information from them. + if !proxy_vars_checked { + proxy_vars = getEnvVars() + proxy_vars_checked = true + } + + // If the proxy is configured, `proxy_vars` will be not `nil`. We append those + // variables to the existing environment to preserve those environment variables. + // If `cmd.Env` is not changed, then the existing environment is also preserved. + if proxy_vars != nil { + cmd.Env = append(os.Environ(), proxy_vars...) + } +} diff --git a/go/extractor/util/registryproxy_test.go b/go/extractor/util/registryproxy_test.go new file mode 100644 index 000000000000..a21b1a215c11 --- /dev/null +++ b/go/extractor/util/registryproxy_test.go @@ -0,0 +1,49 @@ +package util + +import ( + "testing" +) + +func parseRegistryConfigsFail(t *testing.T, str string) { + _, err := parseRegistryConfigs(str) + + if err == nil { + t.Fatal("Expected `parseRegistryConfigs` to fail, but it succeeded.") + } +} + +func parseRegistryConfigsSuccess(t *testing.T, str string) []RegistryConfig { + val, err := parseRegistryConfigs(str) + + if err != nil { + t.Fatalf("Expected `parseRegistryConfigs` to succeed, but it failed: %s", err.Error()) + } + + return val +} + +func TestParseRegistryConfigs(t *testing.T) { + parseRegistryConfigsFail(t, "") + + empty := parseRegistryConfigsSuccess(t, "[]") + + if len(empty) != 0 { + t.Fatal("Expected `parseRegistryConfigs(\"[]\")` to return no configurations, but got some.") + } + + single := parseRegistryConfigsSuccess(t, "[{ \"type\": \"goproxy_server\", \"url\": \"https://proxy.example.com/mod\" }]") + + if len(single) != 1 { + t.Fatalf("Expected `parseRegistryConfigs` to return one configuration, but got %d.", len(single)) + } + + first := single[0] + + if first.Type != "goproxy_server" { + t.Fatalf("Expected `Type` to be `goproxy_server`, but got `%s`", first.Type) + } + + if first.URL != "https://proxy.example.com/mod" { + t.Fatalf("Expected `URL` to be `https://proxy.example.com/mod`, but got `%s`", first.URL) + } +}