Encore Go Patcher

This commit creates a simple shell script which will patch fresh copies
of the Go runtime with the Encore overlays and patches in a reproducable
way.

It does this by moving the Go project into a submodule pointed at Google's
Git hosting, rather than being a copy of the target branch (which this repo
has been to date)

.gitattributes

@@ -0,0 +1,16 @@
+# Treat all files in the Go repo as binary, with no git magic updating
+# line endings. This produces predictable results in different environments.
+#
+# Windows users contributing to Go will need to use a modern version
+# of git and editors capable of LF line endings.
+#
+# Windows .bat files are known to have multiple bugs when run with LF
+# endings, and so they are checked in with CRLF endings, with a test
+# in test/winbatch.go to catch problems. (See golang.org/issue/37791.)
+#
+# We'll prevent accidental CRLF line endings from entering the repo
+# via the git-codereview gofmt checks and tests.
+#
+# See golang.org/issue/9281.
+
+* -text

.github/workflows/release.yml

@@ -0,0 +1,51 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Go version to build ("1.17")'
+        required: true
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        include:
+          - builder: ubuntu-latest
+            goos: linux
+            goarch: amd64
+          - builder: macos-latest
+            goos: darwin
+            goarch: amd64
+          - builder: macos-latest
+            goos: darwin
+            goarch: arm64
+          - builder: windows-latest
+            goos: windows
+            goarch: amd64
+
+    runs-on: ${{ matrix.builder }}
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+      - name: 'Create patched runtime'
+        run: bash ./apply_patch.bash "${{ github.event.inputs.version }}"
+        env:
+          GO111MODULE: "on"
+      - name: Build
+        run: go run . -dst=dist -goos=${{ matrix.goos }} -goarch=${{ matrix.goarch }}
+        env:
+          GO111MODULE: "on"
+      - name: 'Tar artifacts'
+        run: tar -czvf encore-go-${{ github.event.inputs.version }}-${{ matrix.goos }}_${{ matrix.goarch }}.tar.gz -C dist/${{ matrix.goos }}_${{ matrix.goarch }} .
+      - name: Publish artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: encore-go-${{ github.event.inputs.version }}-${{ matrix.goos }}_${{ matrix.goarch }}
+          path: encore-go-${{ github.event.inputs.version }}-${{ matrix.goos }}_${{ matrix.goarch }}.tar.gz

.gitignore

@@ -0,0 +1,54 @@
+.DS_Store
+*.[56789ao]
+*.a[56789o]
+*.so
+*.pyc
+._*
+.nfs.*
+[56789a].out
+*~
+*.orig
+*.rej
+*.exe
+.*.swp
+core
+*.cgo*.go
+*.cgo*.c
+_cgo_*
+_obj
+_test
+_testmain.go
+
+/VERSION.cache
+/bin/
+/build.out
+/doc/articles/wiki/*.bin
+/goinstall.log
+/last-change
+/misc/cgo/life/run.out
+/misc/cgo/stdio/run.out
+/misc/cgo/testso/main
+/pkg/
+/src/*.*/
+/src/cmd/cgo/zdefaultcc.go
+/src/cmd/dist/dist
+/src/cmd/go/internal/cfg/zdefaultcc.go
+/src/cmd/go/internal/cfg/zosarch.go
+/src/cmd/internal/objabi/zbootstrap.go
+/src/go/build/zcgo.go
+/src/go/doc/headscan
+/src/internal/buildcfg/zbootstrap.go
+/src/runtime/internal/sys/zversion.go
+/src/unicode/maketables
+/test.out
+/test/garbage/*.out
+/test/pass.out
+/test/run.out
+/test/times.out
+
+/dist
+
+# This file includes artifacts of Go build that should not be checked in.
+# For files created by specific development environment (e.g. editor),
+# use alternative ways to exclude files from git.
+# For example, set up .git/info/exclude or use a global .gitignore.

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "go"]
+	path = go
+	url = https://go.googlesource.com/go

README.md

@@ -0,0 +1,30 @@
+<div align="center">
+  <a href="https://encore.dev" alt="encore"><img width="189px" src="https://encore.dev/assets/img/logo.svg"></a>
+  <h3><a href="https://encore.dev">Encore – The Backend Development Engine</a></h3>
+</div>
+
+# Encore Rolling Go Fork
+
+This is [Encore's](https://encore.dev) rolling fork of Go with added automatic instrumentation for local development.
+
+This branch contains the raw patches, which allow us to re-apply them ontop of new Go releases.
+
+This system produces [reproducible builds](https://reproducible-builds.org/) of the patched Go runtime. This means you can clone this repository, run the commands listed below and reproduce an identical go binary that we ship with our tooling.
+
+## How to Use
+
+1. Checkout this repository and then initial the Git submodule; `git submodule init && git submodule update`
+2. Create a fresh patched version of the Go runtime, passing in the Go version you want;  `./apply_patch.bash 1.17`
+   (Note: instead of a Go version number, you can pass in `master` to build against the latest Go development commit)
+3. Run our build script using; `go run . --goos "darwin" --goarch "arm64"`
+   (replacing the OS and arch parameters to match your requirement)
+4. Verify your go was build; `./dist/darwin_arm64/encore-go/bin/go version`
+    The output you see should look like this, looking for the `encore-go` string;
+   `go version encore-go1.17.6 encore-go1.17-4d15582aff Thu Jan 6 19:06:43 2022 +0000 darwin/arm64`
+
+## Directory Structure
+
+This branch is broken up into three main folders;
+- `overlay`; this folder contains brand-new Encore specific files which should be copied into the `src` path of a fresh Go Release
+- `patches`; this folder contains patch files to modify the existing Go source code
+- `go`; a submodule checkout of https://go.googlesource.com/go which we apply the above two folders against 

apply_patch.bash

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# NOTE: This script creates a fresh version of a Encore patched Go runtime, however we are aiming to create
+#       reproducible builds, as such all copies and file writes are given fixed timestamps (rather than the system time
+#       when thie script was run). This allows the Git commit at the end of the process to have a deterministic hash,
+#       thus when we build the Go binary it will produce an identical binary.
+
+die () {
+    echo >&2 "$@"
+    exit 1
+}
+
+_(){ eval "$@" 2>&1 | sed "s/^/    /" ; return "${PIPESTATUS[0]}" ;}
+
+[ "$#" -eq 1 ] || die "Usage: $0 [go release]
+
+Patches the Go runtime with Encore tracing code.
+
+Examples:
+========
+A Go release: $0 1.17
+Nightly release: $0 master"
+
+set -e
+
+# Parameters
+GO_VERSION="$1"
+RELEASE_BRANCH="$1"
+if [ "$GO_VERSION" != "master" ]; then
+  RELEASE_BRANCH="release-branch.go$GO_VERSION"
+fi
+
+# Start working in the Go submodule directory
+pushd "$(dirname -- "$0")/go" > /dev/null
+
+# Checkout an updated clean copy of the Go runtime from the $RELEASE_BRANCH
+echo "♻️  Checking out a clean copy of the Go runtime from $RELEASE_BRANCH..."
+  _ git fetch
+
+  _ git checkout -f "$RELEASE_BRANCH"         # Checkout the branch to track it
+  _ git reset --hard origin/"$RELEASE_BRANCH" # Reset against the current head of that branch (ignoring any local commits)
+echo
+
+LAST_COMMIT_TIME=$(git log -1 --format=%cd)
+
+# Copy our overlay in and then apply the patches
+echo "🏗️  Applying Encore changes to the Go runtime..."
+  if [ -f ./VERSION ]; then
+    rm ./VERSION
+  fi
+
+  _ git apply --3way ../patches/*.diff
+  _ cp -p -P -v -R ../overlay/* ./
+echo
+
+echo "🤖  Committing runtime changes..."
+  _ git add .
+
+  # Note; we set all the details of the commit to git hash deterministic
+  GIT_COMMITTER_NAME='Encore Patcher' \
+  GIT_COMMITTER_EMAIL='[email protected]' \
+  GIT_COMMITTER_DATE="$LAST_COMMIT_TIME" \
+  git commit --allow-empty --date="$LAST_COMMIT_TIME" \
+      --author='Encore Patcher <[email protected]>' \
+      -m 'Applied Encore.dev instrumentation changes to the Go runtime' 2>&1 | sed "s/^/    /"
+echo
+
+# Restore the working directory back
+popd > /dev/null
+
+echo "✅  Done"

builder/encore-build.go

@@ -0,0 +1,165 @@
+package builder
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+)
+
+type Builder struct {
+	GOOS        string
+	GOARCH      string
+	goroot      string
+	gorootFinal string
+	dst         string
+	crossBuild  bool
+}
+
+func (b *Builder) PrepareWorkdir() error {
+	if err := os.RemoveAll(b.dst); err != nil {
+		return err
+	} else if err := os.MkdirAll(b.dst, 0755); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (b *Builder) Build() error {
+	var cmd *exec.Cmd
+	switch b.GOOS {
+	case "windows":
+		cmd = exec.Command(".\\make.bat")
+	default:
+		cmd = exec.Command("bash", "./make.bash")
+	}
+	cmd.Dir = join(b.goroot, "src")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = append(os.Environ(),
+		"GOROOT_FINAL=/encore",
+		"GOARCH="+b.GOARCH,
+		"GOOS="+b.GOOS)
+	return cmd.Run()
+}
+
+func (b *Builder) CopyOutput() error {
+	key := b.GOOS + "_" + b.GOARCH
+	filesToCopy := []string{
+		join("pkg", "include"),
+		join("pkg", key),
+		"lib",
+		"src",
+		"LICENSE",
+	}
+
+	// Cross-compilation puts binaries under bin/goos_goarch instead.
+	if b.crossBuild {
+		// Copy go binary from bin/goos_goarch to bin/
+		src := join(b.goroot, "bin", key, "go")
+		dst := join(b.dst, "bin", "go")
+		if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+			return err
+		}
+		if out, err := exec.Command("cp", src, dst).CombinedOutput(); err != nil {
+			return fmt.Errorf("copy go: %s", out)
+		}
+	} else {
+		filesToCopy = append(filesToCopy, join("bin", "go"+exe))
+	}
+
+	filesToCopy = append(filesToCopy, all(join("pkg", "tool", key),
+		"addr2line"+exe, "asm"+exe, "buildid"+exe, "cgo"+exe, "compile"+exe,
+		"link"+exe, "pack"+exe, "test2json"+exe, "vet"+exe,
+	)...)
+
+	for _, c := range filesToCopy {
+		src := join(b.goroot, c)
+		dst := join(b.dst, c)
+		if _, err := os.Stat(src); err != nil {
+			return fmt.Errorf("copy %s: %v", c, err)
+		}
+		if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+			return err
+		}
+		if out, err := exec.Command("cp", "-r", src, dst).CombinedOutput(); err != nil {
+			return fmt.Errorf("copy %s: %s", c, out)
+		}
+	}
+
+	return nil
+}
+
+func (b *Builder) CleanOutput() error {
+	key := b.GOOS + "_" + b.GOARCH
+	rm := []string{
+		join("pkg", key, "cmd"),
+	}
+
+	for _, r := range rm {
+		dst := join(b.dst, r)
+		if _, err := os.Stat(dst); err == nil {
+			if err := os.RemoveAll(dst); err != nil {
+				return fmt.Errorf("clean %s: %v", r, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func join(strs ...string) string {
+	return filepath.Join(strs...)
+}
+
+func all(src string, all ...string) []string {
+	var res []string
+	for _, a := range all {
+		res = append(res, join(src, a))
+	}
+	return res
+}
+
+func BuildEncoreGo(goos, goarch, root, dst string) error {
+	if _, err := os.Stat(filepath.Join(root, "go", "src", "make.bash")); err != nil {
+		return fmt.Errorf("unexpected location for build script, expected in encore-go root")
+	}
+
+	if err := os.Chdir(root); err != nil {
+		return err
+	}
+
+	dst, err := filepath.Abs(dst)
+	if err != nil {
+		return err
+	}
+
+	if goos == "windows" {
+		exe = ".exe"
+	}
+
+	b := &Builder{
+		GOOS:       goos,
+		GOARCH:     goarch,
+		goroot:     join(root, "go"),
+		dst:        join(dst, goos+"_"+goarch, "encore-go"),
+		crossBuild: runtime.GOOS != goos || runtime.GOARCH != goarch,
+	}
+
+	for _, f := range []func() error{
+		b.PrepareWorkdir,
+		b.Build,
+		b.CopyOutput,
+		b.CleanOutput,
+	} {
+		if err := f(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// exe suffix
+var exe string