From 8793b4e958fa79f88cd4b7715157c599f915d1eb Mon Sep 17 00:00:00 2001 From: Dominic Black Date: Mon, 24 Jan 2022 13:42:05 +0000 Subject: [PATCH] Encore Go Patcher This commit creates a simple shell script which will patch fresh copies of the Go runtime with the Encore overlays and patches in a reproducable way. It does this by moving the Go project into a submodule pointed at Google's Git hosting, rather than being a copy of the target branch (which this repo has been to date) --- .gitattributes | 16 ++ .github/workflows/release.yml | 51 +++++ .gitignore | 54 +++++ .gitmodules | 3 + README.md | 30 +++ apply_patch.bash | 71 +++++++ builder/encore-build.go | 165 ++++++++++++++++ go | 1 + go.mod | 3 + main.go | 28 +++ overlay/src/net/http/encore.s | 0 overlay/src/net/http/encore_app.go | 15 ++ overlay/src/net/http/encore_noapp.go | 14 ++ overlay/src/runtime/encore.go | 281 +++++++++++++++++++++++++++ overlay/src/runtime/encore_app.go | 9 + overlay/src/runtime/encore_noapp.go | 9 + patches/net_patch.diff | 32 +++ patches/runtime_patch.diff | 76 ++++++++ patches/version_patch.diff | 23 +++ 19 files changed, 881 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 README.md create mode 100755 apply_patch.bash create mode 100644 builder/encore-build.go create mode 160000 go create mode 100644 go.mod create mode 100644 main.go create mode 100644 overlay/src/net/http/encore.s create mode 100644 overlay/src/net/http/encore_app.go create mode 100644 overlay/src/net/http/encore_noapp.go create mode 100644 overlay/src/runtime/encore.go create mode 100644 overlay/src/runtime/encore_app.go create mode 100644 overlay/src/runtime/encore_noapp.go create mode 100644 patches/net_patch.diff create mode 100644 patches/runtime_patch.diff create mode 100644 patches/version_patch.diff diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..f1ca298ec5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,16 @@ +# Treat all files in the Go repo as binary, with no git magic updating +# line endings. This produces predictable results in different environments. +# +# Windows users contributing to Go will need to use a modern version +# of git and editors capable of LF line endings. +# +# Windows .bat files are known to have multiple bugs when run with LF +# endings, and so they are checked in with CRLF endings, with a test +# in test/winbatch.go to catch problems. (See golang.org/issue/37791.) +# +# We'll prevent accidental CRLF line endings from entering the repo +# via the git-codereview gofmt checks and tests. +# +# See golang.org/issue/9281. + +* -text \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..6586a84f1c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,51 @@ +name: Release + +on: + workflow_dispatch: + inputs: + version: + description: 'Go version to build ("1.17")' + required: true + +jobs: + build: + strategy: + matrix: + include: + - builder: ubuntu-latest + goos: linux + goarch: amd64 + - builder: macos-latest + goos: darwin + goarch: amd64 + - builder: macos-latest + goos: darwin + goarch: arm64 + - builder: windows-latest + goos: windows + goarch: amd64 + + runs-on: ${{ matrix.builder }} + steps: + - name: Check out repo + uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 + - name: 'Create patched runtime' + run: bash ./apply_patch.bash "${{ github.event.inputs.version }}" + env: + GO111MODULE: "on" + - name: Build + run: go run . -dst=dist -goos=${{ matrix.goos }} -goarch=${{ matrix.goarch }} + env: + GO111MODULE: "on" + - name: 'Tar artifacts' + run: tar -czvf encore-go-${{ github.event.inputs.version }}-${{ matrix.goos }}_${{ matrix.goarch }}.tar.gz -C dist/${{ matrix.goos }}_${{ matrix.goarch }} . + - name: Publish artifact + uses: actions/upload-artifact@v2 + with: + name: encore-go-${{ github.event.inputs.version }}-${{ matrix.goos }}_${{ matrix.goarch }} + path: encore-go-${{ github.event.inputs.version }}-${{ matrix.goos }}_${{ matrix.goarch }}.tar.gz diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..b2d0dba0c6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,54 @@ +.DS_Store +*.[56789ao] +*.a[56789o] +*.so +*.pyc +._* +.nfs.* +[56789a].out +*~ +*.orig +*.rej +*.exe +.*.swp +core +*.cgo*.go +*.cgo*.c +_cgo_* +_obj +_test +_testmain.go + +/VERSION.cache +/bin/ +/build.out +/doc/articles/wiki/*.bin +/goinstall.log +/last-change +/misc/cgo/life/run.out +/misc/cgo/stdio/run.out +/misc/cgo/testso/main +/pkg/ +/src/*.*/ +/src/cmd/cgo/zdefaultcc.go +/src/cmd/dist/dist +/src/cmd/go/internal/cfg/zdefaultcc.go +/src/cmd/go/internal/cfg/zosarch.go +/src/cmd/internal/objabi/zbootstrap.go +/src/go/build/zcgo.go +/src/go/doc/headscan +/src/internal/buildcfg/zbootstrap.go +/src/runtime/internal/sys/zversion.go +/src/unicode/maketables +/test.out +/test/garbage/*.out +/test/pass.out +/test/run.out +/test/times.out + +/dist + +# This file includes artifacts of Go build that should not be checked in. +# For files created by specific development environment (e.g. editor), +# use alternative ways to exclude files from git. +# For example, set up .git/info/exclude or use a global .gitignore. diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..ceab419305 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "go"] + path = go + url = https://go.googlesource.com/go diff --git a/README.md b/README.md new file mode 100644 index 0000000000..9439a51097 --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ +
+ +

Encore – The Backend Development Engine

+
+ +# Encore Rolling Go Fork + +This is [Encore's](https://encore.dev) rolling fork of Go with added automatic instrumentation for local development. + +This branch contains the raw patches, which allow us to re-apply them ontop of new Go releases. + +This system produces [reproducible builds](https://reproducible-builds.org/) of the patched Go runtime. This means you can clone this repository, run the commands listed below and reproduce an identical go binary that we ship with our tooling. + +## How to Use + +1. Checkout this repository and then initial the Git submodule; `git submodule init && git submodule update` +2. Create a fresh patched version of the Go runtime, passing in the Go version you want; `./apply_patch.bash 1.17` + (Note: instead of a Go version number, you can pass in `master` to build against the latest Go development commit) +3. Run our build script using; `go run . --goos "darwin" --goarch "arm64"` + (replacing the OS and arch parameters to match your requirement) +4. Verify your go was build; `./dist/darwin_arm64/encore-go/bin/go version` + The output you see should look like this, looking for the `encore-go` string; + `go version encore-go1.17.6 encore-go1.17-4d15582aff Thu Jan 6 19:06:43 2022 +0000 darwin/arm64` + +## Directory Structure + +This branch is broken up into three main folders; +- `overlay`; this folder contains brand-new Encore specific files which should be copied into the `src` path of a fresh Go Release +- `patches`; this folder contains patch files to modify the existing Go source code +- `go`; a submodule checkout of https://go.googlesource.com/go which we apply the above two folders against diff --git a/apply_patch.bash b/apply_patch.bash new file mode 100755 index 0000000000..e5ed0824a9 --- /dev/null +++ b/apply_patch.bash @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# NOTE: This script creates a fresh version of a Encore patched Go runtime, however we are aiming to create +# reproducible builds, as such all copies and file writes are given fixed timestamps (rather than the system time +# when thie script was run). This allows the Git commit at the end of the process to have a deterministic hash, +# thus when we build the Go binary it will produce an identical binary. + +die () { + echo >&2 "$@" + exit 1 +} + +_(){ eval "$@" 2>&1 | sed "s/^/ /" ; return "${PIPESTATUS[0]}" ;} + +[ "$#" -eq 1 ] || die "Usage: $0 [go release] + +Patches the Go runtime with Encore tracing code. + +Examples: +======== +A Go release: $0 1.17 +Nightly release: $0 master" + +set -e + +# Parameters +GO_VERSION="$1" +RELEASE_BRANCH="$1" +if [ "$GO_VERSION" != "master" ]; then + RELEASE_BRANCH="release-branch.go$GO_VERSION" +fi + +# Start working in the Go submodule directory +pushd "$(dirname -- "$0")/go" > /dev/null + +# Checkout an updated clean copy of the Go runtime from the $RELEASE_BRANCH +echo "♻️ Checking out a clean copy of the Go runtime from $RELEASE_BRANCH..." + _ git fetch + + _ git checkout -f "$RELEASE_BRANCH" # Checkout the branch to track it + _ git reset --hard origin/"$RELEASE_BRANCH" # Reset against the current head of that branch (ignoring any local commits) +echo + +LAST_COMMIT_TIME=$(git log -1 --format=%cd) + +# Copy our overlay in and then apply the patches +echo "🏗️ Applying Encore changes to the Go runtime..." + if [ -f ./VERSION ]; then + rm ./VERSION + fi + + _ git apply --3way ../patches/*.diff + _ cp -p -P -v -R ../overlay/* ./ +echo + +echo "🤖 Committing runtime changes..." + _ git add . + + # Note; we set all the details of the commit to git hash deterministic + GIT_COMMITTER_NAME='Encore Patcher' \ + GIT_COMMITTER_EMAIL='noreply@encore.dev' \ + GIT_COMMITTER_DATE="$LAST_COMMIT_TIME" \ + git commit --allow-empty --date="$LAST_COMMIT_TIME" \ + --author='Encore Patcher ' \ + -m 'Applied Encore.dev instrumentation changes to the Go runtime' 2>&1 | sed "s/^/ /" +echo + +# Restore the working directory back +popd > /dev/null + +echo "✅ Done" diff --git a/builder/encore-build.go b/builder/encore-build.go new file mode 100644 index 0000000000..f2a6ffe499 --- /dev/null +++ b/builder/encore-build.go @@ -0,0 +1,165 @@ +package builder + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" +) + +type Builder struct { + GOOS string + GOARCH string + goroot string + gorootFinal string + dst string + crossBuild bool +} + +func (b *Builder) PrepareWorkdir() error { + if err := os.RemoveAll(b.dst); err != nil { + return err + } else if err := os.MkdirAll(b.dst, 0755); err != nil { + return err + } + return nil +} + +func (b *Builder) Build() error { + var cmd *exec.Cmd + switch b.GOOS { + case "windows": + cmd = exec.Command(".\\make.bat") + default: + cmd = exec.Command("bash", "./make.bash") + } + cmd.Dir = join(b.goroot, "src") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), + "GOROOT_FINAL=/encore", + "GOARCH="+b.GOARCH, + "GOOS="+b.GOOS) + return cmd.Run() +} + +func (b *Builder) CopyOutput() error { + key := b.GOOS + "_" + b.GOARCH + filesToCopy := []string{ + join("pkg", "include"), + join("pkg", key), + "lib", + "src", + "LICENSE", + } + + // Cross-compilation puts binaries under bin/goos_goarch instead. + if b.crossBuild { + // Copy go binary from bin/goos_goarch to bin/ + src := join(b.goroot, "bin", key, "go") + dst := join(b.dst, "bin", "go") + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + if out, err := exec.Command("cp", src, dst).CombinedOutput(); err != nil { + return fmt.Errorf("copy go: %s", out) + } + } else { + filesToCopy = append(filesToCopy, join("bin", "go"+exe)) + } + + filesToCopy = append(filesToCopy, all(join("pkg", "tool", key), + "addr2line"+exe, "asm"+exe, "buildid"+exe, "cgo"+exe, "compile"+exe, + "link"+exe, "pack"+exe, "test2json"+exe, "vet"+exe, + )...) + + for _, c := range filesToCopy { + src := join(b.goroot, c) + dst := join(b.dst, c) + if _, err := os.Stat(src); err != nil { + return fmt.Errorf("copy %s: %v", c, err) + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + if out, err := exec.Command("cp", "-r", src, dst).CombinedOutput(); err != nil { + return fmt.Errorf("copy %s: %s", c, out) + } + } + + return nil +} + +func (b *Builder) CleanOutput() error { + key := b.GOOS + "_" + b.GOARCH + rm := []string{ + join("pkg", key, "cmd"), + } + + for _, r := range rm { + dst := join(b.dst, r) + if _, err := os.Stat(dst); err == nil { + if err := os.RemoveAll(dst); err != nil { + return fmt.Errorf("clean %s: %v", r, err) + } + } + } + + return nil +} + +func join(strs ...string) string { + return filepath.Join(strs...) +} + +func all(src string, all ...string) []string { + var res []string + for _, a := range all { + res = append(res, join(src, a)) + } + return res +} + +func BuildEncoreGo(goos, goarch, root, dst string) error { + if _, err := os.Stat(filepath.Join(root, "go", "src", "make.bash")); err != nil { + return fmt.Errorf("unexpected location for build script, expected in encore-go root") + } + + if err := os.Chdir(root); err != nil { + return err + } + + dst, err := filepath.Abs(dst) + if err != nil { + return err + } + + if goos == "windows" { + exe = ".exe" + } + + b := &Builder{ + GOOS: goos, + GOARCH: goarch, + goroot: join(root, "go"), + dst: join(dst, goos+"_"+goarch, "encore-go"), + crossBuild: runtime.GOOS != goos || runtime.GOARCH != goarch, + } + + for _, f := range []func() error{ + b.PrepareWorkdir, + b.Build, + b.CopyOutput, + b.CleanOutput, + } { + if err := f(); err != nil { + return err + } + } + + return nil +} + +// exe suffix +var exe string diff --git a/go b/go new file mode 160000 index 0000000000..a9c82eae9a --- /dev/null +++ b/go @@ -0,0 +1 @@ +Subproject commit a9c82eae9a0e6d4efc52d405b4fa4876602d92cb diff --git a/go.mod b/go.mod new file mode 100644 index 0000000000..3069904606 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module go.encore.dev/go + +go 1.17 diff --git a/main.go b/main.go new file mode 100644 index 0000000000..af0ada1fd8 --- /dev/null +++ b/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "flag" + "log" + "os" + + "go.encore.dev/go/builder" +) + +func main() { + goos := flag.String("goos", "", "GOOS") + goarch := flag.String("goarch", "", "GOARCH") + dst := flag.String("dst", "dist", "build destination") + flag.Parse() + if *goos == "" || *goarch == "" || *dst == "" { + log.Fatalf("missing -dst %q, -goos %q, or -goarch %q", *dst, *goos, *goarch) + } + + root, err := os.Getwd() + if err != nil { + log.Fatalln(err) + } + + if err := builder.BuildEncoreGo(*goos, *goarch, root, *dst); err != nil { + log.Fatalln(err) + } +} diff --git a/overlay/src/net/http/encore.s b/overlay/src/net/http/encore.s new file mode 100644 index 0000000000..e69de29bb2 diff --git a/overlay/src/net/http/encore_app.go b/overlay/src/net/http/encore_app.go new file mode 100644 index 0000000000..84a87e39c6 --- /dev/null +++ b/overlay/src/net/http/encore_app.go @@ -0,0 +1,15 @@ +//go:build encore +// +build encore + +package http + +import ( + "context" + _ "unsafe" +) + +// encoreBeginRoundTrip is called by net/http when a RoundTrip begins. +func encoreBeginRoundTrip(req *Request) (context.Context, error) + +// encoreFinishRoundTrip is called by net/http when a RoundTrip completes. +func encoreFinishRoundTrip(req *Request, resp *Response, err error) diff --git a/overlay/src/net/http/encore_noapp.go b/overlay/src/net/http/encore_noapp.go new file mode 100644 index 0000000000..2046ab9007 --- /dev/null +++ b/overlay/src/net/http/encore_noapp.go @@ -0,0 +1,14 @@ +//go:build !encore +// +build !encore + +package http + +import ( + "context" +) + +// encoreBeginRoundTrip is called by net/http when a RoundTrip begins. +func encoreBeginRoundTrip(req *Request) (context.Context, error) { return req.Context(), nil } + +// encoreFinishRoundTrip is called by net/http when a RoundTrip completes. +func encoreFinishRoundTrip(req *Request, resp *Response, err error) {} diff --git a/overlay/src/runtime/encore.go b/overlay/src/runtime/encore.go new file mode 100644 index 0000000000..54d3d1af02 --- /dev/null +++ b/overlay/src/runtime/encore.go @@ -0,0 +1,281 @@ +package runtime + +// This file implements Encore's request tracking and runtime tracing. +// +// Request Tracking +// +// Encore tracks the current request being processed, in order for code +// to seamlessly access the right infrastructure resources and do request logging. +// +// The request tracking spans goroutines: when a goroutine starts it copies the +// current request from its parent goroutine. The Encore runtime can then look up +// the current request at will. +// +// During processing, a request can call other APIs which yield child requests. +// This group of requests is collectively called an operation. Operations +// are ref-counted (number of active requests + 1). The +1 allows for multiple +// root requests to be a part of a single operation; the creator of the operation +// marks the operation as completed which decrements the refcount by one. +// Note that a finished operation may continue being processed while child requests +// are still running. +// +// Tracing +// +// Tracing extends the request tracking machinery to implement performant tracing. +// Traces are associated with an operation at creation time. The runtime maintains +// an in-memory buffer which is appended to to log events. When the operation +// refcount reaches zero, the trace is marked completed and sent. + +import ( + "sync/atomic" + "unsafe" +) + +// encoreG tracks per-goroutine Encore-specific data. +type encoreG struct { + // op is the current operation the goroutine is a part of. + op *encoreOp + + // req holds Encore-specific request data, or nil if the g + // is not processing a request. + req *encoreReq + + // goid is the per-op goroutine id. + goid uint32 +} + +// encoreOp represents an Encore operation. +type encoreOp struct { + // start is the start time of the operation + start int64 // start time of trace from nanotime() + + // trace is the trace log; it is nil if the op is not traced. + trace *encoreTraceLog + + // refs is the op refcount. It is 1 + number of requests + // that reference this op (see doc comment above). + // It is accessed atomically. + refs int32 + + // goidCtr is a per-operation goroutine counter, for telling + // apart goroutines participating in the operation. + goidCtr uint32 +} + +// encoreSpanID represents a span id in Encore's tracing framework. +type encoreSpanID [8]byte + +// encoreReq represents an Encore API request. +type encoreReq struct { + // spanID is the request span id. + spanID encoreSpanID + // data is request-specific data defined in the Encore runtime. + data unsafe.Pointer +} + +// encoreBeginOp begins a new Encore operation. +// The trace parameter determines if the op is traced. +// +// It tags the current goroutine with the op. +// It panics if the goroutine is already part of an op. +func encoreBeginOp(trace bool) *encoreOp { + op := encoreNewOp(trace) + g := getg().m.curg + encoreTagG(g, op, nil) + return op +} + +// encoreNewOp creates a new encoreOp. +func encoreNewOp(trace bool) *encoreOp { + op := &encoreOp{ + start: nanotime(), + refs: 1, + } + if trace { + op.trace = new(encoreTraceLog) + } + return op +} + +// encoreTagG tags the g as participating in op, and with req +// as its request data. +// It does not increment the ref count, which means req +// must already be an active request. +// g must not already be part of an op. +func encoreTagG(g *g, op *encoreOp, req *encoreReq) (goid uint32) { + if g.encore != nil { + panic("encore.tagG: goroutine already part of another operation") + } + goid = atomic.AddUint32(&op.goidCtr, 1) + g.encore = &encoreG{ + op: op, + req: req, + goid: goid, + } + return goid +} + +// encoreGetG gets the encore data for the current g, or nil. +func encoreGetG() *encoreG { + return getg().m.curg.encore +} + +// encoreFinishOp marks an operation as finished +// and unsets the operation tag on the g. +// It must be part of an operation. +func encoreFinishOp() { + g := getg().m.curg + e := g.encore + if e == nil { + panic("encore.completeReq: goroutine not in an operation") + } + e.op.decRef() + g.encore = nil +} + +// incRef increases the op's refcount by one. +func (op *encoreOp) incRef() int32 { + return atomic.AddInt32(&op.refs, 1) +} + +// decRef decreases the op's refcount by one. +// If it reaches zero and the op is traced, it sends off the trace. +func (op *encoreOp) decRef() int32 { + n := atomic.AddInt32(&op.refs, -1) + if n == 0 && op.trace != nil { + op.trace.send() + } + return n +} + +// encoreTraceEvent adds the event to the trace. +// The g must already be part of an operation. +func encoreTraceEvent(event byte, data []byte) { + e := getg().m.curg.encore + if e == nil { + println("encore.traceEvent: goroutine not in an operation, skipping") + return + } + e.op.trace.log(event, data) +} + +// encoreBeginReq sets the request data for the current g, +// and increases the ref count on the operation. +// If the g is not part of an op, it creates a new op +// that is bound to the request lifetime. +func encoreBeginReq(spanID encoreSpanID, data unsafe.Pointer, trace bool) { + g := getg().m.curg + e := g.encore + req := &encoreReq{spanID: spanID, data: data} + if e == nil { + op := encoreNewOp(trace) + encoreTagG(g, op, req) + // Don't increment the op refcount since it starts at one, + // and this is not a standalone op. + } else { + if e.req != nil { + panic("encore.beginReq: request already running") + } + e.op.incRef() + e.req = req + } +} + +// encoreCompleteReq completes the request and decreases the +// ref count on the operation. +// The g must be processing a request. +func encoreCompleteReq() { + e := getg().m.curg.encore + if e == nil { + panic("encore.completeReq: goroutine not in an operation") + } else if e.req == nil { + panic("encore.completeReq: no current request") + } + e.op.decRef() + e.req = nil +} + +// encoreCleareq clears request data from the running g +// without decrementing the ref count. +// The g must be processing a request. +func encoreClearReq() { + e := getg().m.curg.encore + if e == nil { + panic("encore.replaceReq: goroutine not in an operation") + } else if e.req == nil { + panic("encore.replaceReq: no current request") + } + spanID := e.req.spanID + e.req = nil + if e.op.trace != nil { + e.op.trace.log(0x05, []byte{ + spanID[0], + spanID[1], + spanID[2], + spanID[3], + spanID[4], + spanID[5], + spanID[6], + spanID[7], + byte(e.goid), + byte(e.goid >> 8), + byte(e.goid >> 16), + byte(e.goid >> 24), + }) + } +} + +type encoreTraceLog struct { + mu mutex + data []byte +} + +func (tl *encoreTraceLog) send() { + lock(&tl.mu) + data := tl.data + tl.data = tl.data[len(tl.data):] + unlock(&tl.mu) + encoreSendTrace(data) +} + +// log logs a new event in the trace. +// If tr is nil, it does nothing. +func (tl *encoreTraceLog) log(event byte, data []byte) { + if tl == nil { + return + } + ln := len(data) + if ln > (1<<32 - 1) { + println("encore.traceEvent: event too large, dropping") + return + } + + lock(&tl.mu) + defer unlock(&tl.mu) + + // Do this in the critical section to ensure we don't get + // out-of-order timestamps. + t := nanotime() + var b [13]byte + b[0] = event + b[1] = byte(t) + b[2] = byte(t >> 8) + b[3] = byte(t >> 16) + b[4] = byte(t >> 24) + b[5] = byte(t >> 32) + b[6] = byte(t >> 40) + b[7] = byte(t >> 48) + b[8] = byte(t >> 56) + b[9] = byte(ln) + b[10] = byte(ln >> 8) + b[11] = byte(ln >> 16) + b[12] = byte(ln >> 24) + tl.data = append(tl.data, append(b[:], data...)...) +} + +// encoreCallers is like runtime.Callers but also returns the offset +// of the text segment to make the PCs ASLR-independent. +func encoreCallers(skip int, pc []uintptr) (n int, off uintptr) { + n = Callers(skip+1, pc) + return n, firstmoduledata.text +} diff --git a/overlay/src/runtime/encore_app.go b/overlay/src/runtime/encore_app.go new file mode 100644 index 0000000000..92d0a2d475 --- /dev/null +++ b/overlay/src/runtime/encore_app.go @@ -0,0 +1,9 @@ +//go:build encore +// +build encore + +package runtime + +// encoreSendTrace is called by Encore's go runtime to send a trace. +// It must be defined by the Encore runtime and linked using +// go:linkname. +func encoreSendTrace(log []byte) diff --git a/overlay/src/runtime/encore_noapp.go b/overlay/src/runtime/encore_noapp.go new file mode 100644 index 0000000000..c4286f0d84 --- /dev/null +++ b/overlay/src/runtime/encore_noapp.go @@ -0,0 +1,9 @@ +//go:build !encore +// +build !encore + +package runtime + +// encoreSendTrace is called by Encore's go runtime to send a trace. +// It is defined here as a no-op for Encore binaries that were built +// without linking in the Encore runtime to avoid strange build errors. +func encoreSendTrace(log []byte) {} diff --git a/patches/net_patch.diff b/patches/net_patch.diff new file mode 100644 index 0000000000..016618b54d --- /dev/null +++ b/patches/net_patch.diff @@ -0,0 +1,32 @@ +diff --git a/src/net/http/transport.go b/src/net/http/transport.go +index 0aa48273dd..bf53cad76c 100644 +--- a/src/net/http/transport.go ++++ b/src/net/http/transport.go +@@ -499,9 +499,17 @@ func (t *Transport) alternateRoundTripper(req *Request) RoundTripper { + } + + // roundTrip implements a RoundTripper over HTTP. +-func (t *Transport) roundTrip(req *Request) (*Response, error) { ++func (t *Transport) roundTrip(req *Request) (rr *Response, err error) { + t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) +- ctx := req.Context() ++ ctx, err := encoreBeginRoundTrip(req) ++ if err != nil { ++ req.closeBody() ++ return nil, err ++ } ++ req.ctx = ctx ++ defer func() { ++ encoreFinishRoundTrip(req, rr, err) ++ }() + trace := httptrace.ContextClientTrace(ctx) + + if req.URL == nil { +@@ -512,6 +520,7 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { + req.closeBody() + return nil, errors.New("http: nil Request.Header") + } ++ + scheme := req.URL.Scheme + isHTTP := scheme == "http" || scheme == "https" + if isHTTP { diff --git a/patches/runtime_patch.diff b/patches/runtime_patch.diff new file mode 100644 index 0000000000..853b70fe5b --- /dev/null +++ b/patches/runtime_patch.diff @@ -0,0 +1,76 @@ +diff --git a/src/runtime/proc.go b/src/runtime/proc.go +index 73a789c189..846b9d88fb 100644 +--- a/src/runtime/proc.go ++++ b/src/runtime/proc.go +@@ -3376,6 +3376,27 @@ func goexit1() { + func goexit0(gp *g) { + _g_ := getg() + ++ if e := gp.encore; e != nil { ++ gp.encore = nil ++ if e.req != nil && e.op.trace != nil { ++ spanID := e.req.spanID ++ e.op.trace.log(0x04, []byte{ ++ spanID[0], ++ spanID[1], ++ spanID[2], ++ spanID[3], ++ spanID[4], ++ spanID[5], ++ spanID[6], ++ spanID[7], ++ byte(e.goid), ++ byte(e.goid >> 8), ++ byte(e.goid >> 16), ++ byte(e.goid >> 24), ++ }) ++ } ++ } ++ + casgstatus(gp, _Grunning, _Gdead) + if isSystemGoroutine(gp, false) { + atomic.Xadd(&sched.ngsys, -1) +@@ -4067,6 +4088,30 @@ func newproc1(fn *funcval, argp unsafe.Pointer, narg int32, callergp *g, callerp + gostartcallfn(&newg.sched, fn) + newg.gopc = callerpc + newg.ancestors = saveAncestors(callergp) ++ ++ if e := callergp.encore; e != nil { ++ goid := encoreTagG(newg, e.op, e.req) ++ ++ // Log an event if we are tracing this ++ if e.req != nil && e.op.trace != nil { ++ spanID := e.req.spanID ++ e.op.trace.log(0x03, []byte{ ++ spanID[0], ++ spanID[1], ++ spanID[2], ++ spanID[3], ++ spanID[4], ++ spanID[5], ++ spanID[6], ++ spanID[7], ++ byte(goid), ++ byte(goid >> 8), ++ byte(goid >> 16), ++ byte(goid >> 24), ++ }) ++ } ++ } ++ + newg.startpc = fn.fn + if _g_.m.curg != nil { + newg.labels = _g_.m.curg.labels +diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go +index 109f0da131..d7e0158bfc 100644 +--- a/src/runtime/runtime2.go ++++ b/src/runtime/runtime2.go +@@ -471,6 +471,8 @@ type g struct { + timer *timer // cached timer for time.Sleep + selectDone uint32 // are we participating in a select and did someone win the race? + ++ encore *encoreG // encore-specific goroutine data ++ + // Per-G GC state + + // gcAssistBytes is this G's GC assist credit in terms of diff --git a/patches/version_patch.diff b/patches/version_patch.diff new file mode 100644 index 0000000000..04729bd9ef --- /dev/null +++ b/patches/version_patch.diff @@ -0,0 +1,23 @@ +diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go +index d37c3f83ef..ea7f17eaa8 100644 +--- a/src/cmd/dist/build.go ++++ b/src/cmd/dist/build.go +@@ -341,6 +341,9 @@ func branchtag(branch string) (tag string, precise bool) { + } + break + } ++ ++ tag = "encore-" + tag ++ + return + } + +@@ -422,7 +425,7 @@ func findgoversion() string { + if m == nil { + fatalf("internal/goversion/goversion.go does not contain 'const Version = ...'") + } +- tag += fmt.Sprintf(" go1.%s-", m[1]) ++ tag += fmt.Sprintf(" encore-go1.%s-", m[1]) + + tag += chomp(run(goroot, CheckExit, "git", "log", "-n", "1", "--format=format:%h %cd", "HEAD")) + }