Fixes #1 - use streamed_output to get around 2GB limitation (#10)

README.md

@@ -80,7 +80,6 @@ A couple reasons:
 
 * Some of the short codes for the CLI are different (only supporting one character instead of two)
 * Due to some minor implementation differences, the actual hashes generated by the two programs are different
-* `bazel-differ` isn't using `streamed_proto` output from Bazel query (I'm not sure if there is a Go implemntation of this?). In some minimal testing against some large repositories, `bazel-differ` still seems to outperform `bazel-diff` by 2x.
 
 3. What target types has this been tested against?
 

internal/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
         "filesystem.go",
         "git_client.go",
         "io_utils.go",
+        "proto_delimited.go",
         "target_hashing_client.go",
     ],
     importpath = "github.com/ewhauser/bazel-differ/internal",
@@ -24,9 +25,15 @@ go_library(
 
 go_test(
     name = "internal_test",
-    srcs = ["target_hashing_client_test.go"],
+    srcs = [
+        "proto_delimited_test.go",
+        "target_hashing_client_test.go",
+    ],
+    data = [
+        "query.protodelim",  # keep
+    ],
+    embed = [":internal"],
     deps = [
-        ":internal",
         "//mocks",
         "@com_github_golang_mock//gomock",
     ],

internal/bazel.go

@@ -18,8 +18,6 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"fmt"
-	"google.golang.org/protobuf/proto"
 	"io"
 	"log"
 	"os"
@@ -122,7 +120,7 @@ type Bazel interface {
 	WriteToStderr(v bool)
 	WriteToStdout(v bool)
 	Info() (map[string]string, error)
-	Query(args ...string) (*QueryResult, error)
+	Query(args ...string) ([]*Target, error)
 	Build(args ...string) (*bytes.Buffer, error)
 	Test(args ...string) (*bytes.Buffer, error)
 	Run(args ...string) (*exec.Cmd, *bytes.Buffer, error)
@@ -286,8 +284,8 @@ func (b *bazel) processInfo(info string) (map[string]string, error) {
 // or to find a dependency path between //path/to/package:target and //dependency:
 //
 //   res, err := b.Query('somepath(//path/to/package:target, //dependency)')
-func (b *bazel) Query(args ...string) (*QueryResult, error) {
-	blazeArgs := append([]string(nil), "--output=proto", "--order_output=no", "--color=no")
+func (b *bazel) Query(args ...string) ([]*Target, error) {
+	blazeArgs := append([]string(nil), "--output=streamed_proto", "--order_output=no", "--color=no")
 	blazeArgs = append(blazeArgs, args...)
 
 	b.WriteToStderr(true)
@@ -302,14 +300,9 @@ func (b *bazel) Query(args ...string) (*QueryResult, error) {
 	return b.processQuery(stdoutBuffer.Bytes())
 }
 
-func (b *bazel) processQuery(out []byte) (*QueryResult, error) {
-	var qr QueryResult
-	if err := proto.Unmarshal(out, &qr); err != nil {
-		fmt.Fprintf(os.Stderr, "Could not read blaze query response. Error: %s\nOutput: %s\n", err, out)
-		return nil, err
-	}
-
-	return &qr, nil
+func (b *bazel) processQuery(out []byte) ([]*Target, error) {
+	reader := NewReader(bytes.NewReader(out))
+	return reader.ReadTargets()
 }
 
 func (b *bazel) Build(args ...string) (*bytes.Buffer, error) {

internal/bazel_client.go

@@ -146,5 +146,5 @@ func (b bazelClient) performBazelQuery(query string) ([]*Target, error) {
 	if err != nil {
 		return nil, err
 	}
-	return result.Target, nil
+	return result, nil
 }

internal/proto_delimited.go

@@ -0,0 +1,47 @@
+package internal
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"google.golang.org/protobuf/proto"
+	"io"
+)
+
+type ProtoDelimitedReader struct {
+	buf *bufio.Reader
+}
+
+func NewReader(r io.Reader) *ProtoDelimitedReader {
+	return &ProtoDelimitedReader{
+		buf: bufio.NewReader(r),
+	}
+}
+
+func (r *ProtoDelimitedReader) Next() ([]byte, error) {
+	size, err := binary.ReadUvarint(r.buf)
+	if err != nil {
+		return nil, err
+	}
+	data := make([]byte, size)
+	if _, err := io.ReadFull(r.buf, data); err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func (r *ProtoDelimitedReader) ReadTargets() ([]*Target, error) {
+	var targets []*Target
+	var err error
+	for buf, err := r.Next(); err == nil; buf, err = r.Next() {
+		var target Target
+		if err := proto.Unmarshal(buf, &target); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal Target message: %w", err)
+		}
+		targets = append(targets, &target)
+	}
+	if err != nil && err != io.EOF {
+		return nil, fmt.Errorf("error while reading stdout from bazel command: %w", err)
+	}
+	return targets, nil
+}

internal/proto_delimited_test.go

@@ -0,0 +1,23 @@
+package internal
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+)
+
+func TestDelimitedReader(t *testing.T) {
+	protoBytes, err := ioutil.ReadFile("query.protodelim")
+	if err != nil {
+		t.Errorf("Error reading file")
+	}
+	reader := NewReader(bytes.NewReader(protoBytes))
+	targets, err := reader.ReadTargets()
+	if err != nil {
+		t.Errorf("error marshalling: %s", err)
+		t.FailNow()
+	}
+	if len(targets) != 49 {
+		t.Errorf("Expecting 49 targets but got %d", len(targets))
+	}
+}