fix: increase MaxTokenSize to handle large snapshot lines (#76)

* fix: increase MaxTokenSize to handle large snapshot lines

* chore: bump go to 1.21.0

.github/workflows/go.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
@@ -42,6 +42,6 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
       - name: Run Tests
         run: make test-verbose

go.mod

@@ -1,15 +1,12 @@
 module github.com/gkampitakis/go-snaps
 
-go 1.20
+go 1.21.0
 
 require (
 	github.com/gkampitakis/ciinfo v0.2.5
 	github.com/gkampitakis/go-diff v1.3.2
 	github.com/kr/pretty v0.3.1
-)
-
-require (
-	github.com/tidwall/gjson v1.14.4
+	github.com/tidwall/gjson v1.16.0
 	github.com/tidwall/pretty v1.2.1
 	github.com/tidwall/sjson v1.2.5
 )

go.sum

@@ -12,8 +12,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
-github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg=
+github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=

snaps/clean.go

@@ -1,7 +1,6 @@
 package snaps
 
 import (
-	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
@@ -157,7 +156,7 @@ func examineSnaps(
 		var hasDiffs bool
 
 		registeredTests := occurrences(registry[snapPath])
-		s := bufio.NewScanner(f)
+		s := snapshotScanner(f)
 
 		for s.Scan() {
 			b := s.Bytes()
@@ -192,6 +191,10 @@ func examineSnaps(
 			}
 		}
 
+		if err := s.Err(); err != nil {
+			return nil, err
+		}
+
 		if !hasDiffs || !shouldUpdate {
 			continue
 		}

snaps/snapshot.go

@@ -116,7 +116,7 @@ func getPrevSnapshot(testID, snapPath string) (string, int, error) {
 	lineNumber := 1
 	tid := []byte(testID)
 
-	s := bufio.NewScanner(bytes.NewReader(f))
+	s := snapshotScanner(bytes.NewReader(f))
 	for s.Scan() {
 		l := s.Bytes()
 		if !bytes.Equal(l, tid) {
@@ -136,6 +136,10 @@ func getPrevSnapshot(testID, snapPath string) (string, int, error) {
 		}
 	}
 
+	if err := s.Err(); err != nil {
+		return "", -1, err
+	}
+
 	return "", -1, errSnapNotFound
 }
 
@@ -172,7 +176,7 @@ func updateSnapshot(testID, snapshot, snapPath string) error {
 		updatedSnapFile.Grow(int(i.Size()))
 	}
 
-	s := bufio.NewScanner(f)
+	s := snapshotScanner(f)
 	for s.Scan() {
 		b := s.Bytes()
 		updatedSnapFile.Write(b)
@@ -190,6 +194,10 @@ func updateSnapshot(testID, snapshot, snapPath string) error {
 		updatedSnapFile.WriteByte('\n')
 	}
 
+	if err := s.Err(); err != nil {
+		return err
+	}
+
 	return overwriteFile(f, updatedSnapFile.Bytes())
 }
 

snaps/utils.go

@@ -1,7 +1,10 @@
 package snaps
 
 import (
+	"bufio"
 	"errors"
+	"io"
+	"math"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -103,6 +106,13 @@ func baseCaller(skip int) string {
 	}
 }
 
+// snapshotScanner returns a new *bufio.Scanner with a `MaxScanTokenSize == math.MaxInt` to read from r.
+func snapshotScanner(r io.Reader) *bufio.Scanner {
+	s := bufio.NewScanner(r)
+	s.Buffer([]byte{}, math.MaxInt)
+	return s
+}
+
 // shouldUpdateSingle returns if a single should be updated or not
 //
 // it depends on the general should update or if the given name on `UPDATE_SNAPS` matches the current test.