rlp, trie: faster trie node encoding (#24126)

commit ethereum/go-ethereum@65ed1a6.

This change speeds up trie hashing and all other activities that require
RLP encoding of trie nodes by approximately 20%. The speedup is achieved by
avoiding reflection overhead during node encoding.

The interface type trie.node now contains a method 'encode' that works with
rlp.EncoderBuffer. Management of EncoderBuffers is left to calling code.
trie.hasher, which is pooled to avoid allocations, now maintains an
EncoderBuffer. This means memory resources related to trie node encoding
are tied to the hasher pool.

This also refactors some functions in rlp package.

goos: linux
goarch: amd64
cpu: 11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz
                          │   old.txt    │               new.txt                │
                          │    sec/op    │    sec/op     vs base                │
DeriveSha200/std_trie-8     725.1µ ± 31%   613.8µ ± 37%        ~ (p=0.481 n=10)
DeriveSha200/stack_trie-8   572.3µ ± 10%   493.1µ ± 13%  -13.85% (p=0.005 n=10)
geomean                     644.2µ         550.1µ        -14.61%

                          │   old.txt    │               new.txt                │
                          │     B/op     │     B/op      vs base                │
DeriveSha200/std_trie-8     287.4Ki ± 0%   283.0Ki ± 0%   -1.53% (p=0.000 n=10)
DeriveSha200/stack_trie-8   56.34Ki ± 0%   42.43Ki ± 0%  -24.69% (p=0.000 n=10)
geomean                     127.2Ki        109.6Ki       -13.88%

                          │   old.txt   │               new.txt               │
                          │  allocs/op  │  allocs/op   vs base                │
DeriveSha200/std_trie-8     2.931k ± 0%   2.917k ± 0%   -0.46% (p=0.000 n=10)
DeriveSha200/stack_trie-8   1.462k ± 0%   1.246k ± 0%  -14.77% (p=0.000 n=10)
geomean                     2.070k        1.907k        -7.90%

                         │   old.txt    │               new.txt                │
                         │    sec/op    │    sec/op     vs base                │
Prove-8                    664.0µ ± 21%   450.2µ ± 27%  -32.20% (p=0.000 n=10)
VerifyProof-8              8.643µ ± 18%   9.009µ ± 33%        ~ (p=0.684 n=10)
VerifyRangeProof10-8       99.18µ ± 25%   67.60µ ± 67%        ~ (p=0.089 n=10)
VerifyRangeProof100-8      496.3µ ± 20%   487.0µ ± 33%        ~ (p=0.739 n=10)
VerifyRangeProof1000-8     5.149m ± 32%   4.095m ± 49%        ~ (p=0.971 n=10)
VerifyRangeProof5000-8     19.79m ± 60%   19.16m ± 28%        ~ (p=0.631 n=10)
VerifyRangeNoProof10-8     499.0µ ± 15%   422.8µ ± 29%  -15.25% (p=0.035 n=10)
VerifyRangeNoProof500-8    1.747m ± 30%   1.417m ± 24%  -18.91% (p=0.023 n=10)
VerifyRangeNoProof1000-8   3.025m ± 29%   2.239m ± 33%  -25.98% (p=0.009 n=10)
geomean                    750.9µ         622.6µ        -17.09%

                     │    old.txt    │               new.txt                │
                     │    sec/op     │    sec/op     vs base                │
HashFixedSize/10-8      60.30µ ± 19%   44.84µ ± 17%  -25.64% (p=0.000 n=10)
HashFixedSize/100-8     205.9µ ± 32%   145.2µ ± 19%  -29.48% (p=0.000 n=10)
HashFixedSize/1K-8     1326.5µ ± 23%   939.2µ ± 25%  -29.20% (p=0.002 n=10)
HashFixedSize/10K-8     14.77m ± 25%   12.74m ± 19%        ~ (p=0.075 n=10)
HashFixedSize/100K-8    135.2m ± 19%   104.1m ± 18%  -23.03% (p=0.003 n=10)
geomean                 2.011m         1.520m        -24.43%

                     │    old.txt    │               new.txt                │
                     │     B/op      │     B/op      vs base                │
HashFixedSize/10-8     11.729Ki ± 0%   9.752Ki ± 0%  -16.85% (p=0.000 n=10)
HashFixedSize/100-8     58.56Ki ± 0%   49.23Ki ± 0%  -15.93% (p=0.000 n=10)
HashFixedSize/1K-8      578.1Ki ± 0%   481.5Ki ± 0%  -16.72% (p=0.000 n=10)
HashFixedSize/10K-8     6.019Mi ± 0%   4.985Mi ± 0%  -17.18% (p=0.000 n=10)
HashFixedSize/100K-8    59.53Mi ± 0%   49.29Mi ± 0%  -17.20% (p=0.000 n=10)
geomean                 683.5Ki        568.8Ki       -16.78%

                     │   old.txt   │              new.txt               │
                     │  allocs/op  │  allocs/op   vs base               │
HashFixedSize/10-8      149.0 ± 0%    142.0 ± 0%  -4.70% (p=0.000 n=10)
HashFixedSize/100-8     772.0 ± 0%    739.0 ± 0%  -4.27% (p=0.000 n=10)
HashFixedSize/1K-8     7.443k ± 0%   7.099k ± 0%  -4.62% (p=0.000 n=10)
HashFixedSize/10K-8    77.09k ± 0%   73.32k ± 0%  -4.89% (p=0.000 n=10)
HashFixedSize/100K-8   767.8k ± 0%   730.5k ± 0%  -4.86% (p=0.000 n=10)
geomean                8.729k        8.321k       -4.67%

Co-authored-by: Felix Lange <[email protected]>

rlp/encbuffer.go

@@ -39,27 +39,31 @@ func (buf *encBuffer) size() int {
 	return len(buf.str) + buf.lhsize
 }
 
-// toBytes creates the encoder output.
-func (w *encBuffer) toBytes() []byte {
+// makeBytes creates the encoder output.
+func (w *encBuffer) makeBytes() []byte {
 	out := make([]byte, w.size())
+	w.copyTo(out)
+	return out
+}
+
+func (w *encBuffer) copyTo(dst []byte) {
 	strpos := 0
 	pos := 0
 	for _, head := range w.lheads {
 		// write string data before header
-		n := copy(out[pos:], w.str[strpos:head.offset])
+		n := copy(dst[pos:], w.str[strpos:head.offset])
 		pos += n
 		strpos += n
 		// write the header
-		enc := head.encode(out[pos:])
+		enc := head.encode(dst[pos:])
 		pos += len(enc)
 	}
 	// copy string data after the last list header
-	copy(out[pos:], w.str[strpos:])
-	return out
+	copy(dst[pos:], w.str[strpos:])
 }
 
-// toWriter writes the encoder output to w.
-func (buf *encBuffer) toWriter(w io.Writer) (err error) {
+// writeTo writes the encoder output to w.
+func (buf *encBuffer) writeTo(w io.Writer) (err error) {
 	strpos := 0
 	for _, head := range buf.lheads {
 		// write string data before header
@@ -268,6 +272,19 @@ func (r *encReader) next() []byte {
 	}
 }
 
+func encBufferFromWriter(w io.Writer) *encBuffer {
+	switch w := w.(type) {
+	case EncoderBuffer:
+		return w.buf
+	case *EncoderBuffer:
+		return w.buf
+	case *encBuffer:
+		return w
+	default:
+		return nil
+	}
+}
+
 // EncoderBuffer is a buffer for incremental encoding.
 //
 // The zero value is NOT ready for use. To get a usable buffer,
@@ -295,14 +312,10 @@ func (w *EncoderBuffer) Reset(dst io.Writer) {
 	// If the destination writer has an *encBuffer, use it.
 	// Note that w.ownBuffer is left false here.
 	if dst != nil {
-		if outer, ok := dst.(*encBuffer); ok {
+		if outer := encBufferFromWriter(dst); outer != nil {
 			*w = EncoderBuffer{outer, nil, false}
 			return
 		}
-		if outer, ok := dst.(EncoderBuffer); ok {
-			*w = EncoderBuffer{outer.buf, nil, false}
-			return
-		}
 	}
 
 	// Get a fresh buffer.
@@ -319,7 +332,7 @@ func (w *EncoderBuffer) Reset(dst io.Writer) {
 func (w *EncoderBuffer) Flush() error {
 	var err error
 	if w.dst != nil {
-		err = w.buf.toWriter(w.dst)
+		err = w.buf.writeTo(w.dst)
 	}
 	// Release the internal buffer.
 	if w.ownBuffer {
@@ -331,7 +344,15 @@ func (w *EncoderBuffer) Flush() error {
 
 // ToBytes returns the encoded bytes.
 func (w *EncoderBuffer) ToBytes() []byte {
-	return w.buf.toBytes()
+	return w.buf.makeBytes()
+}
+
+// AppendToBytes appends the encoded bytes to dst.
+func (w *EncoderBuffer) AppendToBytes(dst []byte) []byte {
+	size := w.buf.size()
+	out := append(dst, make([]byte, size)...)
+	w.buf.copyTo(out[len(dst):])
+	return out
 }
 
 // Write appends b directly to the encoder output.

rlp/encode.go

@@ -57,20 +57,16 @@ type Encoder interface {
 // Please see package-level documentation of encoding rules.
 func Encode(w io.Writer, val interface{}) error {
 	// Optimization: reuse *encBuffer when called by EncodeRLP.
-	if buf, ok := w.(*encBuffer); ok {
+	if buf := encBufferFromWriter(w); buf != nil {
 		return buf.encode(val)
 	}
-	if ebuf, ok := w.(EncoderBuffer); ok {
-		return ebuf.buf.encode(val)
-	}
 
 	buf := getEncBuffer()
 	defer encBufferPool.Put(buf)
-
 	if err := buf.encode(val); err != nil {
 		return err
 	}
-	return buf.toWriter(w)
+	return buf.writeTo(w)
 }
 
 // EncodeToBytes returns the RLP encoding of val.
@@ -82,7 +78,7 @@ func EncodeToBytes(val interface{}) ([]byte, error) {
 	if err := buf.encode(val); err != nil {
 		return nil, err
 	}
-	return buf.toBytes(), nil
+	return buf.makeBytes(), nil
 }
 
 // EncodeToReader returns a reader from which the RLP encoding of val

rlp/encode_test.go

@@ -431,6 +431,21 @@ func TestEncodeToBytes(t *testing.T) {
 	runEncTests(t, EncodeToBytes)
 }
 
+func TestEncodeAppendToBytes(t *testing.T) {
+	buffer := make([]byte, 20)
+	runEncTests(t, func(val interface{}) ([]byte, error) {
+		w := NewEncoderBuffer(nil)
+		defer w.Flush()
+
+		err := Encode(w, val)
+		if err != nil {
+			return nil, err
+		}
+		output := w.AppendToBytes(buffer[:0])
+		return output, nil
+	})
+}
+
 func TestEncodeToReader(t *testing.T) {
 	runEncTests(t, func(val interface{}) ([]byte, error) {
 		_, r, err := EncodeToReader(val)

trie/committer.go

@@ -35,7 +35,6 @@ const leafChanSize = 200
 // capture all dirty nodes during the commit process and keep them cached in
 // insertion order.
 type committer struct {
-	tmp sliceBuffer
 	sha crypto.KeccakState
 
 	owner       common.Hash // TODO: same as nodes.owner, consider removing
@@ -48,7 +47,6 @@ type committer struct {
 var committerPool = sync.Pool{
 	New: func() interface{} {
 		return &committer{
-			tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode.
 			sha: sha3.NewLegacyKeccak256().(crypto.KeccakState),
 		}
 	},

trie/hasher.go

@@ -24,31 +24,22 @@ import (
 	"golang.org/x/crypto/sha3"
 )
 
-type sliceBuffer []byte
-
-func (b *sliceBuffer) Write(data []byte) (n int, err error) {
-	*b = append(*b, data...)
-	return len(data), nil
-}
-
-func (b *sliceBuffer) Reset() {
-	*b = (*b)[:0]
-}
-
 // hasher is a type used for the trie Hash operation. A hasher has some
 // internal preallocated temp space
 type hasher struct {
 	sha      crypto.KeccakState
-	tmp      sliceBuffer
+	tmp      []byte
+	encbuf   rlp.EncoderBuffer
 	parallel bool // Whether to use paralallel threads when hashing
 }
 
 // hasherPool holds pureHashers
 var hasherPool = sync.Pool{
 	New: func() interface{} {
 		return &hasher{
-			tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode.
-			sha: sha3.NewLegacyKeccak256().(crypto.KeccakState),
+			tmp:    make([]byte, 0, 550), // cap is as large as a full fullNode.
+			sha:    sha3.NewLegacyKeccak256().(crypto.KeccakState),
+			encbuf: rlp.NewEncoderBuffer(nil),
 		}
 	},
 }
@@ -153,30 +144,41 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached
 // into compact form for RLP encoding.
 // If the rlp data is smaller than 32 bytes, `nil` is returned.
 func (h *hasher) shortnodeToHash(n *shortNode, force bool) node {
-	h.tmp.Reset()
-	if err := rlp.Encode(&h.tmp, n); err != nil {
-		panic("encode error: " + err.Error())
-	}
+	n.encode(h.encbuf)
+	enc := h.encodedBytes()
 
-	if len(h.tmp) < 32 && !force {
+	if len(enc) < 32 && !force {
 		return n // Nodes smaller than 32 bytes are stored inside their parent
 	}
-	return h.hashData(h.tmp)
+	return h.hashData(enc)
 }
 
 // shortnodeToHash is used to creates a hashNode from a set of hashNodes, (which
 // may contain nil values)
 func (h *hasher) fullnodeToHash(n *fullNode, force bool) node {
-	h.tmp.Reset()
-	// Generate the RLP encoding of the node
-	if err := n.EncodeRLP(&h.tmp); err != nil {
-		panic("encode error: " + err.Error())
-	}
+	n.encode(h.encbuf)
+	enc := h.encodedBytes()
 
-	if len(h.tmp) < 32 && !force {
+	if len(enc) < 32 && !force {
 		return n // Nodes smaller than 32 bytes are stored inside their parent
 	}
-	return h.hashData(h.tmp)
+	return h.hashData(enc)
+}
+
+// encodedBytes returns the result of the last encoding operation on h.encbuf.
+// This also resets the encoder buffer.
+//
+// All node encoding must be done like this:
+//
+//     node.encode(h.encbuf)
+//     enc := h.encodedBytes()
+//
+// This convention exists because node.encode can only be inlined/escape-analyzed when
+// called on a concrete receiver type.
+func (h *hasher) encodedBytes() []byte {
+	h.tmp = h.encbuf.AppendToBytes(h.tmp[:0])
+	h.encbuf.Reset(nil)
+	return h.tmp
 }
 
 // hashData hashes the provided data

trie/iterator.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // NodeResolver is used for looking up trie nodes before reaching into the real
@@ -223,8 +222,7 @@ func (it *nodeIterator) LeafProof() [][]byte {
 				// Gather nodes that end up as hash nodes (or the root)
 				node, hashed := hasher.proofHash(item.node)
 				if _, ok := hashed.(hashNode); ok || i == 0 {
-					enc, _ := rlp.EncodeToBytes(node)
-					proofs = append(proofs, enc)
+					proofs = append(proofs, nodeToBytes(node))
 				}
 			}
 			return proofs

trie/node.go

@@ -28,8 +28,9 @@ import (
 var indices = []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "[17]"}
 
 type node interface {
-	fstring(string) string
 	cache() (hashNode, bool)
+	encode(w rlp.EncoderBuffer)
+	fstring(string) string
 }
 
 type (
@@ -52,16 +53,9 @@ var nilValueNode = valueNode(nil)
 
 // EncodeRLP encodes a full node into the consensus RLP format.
 func (n *fullNode) EncodeRLP(w io.Writer) error {
-	var nodes [17]node
-
-	for i, child := range &n.Children {
-		if child != nil {
-			nodes[i] = child
-		} else {
-			nodes[i] = nilValueNode
-		}
-	}
-	return rlp.Encode(w, nodes)
+	eb := rlp.NewEncoderBuffer(w)
+	n.encode(eb)
+	return eb.Flush()
 }
 
 func (n *fullNode) copy() *fullNode   { copy := *n; return &copy }

trie/node_enc.go

@@ -0,0 +1,64 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+func nodeToBytes(n node) []byte {
+	w := rlp.NewEncoderBuffer(nil)
+	n.encode(w)
+	result := w.ToBytes()
+	w.Flush()
+	return result
+}
+
+func (n *fullNode) encode(w rlp.EncoderBuffer) {
+	offset := w.List()
+	for _, c := range n.Children {
+		if c != nil {
+			c.encode(w)
+		} else {
+			w.Write(rlp.EmptyString)
+		}
+	}
+	w.ListEnd(offset)
+}
+
+func (n *shortNode) encode(w rlp.EncoderBuffer) {
+	offset := w.List()
+	w.WriteBytes(n.Key)
+	if n.Val != nil {
+		n.Val.encode(w)
+	} else {
+		w.Write(rlp.EmptyString)
+	}
+	w.ListEnd(offset)
+}
+
+func (n hashNode) encode(w rlp.EncoderBuffer) {
+	w.WriteBytes(n)
+}
+
+func (n valueNode) encode(w rlp.EncoderBuffer) {
+	w.WriteBytes(n)
+}
+
+func (n rawNode) encode(w rlp.EncoderBuffer) {
+	w.Write(n)
+}

trie/proof.go

@@ -24,7 +24,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // Prove constructs a merkle proof for key. The result contains all encoded nodes
@@ -94,7 +93,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e
 		if hash, ok := hn.(hashNode); ok || i == 0 {
 			// If the node's database encoding is a hash (or is the
 			// root node), it becomes a proof element.
-			enc, _ := rlp.EncodeToBytes(n)
+			enc := nodeToBytes(n)
 			if !ok {
 				hash = hasher.hashData(enc)
 			}