rpc: simplify the encoding of interface-typed arguments in JSON (tendermint#7600)

Add package jsontypes that implements a subset of the custom libs/json 
package. Specifically it handles encoding and decoding of interface types
wrapped in "tagged" JSON objects. It omits the deep reflection on arbitrary
types, preserving only the handling of type tags wrapper encoding.

- Register interface types (Evidence, PubKey, PrivKey) for tagged encoding.
- Update the existing implementations to satisfy the type.
- Register those types with the jsontypes registry.
- Add string tags to 64-bit integer fields where needed.
- Add marshalers to structs that export interface-typed fields.

Author: M. J. Fromberger

crypto/crypto.go

@@ -2,6 +2,7 @@ package crypto
 
 import (
 	"github.com/tendermint/tendermint/crypto/tmhash"
+	"github.com/tendermint/tendermint/internal/jsontypes"
 	"github.com/tendermint/tendermint/libs/bytes"
 )
 
@@ -25,6 +26,9 @@ type PubKey interface {
 	VerifySignature(msg []byte, sig []byte) bool
 	Equals(PubKey) bool
 	Type() string
+
+	// Implementations must support tagged encoding in JSON.
+	jsontypes.Tagged
 }
 
 type PrivKey interface {
@@ -33,6 +37,9 @@ type PrivKey interface {
 	PubKey() PubKey
 	Equals(PrivKey) bool
 	Type() string
+
+	// Implementations must support tagged encoding in JSON.
+	jsontypes.Tagged
 }
 
 type Symmetric interface {

crypto/ed25519/ed25519.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/crypto/tmhash"
+	"github.com/tendermint/tendermint/internal/jsontypes"
 	tmjson "github.com/tendermint/tendermint/libs/json"
 )
 
@@ -58,11 +59,17 @@ const (
 func init() {
 	tmjson.RegisterType(PubKey{}, PubKeyName)
 	tmjson.RegisterType(PrivKey{}, PrivKeyName)
+
+	jsontypes.MustRegister(PubKey{})
+	jsontypes.MustRegister(PrivKey{})
 }
 
 // PrivKey implements crypto.PrivKey.
 type PrivKey []byte
 
+// TypeTag satisfies the jsontypes.Tagged interface.
+func (PrivKey) TypeTag() string { return PrivKeyName }
+
 // Bytes returns the privkey byte format.
 func (privKey PrivKey) Bytes() []byte {
 	return []byte(privKey)
@@ -151,6 +158,9 @@ var _ crypto.PubKey = PubKey{}
 // PubKeyEd25519 implements crypto.PubKey for the Ed25519 signature scheme.
 type PubKey []byte
 
+// TypeTag satisfies the jsontypes.Tagged interface.
+func (PubKey) TypeTag() string { return PubKeyName }
+
 // Address is the SHA256-20 of the raw pubkey bytes.
 func (pubKey PubKey) Address() crypto.Address {
 	if len(pubKey) != PubKeySize {

crypto/secp256k1/secp256k1.go

@@ -10,6 +10,7 @@ import (
 
 	secp256k1 "github.com/btcsuite/btcd/btcec"
 	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/internal/jsontypes"
 	tmjson "github.com/tendermint/tendermint/libs/json"
 
 	// necessary for Bitcoin address format
@@ -28,13 +29,19 @@ const (
 func init() {
 	tmjson.RegisterType(PubKey{}, PubKeyName)
 	tmjson.RegisterType(PrivKey{}, PrivKeyName)
+
+	jsontypes.MustRegister(PubKey{})
+	jsontypes.MustRegister(PrivKey{})
 }
 
 var _ crypto.PrivKey = PrivKey{}
 
 // PrivKey implements PrivKey.
 type PrivKey []byte
 
+// TypeTag satisfies the jsontypes.Tagged interface.
+func (PrivKey) TypeTag() string { return PrivKeyName }
+
 // Bytes marshalls the private key using amino encoding.
 func (privKey PrivKey) Bytes() []byte {
 	return []byte(privKey)
@@ -138,6 +145,9 @@ const PubKeySize = 33
 // This prefix is followed with the x-coordinate.
 type PubKey []byte
 
+// TypeTag satisfies the jsontypes.Tagged interface.
+func (PubKey) TypeTag() string { return PubKeyName }
+
 // Address returns a Bitcoin style addresses: RIPEMD160(SHA256(pubkey))
 func (pubKey PubKey) Address() crypto.Address {
 	if len(pubKey) != PubKeySize {

crypto/sr25519/encoding.go

@@ -1,6 +1,9 @@
 package sr25519
 
-import tmjson "github.com/tendermint/tendermint/libs/json"
+import (
+	"github.com/tendermint/tendermint/internal/jsontypes"
+	tmjson "github.com/tendermint/tendermint/libs/json"
+)
 
 const (
 	PrivKeyName = "tendermint/PrivKeySr25519"
@@ -10,4 +13,7 @@ const (
 func init() {
 	tmjson.RegisterType(PubKey{}, PubKeyName)
 	tmjson.RegisterType(PrivKey{}, PrivKeyName)
+
+	jsontypes.MustRegister(PubKey{})
+	jsontypes.MustRegister(PrivKey{})
 }

crypto/sr25519/privkey.go

@@ -29,6 +29,9 @@ type PrivKey struct {
 	kp  *sr25519.KeyPair
 }
 
+// TypeTag satisfies the jsontypes.Tagged interface.
+func (PrivKey) TypeTag() string { return PrivKeyName }
+
 // Bytes returns the byte-encoded PrivKey.
 func (privKey PrivKey) Bytes() []byte {
 	if privKey.kp == nil {

crypto/sr25519/pubkey.go

@@ -23,6 +23,9 @@ const (
 // PubKey implements crypto.PubKey.
 type PubKey []byte
 
+// TypeTag satisfies the jsontypes.Tagged interface.
+func (PubKey) TypeTag() string { return PubKeyName }
+
 // Address is the SHA256-20 of the raw pubkey bytes.
 func (pubKey PubKey) Address() crypto.Address {
 	if len(pubKey) != PubKeySize {

internal/jsontypes/jsontypes.go

@@ -0,0 +1,109 @@
+// Package jsontypes supports decoding for interface types whose concrete
+// implementations need to be stored as JSON. To do this, concrete values are
+// packaged in wrapper objects having the form:
+//
+//   {
+//     "type": "<type-tag>",
+//     "value": <json-encoding-of-value>
+//   }
+//
+// This package provides a registry for type tag strings and functions to
+// encode and decode wrapper objects.
+package jsontypes
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"reflect"
+)
+
+// The Tagged interface must be implemented by a type in order to register it
+// with the jsontypes package. The TypeTag method returns a string label that
+// is used to distinguish objects of that type.
+type Tagged interface {
+	TypeTag() string
+}
+
+// registry records the mapping from type tags to value types.  Values in this
+// map must be normalized to non-pointer types.
+var registry = struct {
+	types map[string]reflect.Type
+}{types: make(map[string]reflect.Type)}
+
+// register adds v to the type registry. It reports an error if the tag
+// returned by v is already registered.
+func register(v Tagged) error {
+	tag := v.TypeTag()
+	if t, ok := registry.types[tag]; ok {
+		return fmt.Errorf("type tag %q already registered to %v", tag, t)
+	}
+	typ := reflect.TypeOf(v)
+	if typ.Kind() == reflect.Ptr {
+		typ = typ.Elem()
+	}
+	registry.types[tag] = typ
+	return nil
+}
+
+// MustRegister adds v to the type registry. It will panic if the tag returned
+// by v is already registered. This function is meant for use during program
+// initialization.
+func MustRegister(v Tagged) {
+	if err := register(v); err != nil {
+		panic(err)
+	}
+}
+
+type wrapper struct {
+	Type  string          `json:"type"`
+	Value json.RawMessage `json:"value"`
+}
+
+// Marshal marshals a JSON wrapper object containing v. If v == nil, Marshal
+// returns the JSON "null" value without error.
+func Marshal(v Tagged) ([]byte, error) {
+	if v == nil {
+		return []byte("null"), nil
+	}
+	data, err := json.Marshal(v)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(wrapper{
+		Type:  v.TypeTag(),
+		Value: data,
+	})
+}
+
+// Unmarshal unmarshals a JSON wrapper object into v. It reports an error if
+// the data do not encode a valid wrapper object, if the wrapper's type tag is
+// not registered with jsontypes, or if the resulting value is not compatible
+// with the type of v.
+func Unmarshal(data []byte, v interface{}) error {
+	// Verify that the target is some kind of pointer.
+	target := reflect.ValueOf(v)
+	if target.Kind() != reflect.Ptr {
+		return fmt.Errorf("target %T is not a pointer", v)
+	}
+
+	var w wrapper
+	dec := json.NewDecoder(bytes.NewReader(data))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&w); err != nil {
+		return fmt.Errorf("invalid type wrapper: %w", err)
+	}
+	typ, ok := registry.types[w.Type]
+	if !ok {
+		return fmt.Errorf("unknown type tag: %q", w.Type)
+	} else if !typ.AssignableTo(target.Elem().Type()) {
+		return fmt.Errorf("type %v not assignable to %T", typ, v)
+	}
+
+	obj := reflect.New(typ)
+	if err := json.Unmarshal(w.Value, obj.Interface()); err != nil {
+		return fmt.Errorf("decoding wrapped value: %w", err)
+	}
+	target.Elem().Set(obj.Elem())
+	return nil
+}

internal/jsontypes/jsontypes_test.go

@@ -0,0 +1,83 @@
+package jsontypes_test
+
+import (
+	"testing"
+
+	"github.com/tendermint/tendermint/internal/jsontypes"
+)
+
+type testType struct {
+	Field string `json:"field"`
+}
+
+func (*testType) TypeTag() string { return "test/TaggedType" }
+
+func TestRoundTrip(t *testing.T) {
+	const wantEncoded = `{"type":"test/TaggedType","value":{"field":"hello"}}`
+
+	t.Run("MustRegisterOK", func(t *testing.T) {
+		defer func() {
+			if x := recover(); x != nil {
+				t.Fatalf("Registration panicked: %v", x)
+			}
+		}()
+		jsontypes.MustRegister((*testType)(nil))
+	})
+
+	t.Run("MustRegisterFail", func(t *testing.T) {
+		defer func() {
+			if x := recover(); x != nil {
+				t.Logf("Got expected panic: %v", x)
+			}
+		}()
+		jsontypes.MustRegister((*testType)(nil))
+		t.Fatal("Registration should not have succeeded")
+	})
+
+	t.Run("MarshalNil", func(t *testing.T) {
+		bits, err := jsontypes.Marshal(nil)
+		if err != nil {
+			t.Fatalf("Marshal failed: %v", err)
+		}
+		if got := string(bits); got != "null" {
+			t.Errorf("Marshal nil: got %#q, want null", got)
+		}
+	})
+
+	t.Run("RoundTrip", func(t *testing.T) {
+		obj := testType{Field: "hello"}
+		bits, err := jsontypes.Marshal(&obj)
+		if err != nil {
+			t.Fatalf("Marshal %T failed: %v", obj, err)
+		}
+		if got := string(bits); got != wantEncoded {
+			t.Errorf("Marshal %T: got %#q, want %#q", obj, got, wantEncoded)
+		}
+
+		var cmp testType
+		if err := jsontypes.Unmarshal(bits, &cmp); err != nil {
+			t.Errorf("Unmarshal %#q failed: %v", string(bits), err)
+		}
+		if obj != cmp {
+			t.Errorf("Unmarshal %#q: got %+v, want %+v", string(bits), cmp, obj)
+		}
+	})
+
+	t.Run("Unregistered", func(t *testing.T) {
+		obj := testType{Field: "hello"}
+		bits, err := jsontypes.Marshal(&obj)
+		if err != nil {
+			t.Fatalf("Marshal %T failed: %v", obj, err)
+		}
+		if got := string(bits); got != wantEncoded {
+			t.Errorf("Marshal %T: got %#q, want %#q", obj, got, wantEncoded)
+		}
+
+		var cmp struct {
+			Field string `json:"field"`
+		}
+		if err := jsontypes.Unmarshal(bits, &cmp); err != nil {
+			t.Errorf("Unmarshal %#q: got %+v, want %+v", string(bits), cmp, obj)
+		}
+	})
+}

internal/p2p/conn/secret_connection_test.go

@@ -52,6 +52,7 @@ func (pk privKeyWithNilPubKey) Sign(msg []byte) ([]byte, error) { return pk.orig
 func (pk privKeyWithNilPubKey) PubKey() crypto.PubKey           { return nil }
 func (pk privKeyWithNilPubKey) Equals(pk2 crypto.PrivKey) bool  { return pk.orig.Equals(pk2) }
 func (pk privKeyWithNilPubKey) Type() string                    { return "privKeyWithNilPubKey" }
+func (privKeyWithNilPubKey) TypeTag() string                    { return "test/privKeyWithNilPubKey" }
 
 func TestSecretConnectionHandshake(t *testing.T) {
 	fooSecConn, barSecConn := makeSecretConnPair(t)

privval/file.go

@@ -14,6 +14,7 @@ import (
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/crypto/ed25519"
 	"github.com/tendermint/tendermint/crypto/secp256k1"
+	"github.com/tendermint/tendermint/internal/jsontypes"
 	"github.com/tendermint/tendermint/internal/libs/protoio"
 	"github.com/tendermint/tendermint/internal/libs/tempfile"
 	tmbytes "github.com/tendermint/tendermint/libs/bytes"
@@ -55,6 +56,22 @@ type FilePVKey struct {
 	filePath string
 }
 
+func (pvKey FilePVKey) MarshalJSON() ([]byte, error) {
+	pubk, err := jsontypes.Marshal(pvKey.PubKey)
+	if err != nil {
+		return nil, err
+	}
+	privk, err := jsontypes.Marshal(pvKey.PrivKey)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(struct {
+		Address types.Address   `json:"address"`
+		PubKey  json.RawMessage `json:"pub_key"`
+		PrivKey json.RawMessage `json:"priv_key"`
+	}{Address: pvKey.Address, PubKey: pubk, PrivKey: privk})
+}
+
 // Save persists the FilePVKey to its filePath.
 func (pvKey FilePVKey) Save() error {
 	outFile := pvKey.filePath