forward-fit: Allow setting grpc send and recv message sizes independently (#481)

go/flags/endtoend/topo2topo.txt

@@ -22,6 +22,8 @@ Flags:
       --from_root string                                            topology server root to copy data from
       --from_server string                                          topology server address to copy data from
       --grpc_enable_tracing                                         Enable gRPC tracing.
+      --grpc_max_message_recv_size int                              Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int                              Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int                                   Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_prometheus                                             Enable gRPC monitoring with Prometheus.
   -h, --help                                                        help for topo2topo

go/flags/endtoend/vtbackup.txt

@@ -135,6 +135,8 @@ Flags:
       --grpc_initial_window_size int                                gRPC initial window size
       --grpc_keepalive_time duration                                After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s)
       --grpc_keepalive_timeout duration                             After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s)
+      --grpc_max_message_recv_size int                              Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int                              Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int                                   Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_prometheus                                             Enable gRPC monitoring with Prometheus.
   -h, --help                                                        help for vtbackup

go/flags/endtoend/vtcombo.txt

@@ -156,6 +156,8 @@ Flags:
       --grpc_key string                                                  server private key to use for gRPC connections, requires grpc_cert, enables TLS
       --grpc_max_connection_age duration                                 Maximum age of a client connection before GoAway is sent. (default 2562047h47m16.854775807s)
       --grpc_max_connection_age_grace duration                           Additional grace period after grpc_max_connection_age, after which connections are forcibly closed. (default 2562047h47m16.854775807s)
+      --grpc_max_message_recv_size int                                   Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int                                   Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int                                        Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_port int                                                    Port to listen on for gRPC calls. If zero, do not listen.
       --grpc_prometheus                                                  Enable gRPC monitoring with Prometheus.

go/flags/endtoend/vtctlclient.txt

@@ -16,6 +16,8 @@ Usage of vtctlclient:
       --grpc_initial_window_size int                                gRPC initial window size
       --grpc_keepalive_time duration                                After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s)
       --grpc_keepalive_timeout duration                             After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s)
+      --grpc_max_message_recv_size int                              Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int                              Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int                                   Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_prometheus                                             Enable gRPC monitoring with Prometheus.
   -h, --help                                                        display usage and exit

go/flags/endtoend/vtctld.txt

@@ -75,6 +75,8 @@ Flags:
       --grpc_key string                                                  server private key to use for gRPC connections, requires grpc_cert, enables TLS
       --grpc_max_connection_age duration                                 Maximum age of a client connection before GoAway is sent. (default 2562047h47m16.854775807s)
       --grpc_max_connection_age_grace duration                           Additional grace period after grpc_max_connection_age, after which connections are forcibly closed. (default 2562047h47m16.854775807s)
+      --grpc_max_message_recv_size int                                   Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int                                   Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int                                        Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_port int                                                    Port to listen on for gRPC calls. If zero, do not listen.
       --grpc_prometheus                                                  Enable gRPC monitoring with Prometheus.

go/flags/endtoend/vtctldclient.txt

@@ -115,6 +115,8 @@ Flags:
       --grpc_initial_window_size int           gRPC initial window size
       --grpc_keepalive_time duration           After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s)
       --grpc_keepalive_timeout duration        After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s)
+      --grpc_max_message_recv_size int         Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int         Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int              Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_prometheus                        Enable gRPC monitoring with Prometheus.
   -h, --help                                   help for vtctldclient

go/flags/endtoend/vtgate.txt

@@ -88,6 +88,8 @@ Flags:
       --grpc_key string                                                  server private key to use for gRPC connections, requires grpc_cert, enables TLS
       --grpc_max_connection_age duration                                 Maximum age of a client connection before GoAway is sent. (default 2562047h47m16.854775807s)
       --grpc_max_connection_age_grace duration                           Additional grace period after grpc_max_connection_age, after which connections are forcibly closed. (default 2562047h47m16.854775807s)
+      --grpc_max_message_recv_size int                                   Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int                                   Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int                                        Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_port int                                                    Port to listen on for gRPC calls. If zero, do not listen.
       --grpc_prometheus                                                  Enable gRPC monitoring with Prometheus.

go/flags/endtoend/vtgateclienttest.txt

@@ -32,6 +32,8 @@ Flags:
       --grpc_key string                                                  server private key to use for gRPC connections, requires grpc_cert, enables TLS
       --grpc_max_connection_age duration                                 Maximum age of a client connection before GoAway is sent. (default 2562047h47m16.854775807s)
       --grpc_max_connection_age_grace duration                           Additional grace period after grpc_max_connection_age, after which connections are forcibly closed. (default 2562047h47m16.854775807s)
+      --grpc_max_message_recv_size int                                   Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int                                   Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int                                        Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_port int                                                    Port to listen on for gRPC calls. If zero, do not listen.
       --grpc_prometheus                                                  Enable gRPC monitoring with Prometheus.

go/flags/endtoend/vtorc.txt

@@ -42,6 +42,8 @@ Flags:
       --grpc_initial_window_size int                                gRPC initial window size
       --grpc_keepalive_time duration                                After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s)
       --grpc_keepalive_timeout duration                             After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s)
+      --grpc_max_message_recv_size int                              Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int                              Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int                                   Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_prometheus                                             Enable gRPC monitoring with Prometheus.
   -h, --help                                                        help for vtorc

go/flags/endtoend/vttablet.txt

@@ -185,6 +185,8 @@ Flags:
       --grpc_key string                                                  server private key to use for gRPC connections, requires grpc_cert, enables TLS
       --grpc_max_connection_age duration                                 Maximum age of a client connection before GoAway is sent. (default 2562047h47m16.854775807s)
       --grpc_max_connection_age_grace duration                           Additional grace period after grpc_max_connection_age, after which connections are forcibly closed. (default 2562047h47m16.854775807s)
+      --grpc_max_message_recv_size int                                   Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int                                   Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int                                        Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_port int                                                    Port to listen on for gRPC calls. If zero, do not listen.
       --grpc_prometheus                                                  Enable gRPC monitoring with Prometheus.

go/flags/endtoend/vttestserver.txt

@@ -62,6 +62,8 @@ Flags:
       --grpc_key string                                                  server private key to use for gRPC connections, requires grpc_cert, enables TLS
       --grpc_max_connection_age duration                                 Maximum age of a client connection before GoAway is sent. (default 2562047h47m16.854775807s)
       --grpc_max_connection_age_grace duration                           Additional grace period after grpc_max_connection_age, after which connections are forcibly closed. (default 2562047h47m16.854775807s)
+      --grpc_max_message_recv_size int                                   Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.
+      --grpc_max_message_send_size int                                   Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.
       --grpc_max_message_size int                                        Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_port int                                                    Port to listen on for gRPC calls. If zero, do not listen.
       --grpc_prometheus                                                  Enable gRPC monitoring with Prometheus.

go/vt/grpcclient/client.go

@@ -107,11 +107,12 @@ func Dial(target string, failFast FailFast, opts ...grpc.DialOption) (*grpc.Clie
 // failFast is a non-optional parameter because callers are required to specify
 // what that should be.
 func DialContext(ctx context.Context, target string, failFast FailFast, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
-	msgSize := grpccommon.MaxMessageSize()
+	maxSendSize := grpccommon.MaxMessageSendSize()
+	maxRecvSize := grpccommon.MaxMessageRecvSize()
 	newopts := []grpc.DialOption{
 		grpc.WithDefaultCallOptions(
-			grpc.MaxCallRecvMsgSize(msgSize),
-			grpc.MaxCallSendMsgSize(msgSize),
+			grpc.MaxCallRecvMsgSize(maxRecvSize),
+			grpc.MaxCallSendMsgSize(maxSendSize),
 			grpc.WaitForReady(bool(!failFast)),
 		),
 	}

go/vt/grpccommon/options.go

@@ -28,6 +28,10 @@ var (
 	// accept. Larger messages will be rejected.
 	// Note: We're using 16 MiB as default value because that's the default in MySQL
 	maxMessageSize = 16 * 1024 * 1024
+	// These options override maxMessageSize if > 0, allowing us to control the max
+	// size sending independently from receiving.
+	maxMsgRecvSize = 0
+	maxMsgSendSize = 0
 	// enablePrometheus sets a flag to enable grpc client/server grpc monitoring.
 	enablePrometheus bool
 )
@@ -39,6 +43,8 @@ var (
 // command-line arguments.
 func RegisterFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&maxMessageSize, "grpc_max_message_size", maxMessageSize, "Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'.")
+	fs.IntVar(&maxMsgSendSize, "grpc_max_message_send_size", maxMsgSendSize, "Maximum allowed RPC message size when sending. If 0, defaults to grpc_max_message_size.")
+	fs.IntVar(&maxMsgRecvSize, "grpc_max_message_recv_size", maxMsgRecvSize, "Maximum allowed RPC message size when receiving. If 0, defaults to grpc_max_message_size.")
 	fs.BoolVar(&grpc.EnableTracing, "grpc_enable_tracing", grpc.EnableTracing, "Enable gRPC tracing.")
 	fs.BoolVar(&enablePrometheus, "grpc_prometheus", enablePrometheus, "Enable gRPC monitoring with Prometheus.")
 }
@@ -53,6 +59,20 @@ func MaxMessageSize() int {
 	return maxMessageSize
 }
 
+func MaxMessageRecvSize() int {
+	if maxMsgRecvSize > 0 {
+		return maxMsgRecvSize
+	}
+	return MaxMessageSize()
+}
+
+func MaxMessageSendSize() int {
+	if maxMsgSendSize > 0 {
+		return maxMsgSendSize
+	}
+	return MaxMessageSize()
+}
+
 func init() {
 	stats.NewString("GrpcVersion").Set(grpc.Version)
 }

go/vt/servenv/grpc_server.go

@@ -209,10 +209,12 @@ func createGRPCServer() {
 	// grpc: received message length XXXXXXX exceeding the max size 4194304
 	// Note: For gRPC 1.0.0 it's sufficient to set the limit on the server only
 	// because it's not enforced on the client side.
-	msgSize := grpccommon.MaxMessageSize()
-	log.Infof("Setting grpc max message size to %d", msgSize)
-	opts = append(opts, grpc.MaxRecvMsgSize(msgSize))
-	opts = append(opts, grpc.MaxSendMsgSize(msgSize))
+
+	maxSendSize := grpccommon.MaxMessageSendSize()
+	maxRecvSize := grpccommon.MaxMessageRecvSize()
+	log.Infof("Setting grpc server max message sizes to %d (sending), %d (receiving)", maxSendSize, maxRecvSize)
+	opts = append(opts, grpc.MaxRecvMsgSize(maxRecvSize))
+	opts = append(opts, grpc.MaxSendMsgSize(maxSendSize))
 
 	if gRPCInitialConnWindowSize != 0 {
 		log.Infof("Setting grpc server initial conn window size to %d", int32(gRPCInitialConnWindowSize))