Buffer data if needed when using socket send nowait

lib/ssl/src/dtls_packet_demux.erl

@@ -351,7 +351,10 @@ setup_new_connection(User, From, Client, Msg, #state{dtls_processes = Processes,
                                                      session_id_tracker = Tracker,
 						     emulated_options = EmOpts} = State) ->
     ConnArgs = [server, "localhost", Port, {self(), {Client, Socket}},
-		{DTLSOpts, EmOpts, [{session_id_tracker, Tracker}]}, User, dtls_socket:default_cb_info()],
+		{DTLSOpts,
+                 emulated_opts_list(EmOpts, [mode, active], []),
+                 [{session_id_tracker, Tracker}]},
+                User, dtls_socket:default_cb_info()],
     case dtls_connection_sup:start_child(ConnArgs) of
 	{ok, Pid} ->
 	    erlang:monitor(process, Pid),

lib/ssl/src/dtls_socket.erl

@@ -42,6 +42,7 @@
 
 -export([emulated_options/0,
          emulated_options/1,
+         emulated_socket_options/2,
          internal_inet_values/0,
          default_inet_values/0,
          default_cb_info/0]).
@@ -61,9 +62,8 @@ listen(Port, #config{inet_ssl = SockOpts,
             dtls_packet_demux:new_owner(Listener, self()),
             dtls_packet_demux:set_all_opts(
               Listener, {Options,
-                          emulated_socket_options(EmOpts,
-                                                  #socket_options{}),
-                          SslOpts}),
+                         emulated_socket_options(EmOpts, #socket_options{}),
+                         SslOpts}),
             dtls_listener_sup:register_listener({self(), Listener},
                                                 IP, Port),
             {ok, create_dtls_socket(Config, Listener, Port)};
@@ -92,10 +92,8 @@ connect(Host, Port, #config{transport_info = CbInfo,
     case Transport:open(0, SocketOpts ++ internal_inet_values()) of
 	{ok, Socket} ->
 	    dtls_gen_connection:start_fsm(client, Host, Port, {{Host, Port}, Socket},
-                                          {SslOpts,
-                                           emulated_socket_options(EmOpts, #socket_options{}), undefined},
-                                          self(), CbInfo, Timeout);
-	{error, _} = Error->	
+                                          {SslOpts, EmOpts, undefined}, self(), CbInfo, Timeout);
+	{error, _} = Error->
 	    Error
     end.
 
@@ -152,7 +150,7 @@ socket([Pid], Transport, Socket, ConnectionCb, Tab) when Tab =/= undefined ->
 
 setopts(_, Socket = #sslsocket{socket_handle = {ListenPid, _},
                                listener_config = #config{}}, Options) ->
-    SplitOpts = {_, EmOpts} = tls_socket:split_options(Options),
+    SplitOpts = {_, EmOpts} = tls_socket:split_options(gen_udp, Options),
     check_active_n(EmOpts, Socket),
     dtls_packet_demux:set_sock_opts(ListenPid, SplitOpts);
 %%% Following clauses will not be called for emulated options, they are  handled in the connection process
@@ -189,7 +187,7 @@ check_active_n(EmulatedOpts, Socket = #sslsocket{socket_handle = {ListenPid, _},
     end.
 
 getopts(_, #sslsocket{socket_handle = {ListenPid, _}, listener_config =#config{}}, Options) ->
-    SplitOpts = tls_socket:split_options(Options),
+    SplitOpts = tls_socket:split_options(gen_udp, Options),
     dtls_packet_demux:get_sock_opts(ListenPid, SplitOpts);
 %%% Following clauses will not be called for emulated options, they are  handled in the connection process
 getopts(gen_udp, {_,{{_, _},Socket}}, Options) ->

lib/ssl/src/ssl.erl

@@ -3832,6 +3832,8 @@ add_filter(Filter, Filters) ->
     [Filter | Filters].
 
 -compile({inline, encode_packet/2}).
+encode_packet(0, Data) ->
+    Data;
 encode_packet(Packet, Data) ->
     Len = iolist_size(Data),
     case Packet of

lib/ssl/src/ssl_config.erl

@@ -77,7 +77,7 @@ handle_options(Socket, Opts, Role, Host) ->
     handle_options(Transport, Socket, Opts, Role, Host).
 
 %% Handle all options in listen, connect and handshake
-handle_options(Transport, Socket, Opts0, Role, Host) ->
+handle_options(Transport0, Socket, Opts0, Role, Host) ->
     {UserSslOptsList, SockOpts0} = split_options(Opts0, ssl_options()),
     NeedValidate = not (Socket == undefined) andalso Role =:= server, %% handshake options
     Env = #{role => Role, host => Host,
@@ -87,9 +87,13 @@ handle_options(Transport, Socket, Opts0, Role, Host) ->
 
     %% Handle special options
     #{protocol := Protocol} = SslOpts,
+    CbInfo = handle_option_cb_info(Opts0, Protocol, Socket),
+    Transport = case Transport0 of
+                    undefined -> element(1, CbInfo);
+                    _Else -> Transport0
+                end,
     {Sock, Emulated} = emulated_options(Transport, Socket, Protocol, SockOpts0),
     ConnetionCb = connection_cb(Protocol),
-    CbInfo = handle_option_cb_info(Opts0, Protocol, Socket),
 
     {ok, #config{
             ssl = SslOpts,
@@ -1740,15 +1744,15 @@ tls_validate_version_gap(Versions) ->
             Versions
     end.
 
-emulated_options(undefined, undefined, Protocol, Opts) ->
+emulated_options(Transport, undefined, Protocol, Opts) ->
     case Protocol of
 	tls ->
-	    tls_socket:emulated_options(Opts);
+	    tls_socket:emulated_options(Transport, Opts);
 	dtls ->
 	    dtls_socket:emulated_options(Opts)
     end;
 emulated_options(Transport, Socket, Protocol, Opts) ->
-    EmulatedOptions = tls_socket:emulated_options(),
+    EmulatedOptions = tls_socket:emulated_options(Transport),
     {ok, Inherited} = case Socket of
                           {'$socket', _} ->
                               %% This can't be set on a socket socket,
@@ -1760,12 +1764,11 @@ emulated_options(Transport, Socket, Protocol, Opts) ->
     Get = fun(Key) ->
                   {Key, proplists:get_value(Key, Opts, proplists:get_value(Key, Inherited))}
           end,
-    {Inet, _} = emulated_options(undefined, undefined, Protocol, Opts),
+    {Inet, _} = emulated_options(Transport, undefined, Protocol, Opts),
     Emulated = [Get(Key) || Key <- EmulatedOptions],
     {Inet, Emulated}.
 
-
-handle_cipher_option(Value, Versions)  when is_list(Value) ->       
+handle_cipher_option(Value, Versions)  when is_list(Value) ->
     try binary_cipher_suites(Versions, Value) of
 	Suites ->
 	    Suites

lib/ssl/src/ssl_gen_statem.erl

@@ -140,7 +140,7 @@ dtls_start_link(Role, Host, Port, Socket, {SslOpts, _, _} = Options, User, CbInf
 -spec init(list()) -> no_return().
 %% Description: Initialization
 %%--------------------------------------------------------------------
-init([Role, Sup | [Host, Port, _Socket, {TLSOpts, _, _}, _User, _CbInfo] = InitArgs]) ->
+init([Role, Sup, Host, Port, Socket, {TLSOpts, EmOpts, Trackers}, User, CbInfo]) ->
     process_flag(trap_exit, true),
 
     {ok, {_, Sender,_,_}} = supervisor:which_child(Sup, sender),
@@ -155,6 +155,11 @@ init([Role, Sup | [Host, Port, _Socket, {TLSOpts, _, _}, _User, _CbInfo] = InitA
     init_label(Role, Host, Port, TLSOpts),
     Tab = ets:new(tls_socket, []),
 
+    set_default_opts(Tab, EmOpts),
+
+    SocketOpts = tls_socket:emulated_socket_options(EmOpts, #socket_options{}),
+    InitArgs = [Host, Port, Socket, {TLSOpts, SocketOpts, Trackers}, User, CbInfo],
+
     case Role of
         ?CLIENT_ROLE ->
             case TLSOpts of
@@ -171,12 +176,16 @@ init([Role, Sup | [Host, Port, _Socket, {TLSOpts, _, _}, _User, _CbInfo] = InitA
                     tls_server_connection:init([Role, Sender, Tab|InitArgs])
             end
     end;
-init([Role | [Host, Port, _Socket, {DTLSOpts,_,_}, _User, _CbInfo] = InitArgs]) ->
+init([Role, Host, Port, Socket, {DTLSOpts,EmOpts,Trackers}, User, CbInfo]) ->
     process_flag(trap_exit, true),
 
     init_label(Role, Host, Port, DTLSOpts),
     Tab = ets:new(tls_socket, []),
 
+    SocketOpts = dtls_socket:emulated_socket_options(EmOpts, #socket_options{}),
+    Opts = {DTLSOpts, SocketOpts, Trackers},
+    InitArgs = [Host, Port, Socket, Opts, User, CbInfo],
+
     case Role of
         ?CLIENT_ROLE ->
             dtls_client_connection:init([Role, Tab|InitArgs]);
@@ -675,10 +684,16 @@ downgrade(internal, #alert{description = ?CLOSE_NOTIFY},
 	  #state{static_env = #static_env{transport_cb = Transport,
                                           socket = Socket},
 		 connection_env = #connection_env{downgrade = {Pid, From}},
-                 protocol_buffers = #protocol_buffers{tls_record_buffer = TlsRecordBuffer}
+                 protocol_buffers = #protocol_buffers{tls_record_buffer = TlsRecordBuffer},
+                 protocol_specific = PSpec
                 } = State) ->
-    tls_socket:setopts(Transport, Socket, [{active, false}, {packet, 0}, {mode, binary}]),
+    tls_socket:setopts(Transport, Socket, tls_socket:internal_inet_values(Transport)),
     Transport:controlling_process(Socket, Pid),
+
+    case maps:get(sel_info, PSpec, undefined) of
+        undefined -> ok;
+        SI -> _ = Transport:cancel(Socket, SI)
+    end,
     ReturnValue = case TlsRecordBuffer of
                       {undefined,{[Bin] = _Front, _Size, []}} ->
                           %% Buffered non TLS data returned to downgrade caller
@@ -801,8 +816,9 @@ handle_call({get_opts, OptTags}, From, _,
             #state{static_env = #static_env{protocol_cb = Connection,
                                             socket = Socket,
                                             transport_cb = Transport},
+                   tab = Tab,
                    socket_options = SockOpts}) ->
-    OptsReply = get_socket_opts(Connection, Transport, Socket, OptTags, SockOpts, []),
+    OptsReply = get_socket_opts(Connection, Transport, Socket, Tab, OptTags, SockOpts, []),
     {keep_state_and_data, [{reply, From, OptsReply}]};
 handle_call({set_opts, Opts0}, From, StateName,
 	    #state{static_env =  #static_env{user_socket = UserSocket,
@@ -1963,37 +1979,51 @@ record_cb(tls) ->
 record_cb(dtls) ->
     dtls_record.
 
-get_socket_opts(_, _,_,[], _, Acc) ->
+get_socket_opts(_, _,_,_, [], _, Acc) ->
     {ok, Acc};
-get_socket_opts(Connection, Transport, Socket, [mode | Tags], SockOpts, Acc) ->
-    get_socket_opts(Connection, Transport, Socket, Tags, SockOpts,
+get_socket_opts(Connection, Transport, Socket, Tab, [mode | Tags], SockOpts, Acc) ->
+    get_socket_opts(Connection, Transport, Socket, Tab, Tags, SockOpts,
 		    [{mode, SockOpts#socket_options.mode} | Acc]);
-get_socket_opts(Connection, Transport, Socket, [packet | Tags], SockOpts, Acc) ->
+get_socket_opts(Connection, Transport, Socket, Tab, [packet | Tags], SockOpts, Acc) ->
     case SockOpts#socket_options.packet of
 	{Type, headers} ->
-	    get_socket_opts(Connection, Transport, Socket, Tags, SockOpts, [{packet, Type} | Acc]);
+	    get_socket_opts(Connection, Transport, Socket, Tab, Tags, SockOpts, [{packet, Type} | Acc]);
 	Type ->
-	    get_socket_opts(Connection, Transport, Socket, Tags, SockOpts, [{packet, Type} | Acc])
+	    get_socket_opts(Connection, Transport, Socket, Tab, Tags, SockOpts, [{packet, Type} | Acc])
     end;
-get_socket_opts(Connection, Transport, Socket, [header | Tags], SockOpts, Acc) ->
-    get_socket_opts(Connection, Transport, Socket, Tags, SockOpts,
+get_socket_opts(Connection, Transport, Socket, Tab, [header | Tags], SockOpts, Acc) ->
+    get_socket_opts(Connection, Transport, Socket, Tab, Tags, SockOpts,
 		    [{header, SockOpts#socket_options.header} | Acc]);
-get_socket_opts(Connection, Transport, Socket, [active | Tags], SockOpts, Acc) ->
-    get_socket_opts(Connection, Transport, Socket, Tags, SockOpts,
+get_socket_opts(Connection, Transport, Socket, Tab, [active | Tags], SockOpts, Acc) ->
+    get_socket_opts(Connection, Transport, Socket, Tab, Tags, SockOpts,
 		    [{active, SockOpts#socket_options.active} | Acc]);
-get_socket_opts(Connection, Transport, Socket, [packet_size | Tags], SockOpts, Acc) ->
-    get_socket_opts(Connection, Transport, Socket, Tags, SockOpts,
+get_socket_opts(Connection, Transport, Socket, Tab, [packet_size | Tags], SockOpts, Acc) ->
+    get_socket_opts(Connection, Transport, Socket, Tab, Tags, SockOpts,
 		    [{packet_size, SockOpts#socket_options.packet_size} | Acc]);
-get_socket_opts(Connection, Transport, Socket, [Tag | Tags], SockOpts, Acc) ->
+get_socket_opts(tls_gen_connection, tls_socket_tcp, Socket, Tab, [high_watermark | Tags], SockOpts, Acc) ->
+    Emulated = try ets:lookup_element(Tab, high_watermark, 2) of
+                   Val -> Val
+               catch _:_ -> 8196
+               end,
+    get_socket_opts(tls_gen_connection, tls_socket_tcp, Socket, Tab, Tags, SockOpts,
+		    [{high_watermark, Emulated} | Acc]);
+get_socket_opts(tls_gen_connection, tls_socket_tcp, Socket, Tab, [low_watermark | Tags], SockOpts, Acc) ->
+    Emulated = try ets:lookup_element(Tab, low_watermark, 2) of
+                   Val -> Val
+               catch _:_ -> 4096
+               end,
+    get_socket_opts(tls_gen_connection, tls_socket_tcp, Socket, Tab, Tags, SockOpts,
+		    [{low_watermark, Emulated} | Acc]);
+get_socket_opts(Connection, Transport, Socket, Tab, [Tag | Tags], SockOpts, Acc) ->
     case Connection:getopts(Transport, Socket, [Tag]) of
         {ok, [Opt]} ->
-            get_socket_opts(Connection, Transport, Socket, Tags, SockOpts, [Opt | Acc]);
+            get_socket_opts(Connection, Transport, Socket, Tab, Tags, SockOpts, [Opt | Acc]);
         {ok, []} ->
-            get_socket_opts(Connection, Transport, Socket, Tags, SockOpts, Acc);
+            get_socket_opts(Connection, Transport, Socket, Tab, Tags, SockOpts, Acc);
         {error, Reason} ->
             {error, {options, {socket_options, Tag, Reason}}}
     end;
-get_socket_opts(_,_, _,Opts, _,_) ->
+get_socket_opts(_, _, _, _, Opts, _, _) ->
     {error, {options, {socket_options, Opts, function_clause}}}.
 
 set_socket_opts(_,_,_, _Tab, [], SockOpts, []) ->
@@ -2067,6 +2097,25 @@ set_socket_opts(tls_gen_connection, Transport, Socket, Tab, [{packet, Packet}| O
     true = ets:insert(Tab, {{socket_options, packet}, Packet}),
     set_socket_opts(tls_gen_connection, Transport, Socket, Tab, Opts,
 		    SockOpts#socket_options{packet = Packet}, Other);
+
+set_socket_opts(tls_gen_connection, tls_socket_tcp, Socket, Tab,
+                [{high_watermark, Sz}=Opt|Opts], SockOpts, Other) ->
+    case is_integer(Sz) of
+        true ->
+            true = ets:insert(Tab, {high_watermark, Sz}),
+            set_socket_opts(tls_gen_connection, tls_socket_tcp, Socket, Tab, Opts, SockOpts, Other);
+        false ->
+            {{error,{options, {socket_options, Opt}}}, SockOpts}
+    end;
+set_socket_opts(tls_gen_connection, tls_socket_tcp, Socket, Tab,
+                [{low_watermark, Sz}=Opt|Opts], SockOpts, Other) ->
+    case is_integer(Sz) of
+        true ->
+            true = ets:insert(Tab, {low_watermark, Sz}),
+            set_socket_opts(tls_gen_connection, tls_socket_tcp, Socket, Tab, Opts, SockOpts, Other);
+        false ->
+            {{error,{options, {socket_options, Opt}}}, SockOpts}
+    end;
 set_socket_opts(_, _, _, _Tab, [{packet, _} = Opt| _], SockOpts, _) ->
     {{error, {options, {socket_options, Opt}}}, SockOpts};
 set_socket_opts(tls_gen_connection, Transport, Socket, Tab, [{header, Header}| Opts], SockOpts, Other)
@@ -2110,6 +2159,17 @@ ssl_options_list([{ciphers = Key, Value}|T], Acc) ->
 ssl_options_list([{Key, Value}|T], Acc) ->
     ssl_options_list(T, [{Key, Value} | Acc]).
 
+set_default_opts(Tab, EmOpts) ->
+    case proplists:get_value(high_watermark, EmOpts, undefined) of
+        undefined -> ok;
+        High ->
+            Low = proplists:get_value(low_watermark, EmOpts, undefined),
+            true = Low =/= undefined,  %% If High exists low exists
+            true = ets:insert(Tab, {high_watermark, High}),
+            true = ets:insert(Tab, {low_watermark, Low}),
+            ok
+    end.
+
 keylog_1_3(Info) ->
     {client_random, ClientRandomBin} = lists:keyfind(client_random, 1, Info),
     {selected_cipher_suite, #{prf := Prf}} = lists:keyfind(selected_cipher_suite, 1, Info),

lib/ssl/src/tls_dtls_gen_connection.erl

@@ -145,9 +145,7 @@ initial_state(Role, Sender, Tab, Host, Port, Socket, {SSLOptions, SocketOptions,
     %% Use highest supported version for client/server random nonce generation
     #{versions := [Version|_]} = SSLOptions,
     BeastMitigation = maps:get(beast_mitigation, SSLOptions, disabled),
-    ConnectionStates = tls_record:init_connection_states(Role,
-                                                         Version,
-                                                         BeastMitigation),
+    ConnectionStates = tls_record:init_connection_states(Role, Version, BeastMitigation),
     #{session_cb := SessionCacheCb} = ssl_config:pre_1_3_session_opts(Role),
     UserMonitor = erlang:monitor(process, User),