Merge branch 'main' into fix-conformance-tests-2

conformance/exemptions.txt

@@ -1,5 +1,4 @@
 Recommended.Proto2.JsonInput.FieldNameExtension.Validator
-Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInOptionalField.ProtobufOutput
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.ProtobufOutput

lib/protobuf/json.ex

@@ -121,8 +121,8 @@ defmodule Protobuf.JSON do
   """
   @spec encode!(struct, [encode_opt]) :: String.t() | no_return
   def encode!(struct, opts \\ []) do
-    case encode(struct, opts) do
-      {:ok, json} -> json
+    case encode_to_iodata(struct, opts) do
+      {:ok, iodata} -> IO.iodata_to_binary(iodata)
       {:error, error} -> raise error
     end
   end
@@ -169,11 +169,41 @@ defmodule Protobuf.JSON do
   """
   @spec encode(struct, [encode_opt]) ::
           {:ok, String.t()} | {:error, EncodeError.t() | Exception.t()}
-  def encode(%_{} = struct, opts \\ []) when is_list(opts) do
+  def encode(struct, opts \\ []) when is_list(opts) do
+    case encode_to_iodata(struct, opts) do
+      {:ok, iodata} -> {:ok, IO.iodata_to_binary(iodata)}
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @doc """
+  Similar to `encode!/2`, but returns iodata
+
+  See `encode_to_iodata/2` for more information about when this function should
+  be preferred over `encode!/2`.
+  """
+  @spec encode_to_iodata!(struct, [encode_opt]) :: iodata()
+  def encode_to_iodata!(struct, opts \\ []) when is_list(opts) do
+    case encode_to_iodata(struct, opts) do
+      {:ok, iodata} -> iodata
+      {:error, error} -> raise error
+    end
+  end
+
+  @doc """
+  Similar to `encode/2`, but returns iodata
+
+  This function should be preferred to encode/2, if the generated JSON will be
+  handed over to one of the IO functions or sent over the socket. The Erlang
+  runtime is able to leverage vectorised writes and avoid allocating a continuous
+  buffer for the whole resulting string, lowering memory use and increasing
+  performance.
+  """
+  @spec encode_to_iodata(struct, [encode_opt]) ::
+          {:ok, iodata()} | {:error, EncodeError.t() | Exception.t()}
+  def encode_to_iodata(%_{} = struct, opts \\ []) when is_list(opts) do
     if jason = load_jason() do
-      with {:ok, map} <- to_encodable(struct, opts) do
-        jason.encode(map)
-      end
+      with {:ok, map} <- to_encodable(struct, opts), do: jason.encode_to_iodata(map)
     else
       {:error, EncodeError.new(:no_json_lib)}
     end

lib/protobuf/json/encode.ex

@@ -159,22 +159,14 @@ defmodule Protobuf.JSON.Encode do
     :maps.from_list(regular ++ oneofs)
   end
 
-  defp encode_regular_fields(struct, %{field_props: field_props}, opts) do
+  defp encode_regular_fields(struct, %{field_props: field_props, syntax: syntax}, opts) do
     for {_field_num, %{name_atom: name, oneof: nil} = prop} <- field_props,
         %{^name => value} = struct,
-        emit?(prop, value) || opts[:emit_unpopulated] do
+        emit?(syntax, prop, value) || opts[:emit_unpopulated] do
       encode_field(prop, value, opts)
     end
   end
 
-  defp emit?(_prop, nil) do
-    false
-  end
-
-  defp emit?(prop, value) do
-    if default?(prop, value), do: prop.proto3_optional?, else: true
-  end
-
   defp encode_oneof_fields(struct, message_props, opts) do
     %{field_tags: field_tags, field_props: field_props, oneof: oneofs} = message_props
 
@@ -309,15 +301,18 @@ defmodule Protobuf.JSON.Encode do
   defp maybe_repeat(%{repeated?: false}, val, fun), do: fun.(val)
   defp maybe_repeat(%{repeated?: true}, val, fun), do: Enum.map(val, fun)
 
-  defp default?(_prop, +0.0), do: true
-  defp default?(_prop, nil), do: true
-  defp default?(_prop, 0), do: true
-  defp default?(_prop, false), do: true
-  defp default?(_prop, []), do: true
-  defp default?(_prop, ""), do: true
-  defp default?(_prop, %{} = map) when map_size(map) == 0, do: true
-  defp default?(%{type: {:enum, enum}}, key) when is_atom(key), do: enum.value(key) == 0
-  defp default?(_prop, _value), do: false
+  defp emit?(:proto2, %{default: value}, value), do: false
+  defp emit?(:proto2, %{optional?: true}, val), do: not is_nil(val)
+  defp emit?(:proto3, %{proto3_optional?: true}, val), do: not is_nil(val)
+  defp emit?(_syntax, _prop, +0.0), do: false
+  defp emit?(_syntax, _prop, nil), do: false
+  defp emit?(_syntax, _prop, 0), do: false
+  defp emit?(_syntax, _prop, false), do: false
+  defp emit?(_syntax, _prop, []), do: false
+  defp emit?(_syntax, _prop, ""), do: false
+  defp emit?(_syntax, _prop, %{} = map) when map_size(map) == 0, do: false
+  defp emit?(_syntax, %{type: {:enum, enum}}, key) when is_atom(key), do: enum.value(key) != 0
+  defp emit?(_syntax, _prop, _value), do: true
 
   defp transform_module(message, module) do
     if transform_module = module.transform_module() do

lib/protobuf/protoc/generator/message.ex

@@ -339,10 +339,10 @@ defmodule Protobuf.Protoc.Generator.Message do
   end
 
   # Omit `json_name` from the options list when it matches the original field
-  # name to keep the list small. Only Proto3 has JSON support for now.
-  defp add_json_name_to_opts(opts, :proto3, %{name: name, json_name: name}), do: opts
+  # name to keep the list small.
+  defp add_json_name_to_opts(opts, _, %{name: name, json_name: name}), do: opts
 
-  defp add_json_name_to_opts(opts, :proto3, %{json_name: json_name}),
+  defp add_json_name_to_opts(opts, _, %{json_name: json_name}),
     do: Keyword.put(opts, :json_name, json_name)
 
   defp add_json_name_to_opts(opts, _syntax, _props), do: opts

test/protobuf/json_test.exs

@@ -8,6 +8,12 @@ defmodule Protobuf.JSONTest do
     assert Protobuf.JSON.encode!(%TestMsg.Foo2{b: 10}) == ~S|{"b":"10"}|
   end
 
+  test "encode_to_iodata variants encode to iodata" do
+    assert iodata = Protobuf.JSON.encode_to_iodata!(%TestMsg.Foo2{b: 10})
+    assert {:ok, ^iodata} = Protobuf.JSON.encode_to_iodata(%TestMsg.Foo2{b: 10})
+    assert IO.iodata_to_binary(iodata) == ~S|{"b":"10"}|
+  end
+
   test "encoding string field with invalid UTF-8 data" do
     message = %Scalars{string: "   \xff   "}
     assert {:error, %Jason.EncodeError{}} = Protobuf.JSON.encode(message)

test/protobuf/protoc/generator/message_test.exs

@@ -731,27 +731,6 @@ defmodule Protobuf.Protoc.Generator.MessageTest do
       assert msg =~ "field :simple, 1, type: :int32\n"
       assert msg =~ "field :the_field_name, 2, type: :string, json_name: \"theFieldName\"\n"
     end
-
-    test "is omitted when syntax is not proto3" do
-      ctx = %Context{}
-
-      desc = %Google.Protobuf.DescriptorProto{
-        name: "Foo",
-        field: [
-          %Google.Protobuf.FieldDescriptorProto{
-            name: "the_field_name",
-            json_name: "theFieldName",
-            number: 1,
-            type: :TYPE_STRING,
-            label: :LABEL_REQUIRED
-          }
-        ]
-      }
-
-      {[], [{_mod, msg}]} = Generator.generate(ctx, desc)
-
-      assert msg =~ "field :the_field_name, 1, required: true, type: :string\n"
-    end
   end
 
   test "generate/2 repeated enum field" do