From 2037e41ae3c89874d0dfd5d5a5ceaa63a357c223 Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Wed, 27 Mar 2024 12:58:53 -0700 Subject: [PATCH] Add first "known type" implementation This adds the first "known type" implementation, uint64value. We'll start adding other types later. For now, generated protobuf code will require the known type from the protoboeuf gem. We should add a flag to support generating "stand alone" `.rb` files (which should be easy enough since we have the source .proto files) --- Rakefile | 16 ++- lib/protoboeuf/codegen.rb | 15 ++- lib/protoboeuf/parser.rb | 4 + lib/protoboeuf/protobuf/uint64value.proto | 7 ++ lib/protoboeuf/protobuf/uint64value.rb | 147 ++++++++++++++++++++++ test/fixtures/test.proto | 4 + test/helper.rb | 2 + test/message_test.rb | 10 ++ 8 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 lib/protoboeuf/protobuf/uint64value.proto create mode 100644 lib/protoboeuf/protobuf/uint64value.rb diff --git a/Rakefile b/Rakefile index bf11865..15c242c 100644 --- a/Rakefile +++ b/Rakefile @@ -8,11 +8,25 @@ rb_files = proto_files.pathmap("#{BASE_DIR}/lib/proto/test/fixtures/%n_pb.rb") BENCHMARK_UPSTREAM_PB = "bench/lib/upstream/benchmark_pb.rb" BENCHMARK_PROTOBOEUF_PB = "bench/lib/protoboeuf/benchmark_pb.rb" +well_known_types = Rake::FileList[File.join(BASE_DIR, "lib/protoboeuf/protobuf/*.proto")] + # Clobber/clean rules rb_files.each { |x| CLOBBER.append x } CLOBBER.append BENCHMARK_UPSTREAM_PB CLOBBER.append BENCHMARK_PROTOBOEUF_PB +rule ".rb" => "%X.proto" do |t| + require_relative "lib/protoboeuf/codegen" + require_relative "lib/protoboeuf/parser" + + unit = ProtoBoeuf.parse_file t.source + + puts "writing #{t.name}" + File.binwrite t.name, unit.to_ruby +end + +task :well_known_types => well_known_types.pathmap("%X.rb") + # Makefile-like rule to generate "_pb.rb" rule "_pb.rb" => "test/fixtures/%{_pb,}n.proto" do |task| mkdir_p "lib/proto" @@ -55,7 +69,7 @@ end desc "Regenerate protobuf files" task :gen_proto => rb_files -task :test => :gen_proto +task :test => [:gen_proto, :well_known_types] task :default => :test task :bench => [BENCHMARK_UPSTREAM_PB, BENCHMARK_PROTOBOEUF_PB] do diff --git a/lib/protoboeuf/codegen.rb b/lib/protoboeuf/codegen.rb index c88a39e..4de6abd 100644 --- a/lib/protoboeuf/codegen.rb +++ b/lib/protoboeuf/codegen.rb @@ -55,6 +55,7 @@ def initialize(message, toplevel_enums) @optional_field_bit_lut = [] @fields = @message.fields @enum_field_types = toplevel_enums.merge(message.enums.group_by(&:name)) + @requires = Set.new field_types = message.fields.group_by { |field| if field.field? @@ -74,7 +75,12 @@ def initialize(message, toplevel_enums) end def result - "class #{message.name}\n" + class_body + "end\n" + body = "class #{message.name}\n" + class_body + "end\n" + if @requires.empty? + body + else + @requires.map { |r| "require #{r.dump}" }.join("\n") + "\n\n" + body + end end private @@ -653,6 +659,11 @@ def decode_repeated(field) end def pull_message(type, dest, operator) + if type == "google.protobuf.UInt64Value" + @requires << "protoboeuf/protobuf/uint64value" + type = "ProtoBoeuf::Protobuf::UInt64Value" + end + " ## PULL_MESSAGE\n" + pull_uint64("msg_len", "=") + "\n" + " #{dest} #{operator} #{type}.allocate.decode_from(buff, index, index += msg_len)\n" + @@ -772,7 +783,7 @@ def to_ruby tail = "\n" + packages.map { unindent.call + "end" }.join("\n") - head + body + tail + (head + body + tail).gsub(/[ ]*(?=$)/, '') end end end diff --git a/lib/protoboeuf/parser.rb b/lib/protoboeuf/parser.rb index 7d7945b..10a7941 100644 --- a/lib/protoboeuf/parser.rb +++ b/lib/protoboeuf/parser.rb @@ -50,6 +50,10 @@ class Unit < Struct.new(:package, :options, :imports, :messages, :enums) def accept(viz) viz.visit_unit self end + + def to_ruby + ProtoBoeuf::CodeGen.new(self).to_ruby + end end class Option < Struct.new(:name, :value, :pos) diff --git a/lib/protoboeuf/protobuf/uint64value.proto b/lib/protoboeuf/protobuf/uint64value.proto new file mode 100644 index 0000000..1342394 --- /dev/null +++ b/lib/protoboeuf/protobuf/uint64value.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package proto_boeuf.protobuf; + +message UInt64Value { + uint64 value = 1; +} diff --git a/lib/protoboeuf/protobuf/uint64value.rb b/lib/protoboeuf/protobuf/uint64value.rb new file mode 100644 index 0000000..2eff0b0 --- /dev/null +++ b/lib/protoboeuf/protobuf/uint64value.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +module ProtoBoeuf + module Protobuf + + class UInt64Value + def self.decode(buff) + buff = buff.dup + buff.force_encoding("UTF-8") + allocate.decode_from(buff, 0, buff.bytesize) + end + + # required field readers + attr_accessor :value + + # enum readers + + # enum writers + + def initialize(value: 0) + @value = value + end + + def decode_from(buff, index, len) + + @value = 0 + + tag = buff.getbyte(index) + index += 1 + + + while true + if tag == 0x8 + ## PULL_UINT64 + @value = if (byte0 = buff.getbyte(index)) < 0x80 + index += 1 + byte0 + else + if (byte1 = buff.getbyte(index + 1)) < 0x80 + index += 2 + (byte1 << 7) | (byte0 & 0x7F) + else + if (byte2 = buff.getbyte(index + 2)) < 0x80 + index += 3 + (byte2 << 14) | + ((byte1 & 0x7F) << 7) | + (byte0 & 0x7F) + else + if (byte3 = buff.getbyte(index + 3)) < 0x80 + index += 4 + (byte3 << 21) | + ((byte2 & 0x7F) << 14) | + ((byte1 & 0x7F) << 7) | + (byte0 & 0x7F) + else + if (byte4 = buff.getbyte(index + 4)) < 0x80 + index += 5 + (byte4 << 28) | + ((byte3 & 0x7F) << 21) | + ((byte2 & 0x7F) << 14) | + ((byte1 & 0x7F) << 7) | + (byte0 & 0x7F) + else + if (byte5 = buff.getbyte(index + 5)) < 0x80 + index += 6 + (byte5 << 35) | + ((byte4 & 0x7F) << 28) | + ((byte3 & 0x7F) << 21) | + ((byte2 & 0x7F) << 14) | + ((byte1 & 0x7F) << 7) | + (byte0 & 0x7F) + else + if (byte6 = buff.getbyte(index + 6)) < 0x80 + index += 7 + (byte6 << 42) | + ((byte5 & 0x7F) << 35) | + ((byte4 & 0x7F) << 28) | + ((byte3 & 0x7F) << 21) | + ((byte2 & 0x7F) << 14) | + ((byte1 & 0x7F) << 7) | + (byte0 & 0x7F) + else + if (byte7 = buff.getbyte(index + 7)) < 0x80 + index += 8 + (byte7 << 49) | + ((byte6 & 0x7F) << 42) | + ((byte5 & 0x7F) << 35) | + ((byte4 & 0x7F) << 28) | + ((byte3 & 0x7F) << 21) | + ((byte2 & 0x7F) << 14) | + ((byte1 & 0x7F) << 7) | + (byte0 & 0x7F) + else + if (byte8 = buff.getbyte(index + 8)) < 0x80 + index += 9 + (byte8 << 56) | + ((byte7 & 0x7F) << 49) | + ((byte6 & 0x7F) << 42) | + ((byte5 & 0x7F) << 35) | + ((byte4 & 0x7F) << 28) | + ((byte3 & 0x7F) << 21) | + ((byte2 & 0x7F) << 14) | + ((byte1 & 0x7F) << 7) | + (byte0 & 0x7F) + else + if (byte9 = buff.getbyte(index + 9)) < 0x80 + index += 10 + + (byte9 << 63) | + ((byte8 & 0x7F) << 56) | + ((byte7 & 0x7F) << 49) | + ((byte6 & 0x7F) << 42) | + ((byte5 & 0x7F) << 35) | + ((byte4 & 0x7F) << 28) | + ((byte3 & 0x7F) << 21) | + ((byte2 & 0x7F) << 14) | + ((byte1 & 0x7F) << 7) | + (byte0 & 0x7F) + else + raise "integer decoding error" + end + end + end + end + end + end + end + end + end + end + + ## END PULL_UINT64 + + + return self if index >= len + tag = buff.getbyte(index) + index += 1 + + end + + return self if index >= len + raise NotImplementedError + end + end + end + end +end \ No newline at end of file diff --git a/test/fixtures/test.proto b/test/fixtures/test.proto index f9d8dd8..dd101e7 100644 --- a/test/fixtures/test.proto +++ b/test/fixtures/test.proto @@ -210,3 +210,7 @@ message HasSubEnum { Thing a = 1; } + +message HasKnownValue { + google.protobuf.UInt64Value id = 1; +} diff --git a/test/helper.rb b/test/helper.rb index b30b8d8..4ce1995 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -1,3 +1,5 @@ +ENV["MT_NO_PLUGINS"] = "1" + require "minitest/autorun" require "minitest/color" require "proto/test/fixtures/test_pb" diff --git a/test/message_test.rb b/test/message_test.rb index 25178d3..878f6ee 100644 --- a/test/message_test.rb +++ b/test/message_test.rb @@ -471,4 +471,14 @@ def test_subenum_methods assert_equal expected.a, actual.a end + + def test_translate_known_types_in_proto + data = ::HasKnownValue.encode(::HasKnownValue.new.tap { |x| + x.id = Google::Protobuf::UInt64Value.new(value: 123456) + }) + + instance = HasKnownValue.decode(data) + assert_kind_of ::ProtoBoeuf::Protobuf::UInt64Value, instance.id + assert_equal 123456, instance.id.value + end end