Fasttable: added support for varint fields.

upb/port/def.inc

@@ -89,7 +89,7 @@ Error, UINTPTR_MAX is undefined
 /* If we always read/write as a consistent type to each address, this shouldn't
  * violate aliasing.
  */
-#define UPB_PTR_AT(msg, ofs, type) ((type *)((char *)(msg) + (ofs)))
+#define UPB_PTR_AT(msg, ofs, type) ((type*)((char*)(msg) + (ofs)))
 
 // A flexible array member may have lower alignment requirements than the struct
 // overall - in that case, it can overlap with the trailing padding of the rest
@@ -392,6 +392,12 @@ Error, UINTPTR_MAX is undefined
 #define UPB_MUSTTAIL
 #endif
 
+#if UPB_HAS_ATTRIBUTE(preserve_most)
+#define UPB_PRESERVE_MOST __attribute__((preserve_most))
+#else
+#define UPB_PRESERVE_MOST
+#endif
+
 #if UPB_HAS_ATTRIBUTE(preserve_none)
 #define UPB_PRESERVE_NONE __attribute__((preserve_none))
 #else

upb/port/undef.inc

@@ -45,6 +45,7 @@
 #undef UPB_LONGJMP
 #undef UPB_PTRADD
 #undef UPB_MUSTTAIL
+#undef UPB_PRESERVE_MOST
 #undef UPB_PRESERVE_NONE
 #undef UPB_FASTTABLE_SUPPORTED
 #undef UPB_FASTTABLE_MASK

upb/wire/BUILD

@@ -72,7 +72,7 @@ cc_library(
         "//upb/port",
     ] + select({
         "//upb:fasttable_enabled_setting": [
-            "//upb/wire/decode_fast:dispatch",
+            "//upb/wire/decode_fast:field_parsers",
         ],
         "//conditions:default": [],
     }),

upb/wire/decode.c

@@ -1259,25 +1259,54 @@ const char* _upb_Decoder_DecodeFieldNoFast(upb_Decoder* d, const char* ptr,
   return ptr;
 }
 
+UPB_FORCEINLINE
+bool _upb_Decoder_TryDecodeMessageFast(upb_Decoder* d, const char** ptr,
+                                       upb_Message* msg,
+                                       const upb_MiniTable* mt,
+                                       uint64_t last_field_index,
+                                       uint64_t data) {
+#ifdef UPB_ENABLE_FASTTABLE
+  if (!mt || mt->UPB_PRIVATE(table_mask) == (unsigned char)-1 ||
+      (d->options & kUpb_DecodeOption_DisableFastTable)) {
+    // Fast table is unavailable or disabled.
+    return false;
+  }
+
+  intptr_t table = decode_totable(mt);
+  const char* start = *ptr;
+  char* trace_next = _upb_Decoder_TraceNext(d);
+
+  *ptr = upb_DecodeFast_Dispatch(d, *ptr, msg, table, 0, 0);
+
+  if (d->message_is_done) {
+    // The entire message was successfully parsed fast.
+    return true;
+  }
+
+  // *ptr now points to the beginning of a field that could not be parsed fast.
+  // It's possible that some fields were parsed fast, in which case *ptr will
+  // have been updated. However, it's also possible that the very first field
+  // encountered could not be parsed fast, in which case *ptr will be unchanged.
+  //
+  // If the fast decoder consumed any data, it must have emitted at least
+  // one 'F' event into the trace buffer (in addition to the 'D' event
+  // that is always emitted).
+  UPB_ASSERT(_upb_Decoder_TracePtr(d) != trace_next || *ptr == start);
+  _upb_Decoder_Trace(d, '<');
+#endif
+  return false;
+}
+
 UPB_FORCEINLINE
 const char* _upb_Decoder_DecodeField(upb_Decoder* d, const char* ptr,
                                      upb_Message* msg, const upb_MiniTable* mt,
                                      uint64_t last_field_index, uint64_t data) {
-#ifdef UPB_ENABLE_FASTTABLE
-  if (mt && mt->UPB_PRIVATE(table_mask) != (unsigned char)-1 &&
-      !(d->options & kUpb_DecodeOption_DisableFastTable)) {
-    intptr_t table = decode_totable(mt);
-    ptr = upb_DecodeFast_Dispatch(d, ptr, msg, table, 0, 0);
-    if (d->message_is_done) return ptr;
-    _upb_Decoder_Trace(d, '<');
+  if (_upb_Decoder_TryDecodeMessageFast(d, &ptr, msg, mt, last_field_index,
+                                        data)) {
+    return ptr;
   } else if (_upb_Decoder_IsDone(d, &ptr)) {
     return _upb_Decoder_EndMessage(d, ptr);
   }
-#else
-  if (_upb_Decoder_IsDone(d, &ptr)) {
-    return _upb_Decoder_EndMessage(d, ptr);
-  }
-#endif
 
   return _upb_Decoder_DecodeFieldNoFast(d, ptr, msg, mt);
 }

upb/wire/decode_fast/BUILD

@@ -5,6 +5,7 @@
 # license that can be found in the LICENSE file or at
 # https://developers.google.com/open-source/licenses/bsd
 
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
 load("@rules_cc//cc:cc_test.bzl", "cc_test")
 load("@rules_cc//cc:defs.bzl", "cc_library")
@@ -19,51 +20,62 @@ FASTTABLE_COPTS = UPB_DEFAULT_COPTS + select({
     "//conditions:default": [],
 })
 
+bool_flag(
+    name = "debug_trace",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "debug_trace_setting",
+    flag_values = {":debug_trace": "True"},
+)
+
+# Unfortunately field_parsers.h and dispatch.h need to be in the same library because of the
+# bug when calling an __attribute__((preserve_most)) function across a shared library boundary.
 cc_library(
     name = "field_parsers",
     srcs = [
         "cardinality.h",
+        "dispatch.c",
+        "dispatch.h",
         "field_fixed.c",
         "field_generic.c",
         "field_message.c",
         "field_string.c",
         "field_varint.c",
     ],
-    hdrs = ["field_parsers.h"],
+    hdrs = [
+        "dispatch.h",
+        "field_parsers.h",
+    ],
     copts = FASTTABLE_COPTS,
-    visibility = ["//upb:__pkg__"],
+    defines = select({
+        ":debug_trace_setting": ["UPB_TRACE_FASTDECODER=1"],
+        "//conditions:default": [],
+    }),
+    visibility = [
+        "//upb:__pkg__",
+        "//upb/wire:__pkg__",
+    ],
     deps = [
         ":combinations",
         ":data",
-        ":dispatch",
         "//upb/base",
         "//upb/base:internal",
         "//upb/mem",
         "//upb/message",
         "//upb/message:internal",
         "//upb/message:types",
         "//upb/mini_table",
-        "//upb/port",
-        "//upb/wire:decoder",
-        "//upb/wire:eps_copy_input_stream",
-        "//upb/wire:reader",
-    ],
-)
-
-cc_library(
-    name = "dispatch",
-    srcs = ["dispatch.c"],
-    hdrs = ["dispatch.h"],
-    copts = FASTTABLE_COPTS,
-    visibility = ["//upb/wire:__pkg__"],
-    deps = [
-        "//upb/message",
-        "//upb/mini_table",
         "//upb/mini_table:internal",
         "//upb/port",
         "//upb/wire:decoder",
         "//upb/wire:eps_copy_input_stream",
-    ],
+        "//upb/wire:reader",
+    ] + select({
+        ":debug_trace_setting": [":select"],
+        "//conditions:default": [],
+    }),
 )
 
 cc_library(

upb/wire/decode_fast/bisect.sh

@@ -13,13 +13,15 @@
 #
 #   $ third_party/upb/upb/wire/decode_fast/bisect.sh third_party/upb/upb/test:test_generated_code
 
-if [[ $# -ne 1 ]]; then
+if [[ $# -lt 1 ]]; then
   echo "Usage: bisect.sh [blaze test flags] <test_target(s)>"
   exit 1
 fi
 
-/google/data/ro/teams/tetralight/bin/bisect -low 0 -high 128 \
+set -ex
+
+/google/data/ro/teams/tetralight/bin/bisect -low 0 -high 256 \
   "blaze test --//third_party/upb:fasttable_enabled=True \
   --per_file_copt=//third_party/upb/upb/wire/decode_fast:select@-DUPB_DECODEFAST_DISABLE_FUNCTIONS_ABOVE=\$X \
   --host_per_file_copt=//third_party/upb/upb/wire/decode_fast:select@-DUPB_DECODEFAST_DISABLE_FUNCTIONS_ABOVE=\$X \
-  $@"
+  $*"