diff --git a/upb/port/def.inc b/upb/port/def.inc index 9dec28187d1e0..c3bbe541efaa2 100644 --- a/upb/port/def.inc +++ b/upb/port/def.inc @@ -89,7 +89,7 @@ Error, UINTPTR_MAX is undefined /* If we always read/write as a consistent type to each address, this shouldn't * violate aliasing. */ -#define UPB_PTR_AT(msg, ofs, type) ((type *)((char *)(msg) + (ofs))) +#define UPB_PTR_AT(msg, ofs, type) ((type*)((char*)(msg) + (ofs))) // A flexible array member may have lower alignment requirements than the struct // overall - in that case, it can overlap with the trailing padding of the rest @@ -392,6 +392,12 @@ Error, UINTPTR_MAX is undefined #define UPB_MUSTTAIL #endif +#if UPB_HAS_ATTRIBUTE(preserve_most) +#define UPB_PRESERVE_MOST __attribute__((preserve_most)) +#else +#define UPB_PRESERVE_MOST +#endif + #if UPB_HAS_ATTRIBUTE(preserve_none) #define UPB_PRESERVE_NONE __attribute__((preserve_none)) #else diff --git a/upb/port/undef.inc b/upb/port/undef.inc index 56121c73b5356..1fc468aa7dbd1 100644 --- a/upb/port/undef.inc +++ b/upb/port/undef.inc @@ -45,6 +45,7 @@ #undef UPB_LONGJMP #undef UPB_PTRADD #undef UPB_MUSTTAIL +#undef UPB_PRESERVE_MOST #undef UPB_PRESERVE_NONE #undef UPB_FASTTABLE_SUPPORTED #undef UPB_FASTTABLE_MASK diff --git a/upb/wire/BUILD b/upb/wire/BUILD index f1595879e7d12..dec09ad2abd19 100644 --- a/upb/wire/BUILD +++ b/upb/wire/BUILD @@ -72,7 +72,7 @@ cc_library( "//upb/port", ] + select({ "//upb:fasttable_enabled_setting": [ - "//upb/wire/decode_fast:dispatch", + "//upb/wire/decode_fast:field_parsers", ], "//conditions:default": [], }), diff --git a/upb/wire/decode.c b/upb/wire/decode.c index 2ee2b1ea774b4..421f1f3804b3d 100644 --- a/upb/wire/decode.c +++ b/upb/wire/decode.c @@ -1259,25 +1259,54 @@ const char* _upb_Decoder_DecodeFieldNoFast(upb_Decoder* d, const char* ptr, return ptr; } +UPB_FORCEINLINE +bool _upb_Decoder_TryDecodeMessageFast(upb_Decoder* d, const char** ptr, + upb_Message* msg, + const upb_MiniTable* mt, + uint64_t last_field_index, + uint64_t data) { +#ifdef UPB_ENABLE_FASTTABLE + if (!mt || mt->UPB_PRIVATE(table_mask) == (unsigned char)-1 || + (d->options & kUpb_DecodeOption_DisableFastTable)) { + // Fast table is unavailable or disabled. + return false; + } + + intptr_t table = decode_totable(mt); + const char* start = *ptr; + char* trace_next = _upb_Decoder_TraceNext(d); + + *ptr = upb_DecodeFast_Dispatch(d, *ptr, msg, table, 0, 0); + + if (d->message_is_done) { + // The entire message was successfully parsed fast. + return true; + } + + // *ptr now points to the beginning of a field that could not be parsed fast. + // It's possible that some fields were parsed fast, in which case *ptr will + // have been updated. However, it's also possible that the very first field + // encountered could not be parsed fast, in which case *ptr will be unchanged. + // + // If the fast decoder consumed any data, it must have emitted at least + // one 'F' event into the trace buffer (in addition to the 'D' event + // that is always emitted). + UPB_ASSERT(_upb_Decoder_TracePtr(d) != trace_next || *ptr == start); + _upb_Decoder_Trace(d, '<'); +#endif + return false; +} + UPB_FORCEINLINE const char* _upb_Decoder_DecodeField(upb_Decoder* d, const char* ptr, upb_Message* msg, const upb_MiniTable* mt, uint64_t last_field_index, uint64_t data) { -#ifdef UPB_ENABLE_FASTTABLE - if (mt && mt->UPB_PRIVATE(table_mask) != (unsigned char)-1 && - !(d->options & kUpb_DecodeOption_DisableFastTable)) { - intptr_t table = decode_totable(mt); - ptr = upb_DecodeFast_Dispatch(d, ptr, msg, table, 0, 0); - if (d->message_is_done) return ptr; - _upb_Decoder_Trace(d, '<'); + if (_upb_Decoder_TryDecodeMessageFast(d, &ptr, msg, mt, last_field_index, + data)) { + return ptr; } else if (_upb_Decoder_IsDone(d, &ptr)) { return _upb_Decoder_EndMessage(d, ptr); } -#else - if (_upb_Decoder_IsDone(d, &ptr)) { - return _upb_Decoder_EndMessage(d, ptr); - } -#endif return _upb_Decoder_DecodeFieldNoFast(d, ptr, msg, mt); } diff --git a/upb/wire/decode_fast/BUILD b/upb/wire/decode_fast/BUILD index 8cf5995d8c4b9..c932b03bda348 100644 --- a/upb/wire/decode_fast/BUILD +++ b/upb/wire/decode_fast/BUILD @@ -5,6 +5,7 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_cc//cc:cc_test.bzl", "cc_test") load("@rules_cc//cc:defs.bzl", "cc_library") @@ -19,23 +20,46 @@ FASTTABLE_COPTS = UPB_DEFAULT_COPTS + select({ "//conditions:default": [], }) +bool_flag( + name = "debug_trace", + build_setting_default = False, +) + +config_setting( + name = "debug_trace_setting", + flag_values = {":debug_trace": "True"}, +) + +# Unfortunately field_parsers.h and dispatch.h need to be in the same library because of the +# bug when calling an __attribute__((preserve_most)) function across a shared library boundary. cc_library( name = "field_parsers", srcs = [ "cardinality.h", + "dispatch.c", + "dispatch.h", "field_fixed.c", "field_generic.c", "field_message.c", "field_string.c", "field_varint.c", ], - hdrs = ["field_parsers.h"], + hdrs = [ + "dispatch.h", + "field_parsers.h", + ], copts = FASTTABLE_COPTS, - visibility = ["//upb:__pkg__"], + defines = select({ + ":debug_trace_setting": ["UPB_TRACE_FASTDECODER=1"], + "//conditions:default": [], + }), + visibility = [ + "//upb:__pkg__", + "//upb/wire:__pkg__", + ], deps = [ ":combinations", ":data", - ":dispatch", "//upb/base", "//upb/base:internal", "//upb/mem", @@ -43,27 +67,15 @@ cc_library( "//upb/message:internal", "//upb/message:types", "//upb/mini_table", - "//upb/port", - "//upb/wire:decoder", - "//upb/wire:eps_copy_input_stream", - "//upb/wire:reader", - ], -) - -cc_library( - name = "dispatch", - srcs = ["dispatch.c"], - hdrs = ["dispatch.h"], - copts = FASTTABLE_COPTS, - visibility = ["//upb/wire:__pkg__"], - deps = [ - "//upb/message", - "//upb/mini_table", "//upb/mini_table:internal", "//upb/port", "//upb/wire:decoder", "//upb/wire:eps_copy_input_stream", - ], + "//upb/wire:reader", + ] + select({ + ":debug_trace_setting": [":select"], + "//conditions:default": [], + }), ) cc_library( diff --git a/upb/wire/decode_fast/bisect.sh b/upb/wire/decode_fast/bisect.sh index 7ac8d1ebb558d..ec710c74d1196 100755 --- a/upb/wire/decode_fast/bisect.sh +++ b/upb/wire/decode_fast/bisect.sh @@ -13,13 +13,15 @@ # # $ third_party/upb/upb/wire/decode_fast/bisect.sh third_party/upb/upb/test:test_generated_code -if [[ $# -ne 1 ]]; then +if [[ $# -lt 1 ]]; then echo "Usage: bisect.sh [blaze test flags] " exit 1 fi -/google/data/ro/teams/tetralight/bin/bisect -low 0 -high 128 \ +set -ex + +/google/data/ro/teams/tetralight/bin/bisect -low 0 -high 256 \ "blaze test --//third_party/upb:fasttable_enabled=True \ --per_file_copt=//third_party/upb/upb/wire/decode_fast:select@-DUPB_DECODEFAST_DISABLE_FUNCTIONS_ABOVE=\$X \ --host_per_file_copt=//third_party/upb/upb/wire/decode_fast:select@-DUPB_DECODEFAST_DISABLE_FUNCTIONS_ABOVE=\$X \ - $@" + $*" diff --git a/upb/wire/decode_fast/cardinality.h b/upb/wire/decode_fast/cardinality.h index 84fb671e6d7b4..4f6915ca7d042 100644 --- a/upb/wire/decode_fast/cardinality.h +++ b/upb/wire/decode_fast/cardinality.h @@ -24,6 +24,10 @@ #include "upb/wire/internal/decoder.h" #include "upb/wire/types.h" +#if UPB_TRACE_FASTDECODER +#include "upb/wire/decode_fast/select.h" +#endif + // Must be last include. #include "upb/port/def.inc" @@ -177,26 +181,24 @@ bool fastdecode_flippacked(uint64_t* data, int tagbytes) { // Old functions will be removed once the new ones are in use. // -// The new functions start from the premise that we should never hit an arena -// fallback path from the fasttable parser. -// -// We also use the new calling convention where we return an integer indicating -// the next function to call. This is to work around musttail limitations -// without forcing all fasttable code to be in macros. +// We use the new calling convention where we return an integer indicating the +// next function to call. This is to work around musttail limitations without +// forcing all fasttable code to be in macros. typedef struct { - void* dst; // For all fields, where to write the data. - - // For repeated fields. + void* dst; upb_Array* arr; void* end; - int added_elems; uint16_t expected_tag; -} upb_DecodeFastField; +} upb_DecodeFastArray; UPB_FORCEINLINE -void upb_DecodeFastField_AddArraySize(upb_DecodeFastField* field, int elems) { - field->arr->UPB_PRIVATE(size) += elems; +void upb_DecodeFastField_SetArraySize(upb_DecodeFastArray* field, + upb_DecodeFast_Type type) { + field->arr->UPB_PRIVATE(size) = + ((uintptr_t)field->dst - + (uintptr_t)upb_Array_MutableDataPtr(field->arr)) / + upb_DecodeFast_ValueBytes(type); } UPB_FORCEINLINE @@ -214,17 +216,6 @@ bool upb_DecodeFast_MaskedTagIsZero(uint16_t data, return upb_DecodeFast_MaskTag(data, tagsize) == 0; } -UPB_FORCEINLINE -bool upb_DecodeFast_CheckTag(uint16_t data, upb_DecodeFast_TagSize tagsize, - upb_DecodeFastNext* next) { - // The dispatch sequence xors the actual tag with the expected tag, so - // if the masked tag is zero, we know that the tag is valid. - if (!upb_DecodeFast_MaskedTagIsZero(data, tagsize)) { - return UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_FallbackToMiniTable, next); - } - return true; -} - // Checks to see if the tag is packed when we were expecting unpacked, or vice // versa. If so, flips the tag and returns true. UPB_FORCEINLINE @@ -238,108 +229,42 @@ bool upb_DecodeFast_TryFlipPacked(upb_DecodeFast_Type type, } UPB_FORCEINLINE -bool upb_DecodeFast_CheckPackableTag(upb_DecodeFast_Type type, - upb_DecodeFast_Cardinality card, - upb_DecodeFast_TagSize tagsize, - uint64_t* data, - upb_DecodeFastNext next_if_flip, - upb_DecodeFastNext* next) { - if (!upb_DecodeFast_IsRepeated(card)) { - return upb_DecodeFast_CheckTag(*data, tagsize, next); - } - if (UPB_UNLIKELY(!upb_DecodeFast_MaskedTagIsZero(*data, tagsize))) { - if (upb_DecodeFast_TryFlipPacked(type, card, tagsize, data)) { - // We can jump directly to the decoder for the flipped tag. - return UPB_DECODEFAST_EXIT(next_if_flip, next); - } - return UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_FallbackToMiniTable, next); - } - return true; -} - -UPB_FORCEINLINE -bool upb_DecodeFast_GetArrayForAppend(upb_Decoder* d, const char* ptr, - upb_Message* msg, uint64_t data, - uint64_t* hasbits, - upb_DecodeFastField* field, - upb_DecodeFast_Type type, int elems, - upb_DecodeFastNext* next) { - UPB_ASSERT(elems > 0); - - upb_Array** arr_p = - UPB_PTR_AT(msg, upb_DecodeFastData_GetOffset(data), upb_Array*); - int elem_size_lg2 = upb_DecodeFast_ValueBytesLg2(type); +bool upb_DecodeFast_ArrayReserve(upb_Decoder* d, upb_Array* arr, + upb_DecodeFast_Type type, int elems, + upb_DecodeFastNext* next) { + UPB_ASSERT(arr); - // Sync hasbits so we don't have to preserve them across the repeated field. - upb_DecodeFast_SetHasbits(msg, *hasbits); - *hasbits = 0; + size_t existing = upb_Array_Size(arr); + if (upb_Array_Reserve(arr, existing + elems, &d->arena)) return true; - if (UPB_LIKELY(!*arr_p)) { - // upb_Array does not exist yet. Create if with an appropriate initial - // capacity, as long as the arena has enough size in the current block. - int start_cap = 8; - - // A few arbitrary choices on the initial capacity, could be tuned later. - while (start_cap < elems) start_cap *= 2; - - upb_Array* arr = - UPB_PRIVATE(_upb_Array_TryFastNew)(&d->arena, start_cap, elem_size_lg2); - if (!arr) { - return UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_FallbackToMiniTable, next); - } - *arr_p = arr; - } else if (upb_Array_Capacity(*arr_p) - upb_Array_Size(*arr_p) < - (size_t)elems) { - // upb_Array exists, but is too small. Expand it as long as the arena - // has enough size in the current block. - int new_size = upb_Array_Size(*arr_p) + elems; - if (!UPB_PRIVATE(_upb_Array_TryFastRealloc)(*arr_p, new_size, elem_size_lg2, - &d->arena)) { - return UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_FallbackToMiniTable, next); - } - } - - void* start = upb_Array_MutableDataPtr(*arr_p); - int valbytes = upb_DecodeFast_ValueBytes(type); - field->arr = *arr_p; - field->dst = UPB_PTR_AT(start, upb_Array_Size(*arr_p) * valbytes, void); - field->end = UPB_PTR_AT(start, upb_Array_Capacity(*arr_p) * valbytes, void); - field->added_elems = 0; - field->expected_tag = _upb_FastDecoder_LoadTag(ptr); - - return true; + return UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_OutOfMemory, next); } UPB_FORCEINLINE -bool Upb_DecodeFast_GetField(upb_Decoder* d, const char* ptr, upb_Message* msg, - uint64_t data, uint64_t* hasbits, - upb_DecodeFastNext* ret, - upb_DecodeFastField* field, - upb_DecodeFast_Type type, - upb_DecodeFast_Cardinality card) { +bool upb_DecodeFast_GetScalarField(upb_Decoder* d, const char* ptr, + upb_Message* msg, uint64_t data, + uint64_t* hasbits, upb_DecodeFastNext* ret, + void** dst, + upb_DecodeFast_Cardinality card) { UPB_ASSERT(!upb_Message_IsFrozen(msg)); switch (card) { case kUpb_DecodeFast_Scalar: { // Set hasbit and return pointer to scalar field. - field->dst = UPB_PTR_AT(msg, upb_DecodeFastData_GetOffset(data), char); + *dst = UPB_PTR_AT(msg, upb_DecodeFastData_GetOffset(data), char); uint8_t hasbit_index = upb_DecodeFastData_GetPresence(data); *hasbits |= 1ull << hasbit_index; return true; } case kUpb_DecodeFast_Oneof: { - field->dst = UPB_PTR_AT(msg, upb_DecodeFastData_GetOffset(data), char); + *dst = UPB_PTR_AT(msg, upb_DecodeFastData_GetOffset(data), char); uint16_t case_ofs = upb_DecodeFastData_GetCaseOffset(data); uint32_t* oneof_case = UPB_PTR_AT(msg, case_ofs, uint32_t); uint8_t field_number = upb_DecodeFastData_GetPresence(data); *oneof_case = field_number; return true; } - case kUpb_DecodeFast_Repeated: - case kUpb_DecodeFast_Packed: - return upb_DecodeFast_GetArrayForAppend(d, ptr, msg, data, hasbits, field, - type, 1, ret); default: - UPB_UNREACHABLE(); + return false; } } @@ -354,59 +279,161 @@ bool upb_DecodeFast_TagMatches(uint16_t expected, uint16_t tag, } UPB_FORCEINLINE -bool upb_DecodeFast_DoNextRepeated(upb_Decoder* d, const char** ptr, - uint64_t data, upb_DecodeFastNext* next, - upb_DecodeFastField* field, - upb_DecodeFast_Type type, - upb_DecodeFast_Cardinality card, - upb_DecodeFast_TagSize tagsize) { +bool upb_DecodeFast_TryMatchTag(upb_Decoder* d, const char* ptr, + uint16_t expected, upb_DecodeFastNext* next, + upb_DecodeFast_TagSize tagsize) { int overrun; if (UPB_UNLIKELY( - upb_EpsCopyInputStream_IsDoneStatus(&d->input, *ptr, &overrun) != + upb_EpsCopyInputStream_IsDoneStatus(&d->input, ptr, &overrun) != kUpb_IsDoneStatus_NotDone)) { return UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_MessageIsDoneFallback, next); } - uint16_t tag = _upb_FastDecoder_LoadTag(*ptr); + uint16_t tag = _upb_FastDecoder_LoadTag(ptr); - if (!upb_DecodeFast_TagMatches(field->expected_tag, tag, tagsize)) { - // A different tag is encountered; perform regular dispatch. - return UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_TailCallDispatch, next); - } + return upb_DecodeFast_TagMatches(expected, tag, tagsize); +} +UPB_FORCEINLINE +bool upb_DecodeFast_NextRepeated(upb_Decoder* d, const char** ptr, + upb_DecodeFastNext* next, + upb_DecodeFastArray* field, bool has_next, + upb_DecodeFast_Type type, + upb_DecodeFast_TagSize tagsize) { field->dst = UPB_PTR_AT(field->dst, upb_DecodeFast_ValueBytes(type), char); - if (field->dst == field->end) { + if (has_next && field->dst == field->end) { // Out of arena memory; fall back to MiniTable decoder which will realloc. - return UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_Return, next); + UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_FallbackToMiniTable, next); + has_next = false; + } + + if (!has_next) { + upb_DecodeFastField_SetArraySize(field, type); + return false; } // Parse another instance of the repeated field. + *ptr += upb_DecodeFast_TagSizeBytes(tagsize); return true; } UPB_FORCEINLINE -bool upb_DecodeFast_NextRepeated(upb_Decoder* d, const char** ptr, - uint64_t data, upb_DecodeFastNext* next, - upb_DecodeFastField* field, - upb_DecodeFast_Type type, - upb_DecodeFast_Cardinality card, - upb_DecodeFast_TagSize tagsize) { - if (!upb_DecodeFast_IsRepeated(card)) { - // No repetition is possible; perform regular dispatch. - *next = kUpb_DecodeFastNext_TailCallDispatch; +bool upb_DecodeFast_CheckTag(const char** ptr, upb_DecodeFast_Type type, + upb_DecodeFast_Cardinality card, + upb_DecodeFast_TagSize tagsize, uint64_t* data, + upb_DecodeFastNext flipped, + upb_DecodeFastNext* next) { +#if UPB_TRACE_FASTDECODER + size_t idx = UPB_DECODEFAST_FUNCTION_IDX(type, card, tagsize); + fprintf(stderr, "Fasttable enter -> %s\n", + upb_DecodeFast_GetFunctionName(idx)); +#endif + // The dispatch sequence xors the actual tag with the expected tag, so + // if the masked tag is zero, we know that the tag is valid. + if (UPB_UNLIKELY(!upb_DecodeFast_MaskedTagIsZero(*data, tagsize))) { + // If this field is repeated and the field type is packable, we check + // whether the tag can be flipped (ie. packed -> unpacked or vice versa). + // If so, we can jump directly to the decoder for the flipped tag. + if (flipped && upb_DecodeFast_TryFlipPacked(type, card, tagsize, data)) { + // We can jump directly to the decoder for the flipped tag. + return UPB_DECODEFAST_EXIT(flipped, next); + } + return UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_FallbackToMiniTable, next); + } + *ptr += upb_DecodeFast_TagSizeBytes(tagsize); + return true; +} + +UPB_FORCEINLINE +bool upb_DecodeFast_GetArrayForAppend(upb_Decoder* d, const char* ptr, + upb_Message* msg, uint64_t data, + uint64_t* hasbits, + upb_DecodeFastArray* field, + upb_DecodeFast_Type type, int elems, + upb_DecodeFastNext* next) { + UPB_ASSERT(elems > 0); + + upb_Array** arr_p = + UPB_PTR_AT(msg, upb_DecodeFastData_GetOffset(data), upb_Array*); + upb_Array* arr = *arr_p; + int lg2 = upb_DecodeFast_ValueBytesLg2(type); + + // Sync hasbits so we don't have to preserve them across the repeated field. + upb_DecodeFast_SetHasbits(msg, *hasbits); + *hasbits = 0; + + if (UPB_LIKELY(!arr)) { + // upb_Array does not exist yet. Create if with an appropriate initial + // capacity, as long as the arena has enough size in the current block. + int start_cap = 8; + + // A few arbitrary choices on the initial capacity, could be tuned later. + while (start_cap < elems) start_cap *= 2; + + arr = UPB_PRIVATE(_upb_Array_New)(&d->arena, start_cap, lg2); + if (!arr) { + return UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_OutOfMemory, next); + } + *arr_p = arr; + } else if (!upb_DecodeFast_ArrayReserve(d, arr, type, elems, next)) { return false; } - field->added_elems++; + void* start = upb_Array_MutableDataPtr(arr); + int valbytes = upb_DecodeFast_ValueBytes(type); + + field->arr = arr; + field->dst = UPB_PTR_AT(start, upb_Array_Size(arr) * valbytes, void); + field->end = UPB_PTR_AT(start, upb_Array_Capacity(arr) * valbytes, void); + field->expected_tag = _upb_FastDecoder_LoadTag(ptr); - if (!upb_DecodeFast_DoNextRepeated(d, ptr, data, next, field, type, card, - tagsize)) { - // Commit elements already added to the array. - upb_DecodeFastField_AddArraySize(field, field->added_elems); + return true; +} + +typedef bool upb_DecodeFast_Single(upb_Decoder* d, const char** ptr, void* dst, + upb_DecodeFast_Type type, + upb_DecodeFastNext* next); + +UPB_FORCEINLINE +bool upb_DecodeFast_Unpacked(upb_Decoder* d, const char** ptr, upb_Message* msg, + uint64_t* data, uint64_t* hasbits, + upb_DecodeFastNext* ret, upb_DecodeFast_Type type, + upb_DecodeFast_Cardinality card, + upb_DecodeFast_TagSize tagsize, + upb_DecodeFast_Single* single) { + const char* p = *ptr; + if (!upb_DecodeFast_CheckTag(&p, type, card, tagsize, data, + kUpb_DecodeFastNext_TailCallPacked, ret)) { return false; } + void* dst; + + if (upb_DecodeFast_GetScalarField(d, p, msg, *data, hasbits, ret, &dst, + card)) { + if (!single(d, &p, dst, type, ret)) return false; + *ptr = p; + _upb_Decoder_Trace(d, 'F'); + return true; + } + + upb_DecodeFastArray arr; + if (!upb_DecodeFast_GetArrayForAppend(d, *ptr, msg, *data, hasbits, &arr, + type, 1, ret)) { + return false; + } + + bool next_tag_matches; + do { + if (!single(d, &p, arr.dst, type, ret)) return false; + *ptr = p; + _upb_Decoder_Trace(d, 'F'); + next_tag_matches = + upb_DecodeFast_TryMatchTag(d, p, arr.expected_tag, ret, tagsize); + } while (upb_DecodeFast_NextRepeated(d, &p, ret, &arr, next_tag_matches, type, + tagsize)); + return true; } @@ -431,14 +458,46 @@ bool upb_DecodeFast_DecodeSize(upb_Decoder* d, const char** pp, int* size, } UPB_FORCEINLINE -bool upb_DecodeFast_DecodeShortSizeForImmediateRead(upb_Decoder* d, - const char** ptr, int* size, - upb_DecodeFastNext* next) { - if (!upb_DecodeFast_DecodeSize(d, ptr, size, next)) return false; - if (!upb_EpsCopyInputStream_CheckDataSizeAvailable(&d->input, *ptr, *size)) { - return UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_FallbackToMiniTable, next); +bool upb_DecodeFast_Delimited(upb_Decoder* d, const char** ptr, + upb_DecodeFast_Type type, + upb_DecodeFast_Cardinality card, + upb_DecodeFast_TagSize tagsize, uint64_t* data, + upb_EpsCopyInputStream_ParseDelimitedFunc* func, + upb_DecodeFastNext* ret, void* ctx) { + const char* p = *ptr; + int size; + + if (!upb_DecodeFast_CheckTag(&p, type, card, tagsize, data, + kUpb_DecodeFastNext_TailCallUnpacked, ret)) { + return false; + } + + if (!upb_DecodeFast_DecodeSize(d, &p, &size, ret)) return false; + + if (upb_EpsCopyInputStream_TryParseDelimitedFast(&d->input, &p, size, func, + ctx)) { + if (UPB_UNLIKELY(p == NULL)) goto fail; + } else { + if (!upb_EpsCopyInputStream_CheckSize(&d->input, p, size)) { + // Corrupt wire format: invalid limit. + *ptr = NULL; + return UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_Malformed, ret); + } + int delta = upb_EpsCopyInputStream_PushLimit(&d->input, p, size); + p = func(&d->input, p, size, ctx); + if (UPB_UNLIKELY(p == NULL)) goto fail; + upb_EpsCopyInputStream_PopLimit(&d->input, p, delta); } + + *ptr = p; return true; + +fail: + // We can't fall back to the mini table here because we may have already + // advanced past the previous buffer. + UPB_ASSERT(*ret != kUpb_DecodeFastNext_FallbackToMiniTable); + *ptr = NULL; + return false; } UPB_FORCEINLINE diff --git a/upb/wire/decode_fast/combinations.h b/upb/wire/decode_fast/combinations.h index e83513dcf91d8..36d327e0f5f00 100644 --- a/upb/wire/decode_fast/combinations.h +++ b/upb/wire/decode_fast/combinations.h @@ -199,13 +199,16 @@ UPB_INLINE upb_DecodeFast_Type upb_DecodeFast_GetType(uint32_t function_idx) { // index. Some field types (eg. groups) do not even have a function index at // the moment, and so will be rejected by upb_DecodeFast_TryFillEntry() before // we even get here. -#define UPB_DECODEFAST_COMBINATION_IS_ENABLED(type, card, size) \ - (type == kUpb_DecodeFast_Fixed32 || type == kUpb_DecodeFast_Fixed64) +#define UPB_DECODEFAST_COMBINATION_IS_ENABLED(type, card, size) \ + (type == kUpb_DecodeFast_Fixed32 || type == kUpb_DecodeFast_Fixed64 || \ + ((type == kUpb_DecodeFast_Varint32 || type == kUpb_DecodeFast_Varint64 || \ + type == kUpb_DecodeFast_ZigZag32 || type == kUpb_DecodeFast_ZigZag64 || \ + type == kUpb_DecodeFast_Bool))) #ifdef UPB_DECODEFAST_DISABLE_FUNCTIONS_ABOVE #define UPB_DECODEFAST_ISENABLED(type, card, size) \ (UPB_DECODEFAST_COMBINATION_IS_ENABLED(type, card, size) && \ - (UPB_DECODEFAST_FUNCION_IDX(type, card, size) <= \ + (UPB_DECODEFAST_FUNCTION_IDX(type, card, size) <= \ UPB_DECODEFAST_DISABLE_FUNCTIONS_ABOVE)) #else #define UPB_DECODEFAST_ISENABLED(type, card, size) \ diff --git a/upb/wire/decode_fast/dispatch.c b/upb/wire/decode_fast/dispatch.c index bac147cd674f9..140c737e2eaa7 100644 --- a/upb/wire/decode_fast/dispatch.c +++ b/upb/wire/decode_fast/dispatch.c @@ -46,6 +46,16 @@ UPB_NOINLINE UPB_PRESERVE_NONE const char* upb_DecodeFast_MessageIsDoneFallback( } } +UPB_PRESERVE_MOST +const char* upb_DecodeFast_IsDoneFallback(upb_Decoder* d, const char* ptr) { + int overrun; + upb_IsDoneStatus status = + upb_EpsCopyInputStream_IsDoneStatus(&d->input, ptr, &overrun); + UPB_ASSERT(status == kUpb_IsDoneStatus_NeedFallback); + return _upb_EpsCopyInputStream_IsDoneFallbackInline( + &d->input, ptr, overrun, _upb_Decoder_BufferFlipCallback); +} + const char* _upb_FastDecoder_ErrorJmp2(upb_Decoder* d) { UPB_LONGJMP(d->err, 1); return NULL; diff --git a/upb/wire/decode_fast/dispatch.h b/upb/wire/decode_fast/dispatch.h index c3b96ec4c3643..e92043b7036d7 100644 --- a/upb/wire/decode_fast/dispatch.h +++ b/upb/wire/decode_fast/dispatch.h @@ -43,9 +43,14 @@ UPB_INLINE uint32_t _upb_FastDecoder_LoadTag(const char* ptr) { return tag; } -UPB_INLINE UPB_PRESERVE_NONE const char* _upb_FastDecoder_TagDispatch( - struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, - uint64_t hasbits, uint64_t tag) { +// We have to disable HWASAN for this function because we steal the high byte +// of the `table` pointer for our own purposes (the table mask). This overwrites +// the tag that HWASAN depends on for its own checks. +__attribute__((no_sanitize("hwaddress"))) UPB_INLINE + UPB_PRESERVE_NONE const char* + _upb_FastDecoder_TagDispatch(struct upb_Decoder* d, const char* ptr, + upb_Message* msg, intptr_t table, + uint64_t hasbits, uint64_t tag) { const upb_MiniTable* table_p = decode_totablep(table); uint8_t mask = table; size_t ofs = tag & mask; @@ -85,6 +90,30 @@ UPB_FORCEINLINE UPB_PRESERVE_NONE const char* upb_DecodeFast_Dispatch( UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); } +// Workaround for b/177688959. We need to ensure that this function never goes +// through PLT lookup. It follows that this function may not be called by +// any other cc_library(). +__attribute__((visibility("hidden"))) UPB_PRESERVE_MOST const char* +upb_DecodeFast_IsDoneFallback(upb_Decoder* d, const char* ptr); + +UPB_FORCEINLINE +bool upb_DecodeFast_IsDone(upb_Decoder* d, const char** ptr) { + int overrun; + upb_IsDoneStatus status = + upb_EpsCopyInputStream_IsDoneStatus(&d->input, *ptr, &overrun); + switch (status) { + case kUpb_IsDoneStatus_Done: + return true; + case kUpb_IsDoneStatus_NotDone: + return false; + case kUpb_IsDoneStatus_NeedFallback: + *ptr = upb_DecodeFast_IsDoneFallback(d, *ptr); + return false; + default: + UPB_UNREACHABLE(); + } +} + UPB_FORCEINLINE bool fastdecode_checktag(uint16_t data, int tagbytes) { if (tagbytes == 1) { @@ -142,7 +171,7 @@ const char* fastdecode_delimited( return NULL; } int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, len); - ptr = func(&d->input, ptr, ctx); + ptr = func(&d->input, ptr, len, ctx); upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta); } return ptr; @@ -160,20 +189,17 @@ void upb_DecodeFast_SetHasbits(upb_Message* msg, uint64_t hasbits) { } typedef enum { - // Call the dispatch function using musttail. - kUpb_DecodeFastNext_TailCallDispatch = 0, + kUpb_DecodeFastNext_Dispatch = 0, - // Return from the function with no tail call. This is used either to signal - // a fallback to the mini table or the end of the message if - // d->message_is_done is true. - kUpb_DecodeFastNext_Return = 1, + // Fallback to the MiniTable decoder. This is used either to signal a fallback + // to the mini table or the end of the message if d->message_is_done is true. + kUpb_DecodeFastNext_FallbackToMiniTable = 1, + // Signal an error. kUpb_DecodeFastNext_Error = 2, - // Alias for clarity in the code. - kUpb_DecodeFastNext_FallbackToMiniTable = kUpb_DecodeFastNext_Return, - - // Tail call to the function to parse the current field. + // Handle the case where ptr >= limit, which is either end-of-message or + // end-of-buffer. kUpb_DecodeFastNext_MessageIsDoneFallback = 3, // Tail call to the function to parse the current field, except parse it as @@ -185,8 +211,6 @@ typedef enum { kUpb_DecodeFastNext_TailCallUnpacked = 5, } upb_DecodeFastNext; -const char* upb_DecodeFast_IsDoneFallback(UPB_PARSE_PARAMS); - /* Error function that will abort decoding with longjmp(). We can't declare this * UPB_NORETURN, even though it is appropriate, because if we do then compilers * will "helpfully" refuse to tailcall to it @@ -202,34 +226,30 @@ const char* _upb_FastDecoder_ErrorJmp(upb_Decoder* d, upb_DecodeStatus status) { return _upb_FastDecoder_ErrorJmp2(d); } -#define UPB_DECODEFAST_NEXTMAYBEPACKED(next, func_unpacked, func_packed) \ - if (UPB_UNLIKELY(next != kUpb_DecodeFastNext_TailCallDispatch)) { \ - switch (next) { \ - case kUpb_DecodeFastNext_Return: \ - UPB_MUSTTAIL return _upb_FastDecoder_DecodeGeneric(UPB_PARSE_ARGS); \ - case kUpb_DecodeFastNext_Error: \ - UPB_ASSERT(d->status != kUpb_DecodeStatus_Ok); \ - return _upb_FastDecoder_ErrorJmp2(d); \ - case kUpb_DecodeFastNext_MessageIsDoneFallback: \ - UPB_MUSTTAIL return upb_DecodeFast_MessageIsDoneFallback( \ - UPB_PARSE_ARGS); \ - case kUpb_DecodeFastNext_TailCallPacked: \ - UPB_MUSTTAIL return func_packed(UPB_PARSE_ARGS); \ - case kUpb_DecodeFastNext_TailCallUnpacked: \ - UPB_MUSTTAIL return func_unpacked(UPB_PARSE_ARGS); \ - default: \ - UPB_UNREACHABLE(); \ - } \ - } \ - UPB_MUSTTAIL return upb_DecodeFast_Dispatch(UPB_PARSE_ARGS); - -// Uncomment this to see the exit points from the fast decoder. -// #define UPB_LOG_EXITS +#define UPB_DECODEFAST_NEXTMAYBEPACKED(next, func_unpacked, func_packed) \ + switch (next) { \ + case kUpb_DecodeFastNext_Dispatch: \ + UPB_MUSTTAIL return upb_DecodeFast_Dispatch(UPB_PARSE_ARGS); \ + case kUpb_DecodeFastNext_FallbackToMiniTable: \ + UPB_MUSTTAIL return _upb_FastDecoder_DecodeGeneric(UPB_PARSE_ARGS); \ + case kUpb_DecodeFastNext_Error: \ + UPB_ASSERT(d->status != kUpb_DecodeStatus_Ok); \ + return _upb_FastDecoder_ErrorJmp2(d); \ + case kUpb_DecodeFastNext_MessageIsDoneFallback: \ + UPB_MUSTTAIL return upb_DecodeFast_MessageIsDoneFallback( \ + UPB_PARSE_ARGS); \ + case kUpb_DecodeFastNext_TailCallPacked: \ + UPB_MUSTTAIL return func_packed(UPB_PARSE_ARGS); \ + case kUpb_DecodeFastNext_TailCallUnpacked: \ + UPB_MUSTTAIL return func_unpacked(UPB_PARSE_ARGS); \ + default: \ + UPB_UNREACHABLE(); \ + } UPB_INLINE bool upb_DecodeFast_SetExit(upb_DecodeFastNext* next, upb_DecodeFastNext val, const char* sym, const char* file, int line) { -#ifdef UPB_LOG_EXITS +#ifdef UPB_TRACE_FASTDECODER fprintf(stderr, "Fasttable fallback @ %s:%d -> %s (%d)\n", file, line, sym, val); #endif @@ -241,7 +261,7 @@ UPB_INLINE bool upb_DecodeFast_SetError(upb_Decoder* d, upb_DecodeFastNext* next, upb_DecodeStatus val, const char* sym, const char* file, int line) { -#ifdef UPB_LOG_EXITS +#ifdef UPB_TRACE_FASTDECODER fprintf(stderr, "Fasttable error @ %s:%d -> %s (%d)\n", file, line, sym, val); #endif d->status = val; diff --git a/upb/wire/decode_fast/field_fixed.c b/upb/wire/decode_fast/field_fixed.c index 5b466b99ebd96..929b2fdab6e19 100644 --- a/upb/wire/decode_fast/field_fixed.c +++ b/upb/wire/decode_fast/field_fixed.c @@ -21,106 +21,94 @@ // Must be last. #include "upb/port/def.inc" -UPB_FORCEINLINE -int upb_DecodeFast_UnpackedFixed(upb_Decoder* d, const char** ptr, - upb_Message* msg, intptr_t table, - uint64_t* hasbits, uint64_t* data, - upb_DecodeFast_Type type, - upb_DecodeFast_Cardinality card, - upb_DecodeFast_TagSize tagsize) { - upb_DecodeFastNext ret; - upb_DecodeFastField field; +static bool upb_DecodeFast_SingleFixed(upb_Decoder* d, const char** ptr, + void* dst, upb_DecodeFast_Type type, + upb_DecodeFastNext* next) { int valbytes = upb_DecodeFast_ValueBytes(type); + memcpy(dst, *ptr, valbytes); + *ptr += valbytes; + return true; +} - if (!upb_DecodeFast_CheckPackableTag(type, card, tagsize, data, - kUpb_DecodeFastNext_TailCallPacked, - &ret) || - !Upb_DecodeFast_GetField(d, *ptr, msg, *data, hasbits, &ret, &field, type, - card)) { - return ret; - } +typedef struct { + upb_Decoder* d; + upb_DecodeFast_Type type; + upb_Message* msg; + uint64_t* data; + uint64_t* hasbits; + upb_DecodeFastNext* ret; +} upb_DecodeFast_PackedFixedContext; - do { - *ptr += upb_DecodeFast_TagSizeBytes(tagsize); - memcpy(field.dst, *ptr, valbytes); - *ptr += valbytes; - _upb_Decoder_Trace(d, 'F'); - } while (upb_DecodeFast_NextRepeated(d, ptr, *data, &ret, &field, type, card, - tagsize)); +static const char* upb_DecodeFast_PackedFixed(upb_EpsCopyInputStream* st, + const char* ptr, int size, + void* ctx) { + // This will need to be relaxed if/when we support streaming input. + UPB_ASSERT(upb_EpsCopyInputStream_CheckDataSizeAvailable(st, ptr, size)); - return kUpb_DecodeFastNext_TailCallDispatch; -} + upb_DecodeFast_PackedFixedContext* c = + (upb_DecodeFast_PackedFixedContext*)ctx; -UPB_FORCEINLINE -int upb_DecodeFast_PackedFixed(upb_Decoder* d, const char** ptr, - upb_Message* msg, intptr_t table, - uint64_t* hasbits, uint64_t* data, - upb_DecodeFast_Type type, - upb_DecodeFast_Cardinality card, - upb_DecodeFast_TagSize tagsize) { - upb_DecodeFastNext ret; - int size; - upb_DecodeFastField field; - uint8_t valbytes = upb_DecodeFast_ValueBytes(type); - - const char* data_ptr = *ptr + upb_DecodeFast_TagSizeBytes(tagsize); - - if (!upb_DecodeFast_CheckPackableTag(type, card, tagsize, data, - kUpb_DecodeFastNext_TailCallUnpacked, - &ret) || - !upb_DecodeFast_DecodeShortSizeForImmediateRead(d, &data_ptr, &size, - &ret)) { - return ret; - } + int valbytes = upb_DecodeFast_ValueBytes(c->type); if (size != 0) { if (size % valbytes != 0) { - UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_Malformed, &ret); - return ret; + UPB_DECODEFAST_ERROR(c->d, kUpb_DecodeStatus_Malformed, c->ret); + return NULL; } - if (!upb_DecodeFast_GetArrayForAppend(d, *ptr, msg, *data, hasbits, &field, - type, size / valbytes, &ret)) { - return ret; + upb_DecodeFastArray arr; + + if (!upb_DecodeFast_GetArrayForAppend(c->d, ptr, c->msg, *c->data, + c->hasbits, &arr, c->type, + size / valbytes, c->ret)) { + return NULL; } - upb_DecodeFast_InlineMemcpy(field.dst, data_ptr, size); - upb_DecodeFastField_AddArraySize(&field, size / valbytes); + upb_DecodeFast_InlineMemcpy(arr.dst, ptr, size); + arr.dst = UPB_PTR_AT(arr.dst, size, char); + upb_DecodeFastField_SetArraySize(&arr, c->type); } - *ptr = data_ptr + size; - _upb_Decoder_Trace(d, 'F'); - - return kUpb_DecodeFastNext_TailCallDispatch; + _upb_Decoder_Trace(c->d, 'F'); + return ptr + size; } UPB_FORCEINLINE -int upb_DecodeFast_Fixed(upb_Decoder* d, const char** ptr, upb_Message* msg, - intptr_t table, uint64_t* hasbits, uint64_t* data, - upb_DecodeFast_Type type, - upb_DecodeFast_Cardinality card, - upb_DecodeFast_TagSize tagsize) { +void upb_DecodeFast_Fixed(upb_Decoder* d, const char** ptr, upb_Message* msg, + intptr_t table, uint64_t* hasbits, uint64_t* data, + upb_DecodeFastNext* ret, upb_DecodeFast_Type type, + upb_DecodeFast_Cardinality card, + upb_DecodeFast_TagSize tagsize) { if (card == kUpb_DecodeFast_Packed) { - return upb_DecodeFast_PackedFixed(d, ptr, msg, table, hasbits, data, type, - card, tagsize); + upb_DecodeFast_PackedFixedContext ctx = { + .d = d, + .type = type, + .msg = msg, + .data = data, + .hasbits = hasbits, + .ret = ret, + }; + upb_DecodeFast_Delimited(d, ptr, type, card, tagsize, data, + &upb_DecodeFast_PackedFixed, ret, &ctx); } else { - return upb_DecodeFast_UnpackedFixed(d, ptr, msg, table, hasbits, data, type, - card, tagsize); + upb_DecodeFast_Unpacked(d, ptr, msg, data, hasbits, ret, type, card, + tagsize, &upb_DecodeFast_SingleFixed); } } /* Generate all combinations: * {s,o,r,p} x {f4,f8} x {1bt,2bt} */ -#define F(type, card, tagbytes) \ - UPB_NOINLINE UPB_PRESERVE_NONE const char* UPB_DECODEFAST_FUNCNAME( \ - type, card, tagbytes)(UPB_PARSE_PARAMS) { \ - int next = upb_DecodeFast_Fixed( \ - d, &ptr, msg, table, &hasbits, &data, kUpb_DecodeFast_##type, \ - kUpb_DecodeFast_##card, kUpb_DecodeFast_##tagbytes); \ - UPB_DECODEFAST_NEXTMAYBEPACKED( \ - next, UPB_DECODEFAST_FUNCNAME(type, Repeated, tagbytes), \ - UPB_DECODEFAST_FUNCNAME(type, Packed, tagbytes)); \ +#define F(type, card, tagbytes) \ + UPB_NOINLINE UPB_PRESERVE_NONE const char* UPB_DECODEFAST_FUNCNAME( \ + type, card, tagbytes)(UPB_PARSE_PARAMS) { \ + upb_DecodeFastNext next = kUpb_DecodeFastNext_Dispatch; \ + upb_DecodeFast_Fixed(d, &ptr, msg, table, &hasbits, &data, &next, \ + kUpb_DecodeFast_##type, kUpb_DecodeFast_##card, \ + kUpb_DecodeFast_##tagbytes); \ + UPB_DECODEFAST_NEXTMAYBEPACKED( \ + next, UPB_DECODEFAST_FUNCNAME(type, Repeated, tagbytes), \ + UPB_DECODEFAST_FUNCNAME(type, Packed, tagbytes)); \ } UPB_DECODEFAST_CARDINALITIES(UPB_DECODEFAST_TAGSIZES, F, Fixed32) diff --git a/upb/wire/decode_fast/field_message.c b/upb/wire/decode_fast/field_message.c index 2f9d464a1c9a7..0cf45e87a32d8 100644 --- a/upb/wire/decode_fast/field_message.c +++ b/upb/wire/decode_fast/field_message.c @@ -29,7 +29,7 @@ typedef struct { UPB_FORCEINLINE const char* fastdecode_tosubmsg(upb_EpsCopyInputStream* e, const char* ptr, - void* ctx) { + int size, void* ctx) { upb_Decoder* d = (upb_Decoder*)e; fastdecode_submsgdata* submsg = ctx; ptr = upb_DecodeFast_Dispatch(d, ptr, submsg->msg, submsg->table, 0, 0); diff --git a/upb/wire/decode_fast/field_varint.c b/upb/wire/decode_fast/field_varint.c index 6a52c987b1554..eb4d5c73a5f2a 100644 --- a/upb/wire/decode_fast/field_varint.c +++ b/upb/wire/decode_fast/field_varint.c @@ -9,6 +9,7 @@ #include #include +#include "upb/message/message.h" #include "upb/wire/decode.h" #include "upb/wire/decode_fast/cardinality.h" #include "upb/wire/decode_fast/combinations.h" @@ -16,155 +17,133 @@ #include "upb/wire/decode_fast/field_parsers.h" #include "upb/wire/eps_copy_input_stream.h" #include "upb/wire/internal/decoder.h" +#include "upb/wire/reader.h" // Must be last. #include "upb/port/def.inc" -UPB_FORCEINLINE -uint64_t fastdecode_munge(uint64_t val, int valbytes, bool zigzag) { - if (valbytes == 1) { - return val != 0; - } else if (zigzag) { - if (valbytes == 4) { +static bool upb_DecodeFast_SingleVarint(upb_Decoder* d, const char** ptr, + void* dst, upb_DecodeFast_Type type, + upb_DecodeFastNext* next) { + UPB_ASSERT(dst); + + const char* p = *ptr; + uint64_t val; + + p = upb_WireReader_ReadVarint(p, &val, &d->input); + if (!p) { + return UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_Malformed, next); + } + + switch (type) { + case kUpb_DecodeFast_Bool: + val = val != 0; + break; + case kUpb_DecodeFast_ZigZag32: { uint32_t n = val; - return (n >> 1) ^ -(int32_t)(n & 1); - } else if (valbytes == 8) { - return (val >> 1) ^ -(int64_t)(val & 1); + val = (n >> 1) ^ -(int32_t)(n & 1); + break; + } + case kUpb_DecodeFast_ZigZag64: { + val = (val >> 1) ^ -(int64_t)(val & 1); + break; } - UPB_UNREACHABLE(); + default: + break; } - return val; + + memcpy(dst, &val, upb_DecodeFast_ValueBytes(type)); // Assumes little endian. + *ptr = p; + return true; } +typedef struct { + upb_Decoder* decoder; + upb_DecodeFast_Type type; + upb_Message* msg; + uint64_t* data; + uint64_t* hasbits; + upb_DecodeFastNext* ret; +} upb_DecodeFast_PackedVarintContext; + UPB_FORCEINLINE -const char* fastdecode_varint64(const char* ptr, uint64_t* val) { - ptr++; - *val = (uint8_t)ptr[-1]; - if (UPB_UNLIKELY(*val & 0x80)) { - int i; - for (i = 0; i < 8; i++) { - ptr++; - uint64_t byte = (uint8_t)ptr[-1]; - *val += (byte - 1) << (7 + 7 * i); - if (UPB_LIKELY((byte & 0x80) == 0)) goto done; - } - ptr++; - uint64_t byte = (uint8_t)ptr[-1]; - if (byte > 1) { - return NULL; - } - *val += (byte - 1) << 63; +int upb_DecodeFast_CountVarints(const char* ptr, const char* end) { + int count = 0; + for (; ptr < end; ptr++) { + count += (*ptr & 0x80) == 0; } -done: - UPB_ASSUME(ptr != NULL); - return ptr; + return count; } -#define FASTDECODE_UNPACKEDVARINT(packed, card, d, ptr, msg, table, hasbits, \ - data, type, tagsize) \ - uint64_t val; \ - void* dst; \ - fastdecode_arr farr; \ - int valbytes = upb_DecodeFast_ValueBytes(type); \ - bool zigzag = upb_DecodeFast_IsZigZag(type); \ - int tagbytes = upb_DecodeFast_TagSizeBytes(tagsize); \ - \ - FASTDECODE_CHECKPACKED(tagbytes, card, packed); \ - \ - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ - card); \ - if (card == kUpb_DecodeFast_Repeated) { \ - if (UPB_UNLIKELY(!dst)) { \ - RETURN_GENERIC("need array resize\n"); \ - } \ - } \ - \ - again: \ - if (card == kUpb_DecodeFast_Repeated) { \ - dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ - } \ - \ - ptr += tagbytes; \ - ptr = fastdecode_varint64(ptr, &val); \ - if (ptr == NULL) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ - val = fastdecode_munge(val, valbytes, zigzag); \ - memcpy(dst, &val, valbytes); \ - \ - if (card == kUpb_DecodeFast_Repeated) { \ - fastdecode_nextret ret = fastdecode_nextrepeated( \ - d, dst, &ptr, &farr, data, tagbytes, valbytes); \ - switch (ret.next) { \ - case FD_NEXT_SAMEFIELD: \ - dst = ret.dst; \ - goto again; \ - case FD_NEXT_OTHERFIELD: \ - data = ret.tag; \ - UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ - case FD_NEXT_ATLIMIT: \ - return ptr; \ - } \ - } \ - \ - UPB_MUSTTAIL return upb_DecodeFast_Dispatch(UPB_PARSE_ARGS); +static const char* upb_DecodeFast_PackedVarint(upb_EpsCopyInputStream* st, + const char* ptr, int size, + void* ctx) { + // This will need to be relaxed if/when we support streaming input. + UPB_ASSERT(upb_EpsCopyInputStream_CheckDataSizeAvailable(st, ptr, size)); -typedef struct { - uint8_t valbytes; - bool zigzag; - void* dst; - fastdecode_arr farr; -} fastdecode_varintdata; + upb_DecodeFast_PackedVarintContext* c = + (upb_DecodeFast_PackedVarintContext*)ctx; -UPB_FORCEINLINE -const char* fastdecode_topackedvarint(upb_EpsCopyInputStream* e, - const char* ptr, void* ctx) { - upb_Decoder* d = (upb_Decoder*)e; - fastdecode_varintdata* data = ctx; - void* dst = data->dst; - uint64_t val; + if (size > 0) { + upb_DecodeFastArray arr; + int count = upb_DecodeFast_CountVarints(ptr, ptr + size); + int valbytes = upb_DecodeFast_ValueBytes(c->type); - while (!_upb_Decoder_IsDone(d, &ptr)) { - dst = fastdecode_resizearr(d, dst, &data->farr, data->valbytes); - ptr = fastdecode_varint64(ptr, &val); - if (ptr == NULL) return NULL; - val = fastdecode_munge(val, data->valbytes, data->zigzag); - memcpy(dst, &val, data->valbytes); - dst = (char*)dst + data->valbytes; + // If the last byte is a continuation byte, we have a malformed varint. + if (count == 0 || (ptr[size - 1] & 0x80) != 0) { + UPB_DECODEFAST_ERROR(c->decoder, kUpb_DecodeStatus_Malformed, c->ret); + return NULL; + } + + if (!upb_DecodeFast_GetArrayForAppend(c->decoder, ptr, c->msg, *c->data, + c->hasbits, &arr, c->type, count, + c->ret)) { + return NULL; + } + + UPB_ASSERT(arr.dst); + + int read = 0; + while (!upb_DecodeFast_IsDone(c->decoder, &ptr)) { + UPB_ASSERT(read < count); + ++read; + UPB_ASSERT(arr.dst); + if (!upb_DecodeFast_SingleVarint(c->decoder, &ptr, arr.dst, c->type, + c->ret)) { + return NULL; + } + arr.dst = UPB_PTR_AT(arr.dst, valbytes, char); + } + + upb_DecodeFastField_SetArraySize(&arr, c->type); } - fastdecode_commitarr(dst, &data->farr, data->valbytes); + _upb_Decoder_Trace(c->decoder, 'F'); return ptr; } -#define FASTDECODE_PACKEDVARINT(unpacked, card, d, ptr, msg, table, hasbits, \ - data, type, tagsize) \ - int valbytes = upb_DecodeFast_ValueBytes(type); \ - bool zigzag = upb_DecodeFast_IsZigZag(type); \ - int tagbytes = upb_DecodeFast_TagSizeBytes(tagsize); \ - fastdecode_varintdata ctx = {valbytes, zigzag}; \ - \ - FASTDECODE_CHECKPACKED(tagbytes, kUpb_DecodeFast_Repeated, unpacked); \ - \ - ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \ - valbytes, kUpb_DecodeFast_Repeated); \ - if (UPB_UNLIKELY(!ctx.dst)) { \ - RETURN_GENERIC("need array resize\n"); \ - } \ - \ - ptr += tagbytes; \ - ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \ - \ - if (UPB_UNLIKELY(ptr == NULL)) { \ - _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ - } \ - \ - UPB_MUSTTAIL return upb_DecodeFast_Dispatch(d, ptr, msg, table, hasbits, 0); - -#define FASTDECODE_VARINT(unpacked, packed, card, ...) \ - if (card == kUpb_DecodeFast_Packed) { \ - FASTDECODE_PACKEDVARINT(unpacked, card, __VA_ARGS__) \ - } else { \ - FASTDECODE_UNPACKEDVARINT(packed, card, __VA_ARGS__) \ +UPB_FORCEINLINE +void upb_DecodeFast_Varint(upb_Decoder* d, const char** ptr, upb_Message* msg, + intptr_t table, uint64_t* hasbits, uint64_t* data, + upb_DecodeFastNext* ret, upb_DecodeFast_Type type, + upb_DecodeFast_Cardinality card, + upb_DecodeFast_TagSize tagsize) { + if (card == kUpb_DecodeFast_Packed) { + upb_DecodeFast_PackedVarintContext ctx = { + .decoder = d, + .type = type, + .msg = msg, + .data = data, + .hasbits = hasbits, + .ret = ret, + }; + upb_DecodeFast_Delimited(d, ptr, type, card, tagsize, data, + &upb_DecodeFast_PackedVarint, ret, &ctx); + } else { + upb_DecodeFast_Unpacked(d, ptr, msg, data, hasbits, ret, type, card, + tagsize, &upb_DecodeFast_SingleVarint); } +} /* Generate all combinations: * {s,o,r,p} x {b1,v4,z4,v8,z8} x {1bt,2bt} */ @@ -172,10 +151,13 @@ const char* fastdecode_topackedvarint(upb_EpsCopyInputStream* e, #define F(type, card, tagsize) \ UPB_NOINLINE UPB_PRESERVE_NONE const char* UPB_DECODEFAST_FUNCNAME( \ type, card, tagsize)(UPB_PARSE_PARAMS) { \ - FASTDECODE_VARINT(UPB_DECODEFAST_FUNCNAME(type, Repeated, tagsize), \ - UPB_DECODEFAST_FUNCNAME(type, Packed, tagsize), \ - kUpb_DecodeFast_##card, UPB_PARSE_ARGS, \ - kUpb_DecodeFast_##type, kUpb_DecodeFast_##tagsize); \ + upb_DecodeFastNext next = kUpb_DecodeFastNext_Dispatch; \ + upb_DecodeFast_Varint(d, &ptr, msg, table, &hasbits, &data, &next, \ + kUpb_DecodeFast_##type, kUpb_DecodeFast_##card, \ + kUpb_DecodeFast_##tagsize); \ + UPB_DECODEFAST_NEXTMAYBEPACKED( \ + next, UPB_DECODEFAST_FUNCNAME(type, Repeated, tagsize), \ + UPB_DECODEFAST_FUNCNAME(type, Packed, tagsize)); \ } UPB_DECODEFAST_CARDINALITIES(UPB_DECODEFAST_TAGSIZES, F, Bool) diff --git a/upb/wire/decode_fast/function_array.c b/upb/wire/decode_fast/function_array.c index c6351e30fec32..4f6f7d208773e 100644 --- a/upb/wire/decode_fast/function_array.c +++ b/upb/wire/decode_fast/function_array.c @@ -7,6 +7,7 @@ #include "upb/wire/decode_fast/function_array.h" +#include #include #include "upb/mini_table/internal/message.h" diff --git a/upb/wire/eps_copy_input_stream.h b/upb/wire/eps_copy_input_stream.h index 06f196d44526e..f8c35c09c7016 100644 --- a/upb/wire/eps_copy_input_stream.h +++ b/upb/wire/eps_copy_input_stream.h @@ -424,7 +424,7 @@ UPB_INLINE const char* _upb_EpsCopyInputStream_IsDoneFallbackInline( } typedef const char* upb_EpsCopyInputStream_ParseDelimitedFunc( - upb_EpsCopyInputStream* e, const char* ptr, void* ctx); + upb_EpsCopyInputStream* e, const char* ptr, int size, void* ctx); // Tries to perform a fast-path handling of the given delimited message data. // If the sub-message beginning at `*ptr` and extending for `len` is short and @@ -445,7 +445,7 @@ UPB_FORCEINLINE bool upb_EpsCopyInputStream_TryParseDelimitedFast( e->limit_ptr = *ptr + len; e->limit = e->limit_ptr - e->end; UPB_ASSERT(e->limit_ptr == e->end + UPB_MIN(0, e->limit)); - *ptr = func(e, *ptr, ctx); + *ptr = func(e, *ptr, len, ctx); e->limit_ptr = saved_limit_ptr; e->limit = saved_limit; UPB_ASSERT(e->limit_ptr == e->end + UPB_MIN(0, e->limit)); diff --git a/upb/wire/internal/decoder.h b/upb/wire/internal/decoder.h index 8f5299d973237..70efc4aa481af 100644 --- a/upb/wire/internal/decoder.h +++ b/upb/wire/internal/decoder.h @@ -51,6 +51,7 @@ typedef struct upb_Decoder { #ifndef NDEBUG const char* debug_tagstart; const char* debug_valstart; + char* trace_buf; char* trace_ptr; char* trace_end; #endif @@ -72,6 +73,7 @@ UPB_INLINE const char* upb_Decoder_Init(upb_Decoder* d, const char* buf, d->status = kUpb_DecodeStatus_Ok; d->message_is_done = false; #ifndef NDEBUG + d->trace_buf = trace_buf; d->trace_ptr = trace_buf; d->trace_end = UPB_PTRADD(trace_buf, trace_size); #endif @@ -92,6 +94,32 @@ UPB_INLINE upb_DecodeStatus upb_Decoder_Destroy(upb_Decoder* d, return d->status; } +#ifndef NDEBUG +UPB_INLINE bool _upb_Decoder_TraceBufferFull(upb_Decoder* d) { + return d->trace_ptr == d->trace_end - 1; +} + +UPB_INLINE bool _upb_Decoder_TraceBufferAlmostFull(upb_Decoder* d) { + return d->trace_ptr == d->trace_end - 2; +} +#endif + +UPB_INLINE char* _upb_Decoder_TraceNext(upb_Decoder* d) { +#ifndef NDEBUG + return _upb_Decoder_TraceBufferAlmostFull(d) ? NULL : d->trace_ptr + 1; +#else + return NULL; +#endif +} + +UPB_INLINE char* _upb_Decoder_TracePtr(upb_Decoder* d) { +#ifndef NDEBUG + return d->trace_ptr; +#else + return NULL; +#endif +} + // Trace events are used to trace the progress of the decoder. // Events: // 'D' Fast dispatch @@ -102,7 +130,7 @@ UPB_INLINE upb_DecodeStatus upb_Decoder_Destroy(upb_Decoder* d, UPB_INLINE void _upb_Decoder_Trace(upb_Decoder* d, char event) { #ifndef NDEBUG if (d->trace_ptr == NULL) return; - if (d->trace_ptr == d->trace_end - 1) { + if (_upb_Decoder_TraceBufferFull(d)) { d->trace_ptr[-1] = 'X'; // Truncated. return; } @@ -139,8 +167,9 @@ const char* _upb_Decoder_DecodeMessage(upb_Decoder* d, const char* ptr, const upb_MiniTable* layout); UPB_INLINE bool _upb_Decoder_IsDone(upb_Decoder* d, const char** ptr) { - return upb_EpsCopyInputStream_IsDoneWithCallback( + bool ret = upb_EpsCopyInputStream_IsDoneWithCallback( &d->input, ptr, &_upb_Decoder_IsDoneFallback); + return ret; } UPB_NORETURN void* _upb_Decoder_ErrorJmp(upb_Decoder* d,