diff --git a/docs/combininglock.md b/docs/combininglock.md index ef5372b78..e705691dd 100644 --- a/docs/combininglock.md +++ b/docs/combininglock.md @@ -222,7 +222,7 @@ We can now provide our new version of the `with` function that uses the combinin template inline void with(CombiningLock& lock, F&& f) { - CombiningLockNodeTempl node(std::forward(f)); + CombiningLockNodeTempl node(proxy::forward(f)); // **************ACQUIRE************** // Add ourselves to the end of the queue. diff --git a/src/snmalloc/aal/aal.h b/src/snmalloc/aal/aal.h index 8c97f27c6..53d16f833 100644 --- a/src/snmalloc/aal/aal.h +++ b/src/snmalloc/aal/aal.h @@ -16,8 +16,10 @@ # define SNMALLOC_TICK_USE_CLOCK_GETTIME # endif #endif +#include "snmalloc/proxy/utility.h" + #include -#include + #ifndef SNMALLOC_TICK_USE_CLOCK_GETTIME # include diff --git a/src/snmalloc/backend/backend.h b/src/snmalloc/backend/backend.h index ee170c38f..9290f24ee 100644 --- a/src/snmalloc/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -87,7 +87,7 @@ namespace snmalloc * (remote, sizeclass, slab_metadata) * where slab_metadata, is the second element of the pair return. */ - static std::pair, SlabMetadata*> alloc_chunk( + static proxy::Pair, SlabMetadata*> alloc_chunk( LocalState& local_state, size_t size, uintptr_t ras, diff --git a/src/snmalloc/backend/fixedglobalconfig.h b/src/snmalloc/backend/fixedglobalconfig.h index eb3ab6918..e8600bc91 100644 --- a/src/snmalloc/backend/fixedglobalconfig.h +++ b/src/snmalloc/backend/fixedglobalconfig.h @@ -1,8 +1,11 @@ #pragma once #include "../backend_helpers/backend_helpers.h" +#include "snmalloc/proxy/type_traits.h" #include "standard_range.h" +#include + namespace snmalloc { /** @@ -117,9 +120,16 @@ namespace snmalloc { static_assert(B::wildness == capptr::dimension::Wildness::Wild); - static const size_t sz = sizeof( - std:: - conditional, void>, void*, T>); + constexpr size_t sz = []() constexpr { + if constexpr (proxy::is_same_v, void>) + { + return sizeof(void*); + } + else + { + return sizeof(T); + } + }(); UNUSED(ls); auto address = address_cast(p); diff --git a/src/snmalloc/backend_helpers/pagemap.h b/src/snmalloc/backend_helpers/pagemap.h index 13c559d91..31c760240 100644 --- a/src/snmalloc/backend_helpers/pagemap.h +++ b/src/snmalloc/backend_helpers/pagemap.h @@ -3,8 +3,7 @@ #include "../ds/ds.h" #include "../mem/mem.h" #include "snmalloc/proxy/atomic.h" - -#include +#include "snmalloc/proxy/utility.h" namespace snmalloc { @@ -115,7 +114,7 @@ namespace snmalloc */ template static SNMALLOC_FAST_PATH - proxy::enable_if_t> + proxy::enable_if_t> get_bounds() { static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); diff --git a/src/snmalloc/ds/combininglock.h b/src/snmalloc/ds/combininglock.h index ef6f3074a..d9c8600c9 100644 --- a/src/snmalloc/ds/combininglock.h +++ b/src/snmalloc/ds/combininglock.h @@ -256,7 +256,7 @@ namespace snmalloc reinterpret_cast(self); self_templ->f(); }), - f(std::forward(f_)) + f(proxy::forward(f_)) { attach_slow(lock); } @@ -290,6 +290,6 @@ namespace snmalloc // There is contention for the lock, we need to take the slow path // with the queue. - CombiningLockNodeTempl node(lock, std::forward(f)); + CombiningLockNodeTempl node(lock, proxy::forward(f)); } } // namespace snmalloc diff --git a/src/snmalloc/ds/flaglock.h b/src/snmalloc/ds/flaglock.h index 23847a282..86ca44ad5 100644 --- a/src/snmalloc/ds/flaglock.h +++ b/src/snmalloc/ds/flaglock.h @@ -24,7 +24,8 @@ namespace snmalloc constexpr DebugFlagWord() = default; template - constexpr DebugFlagWord(Args&&... args) : flag(std::forward(args)...) + constexpr DebugFlagWord(Args&&... args) + : flag(proxy::forward(args)...) {} /** @@ -87,7 +88,7 @@ namespace snmalloc template constexpr ReleaseFlagWord(Args&&... args) - : flag(std::forward(args)...) + : flag(proxy::forward(args)...) {} void set_owner() {} diff --git a/src/snmalloc/ds/pagemap.h b/src/snmalloc/ds/pagemap.h index 81aba9ca1..624afeb7e 100644 --- a/src/snmalloc/ds/pagemap.h +++ b/src/snmalloc/ds/pagemap.h @@ -115,7 +115,7 @@ namespace snmalloc * Returns usable range after pagemap has been allocated */ template - proxy::enable_if_t> + proxy::enable_if_t> init(void* b, size_t s) { SNMALLOC_ASSERT(!is_initialised()); @@ -250,7 +250,7 @@ namespace snmalloc } template - proxy::enable_if_t> get_bounds() + proxy::enable_if_t> get_bounds() { SNMALLOC_ASSERT(is_initialised()); diff --git a/src/snmalloc/ds_core/helpers.h b/src/snmalloc/ds_core/helpers.h index d0455800c..9c203ec63 100644 --- a/src/snmalloc/ds_core/helpers.h +++ b/src/snmalloc/ds_core/helpers.h @@ -3,6 +3,7 @@ #include "bits.h" #include "snmalloc/ds_core/defines.h" #include "snmalloc/proxy/type_traits.h" +#include "snmalloc/proxy/utility.h" #include #include @@ -357,8 +358,8 @@ namespace snmalloc if (fmt[0] == '{' && fmt[1] == '}') { - append(std::forward(head)); - return append(fmt + 2, std::forward(tail)...); + append(proxy::forward(head)); + return append(fmt + 2, proxy::forward(tail)...); } append_char(*fmt); @@ -374,7 +375,7 @@ namespace snmalloc SNMALLOC_FAST_PATH MessageBuilder(const char* fmt, Args&&... args) { buffer[SafeLength] = 0; - append(fmt, std::forward(args)...); + append(fmt, proxy::forward(args)...); append_char('\0'); } diff --git a/src/snmalloc/ds_core/seqset.h b/src/snmalloc/ds_core/seqset.h index 19c65025c..02c6a56f6 100644 --- a/src/snmalloc/ds_core/seqset.h +++ b/src/snmalloc/ds_core/seqset.h @@ -3,6 +3,7 @@ #include "../aal/aal.h" #include "../ds_core/ds_core.h" #include "snmalloc/proxy/type_traits.h" +#include "snmalloc/proxy/utility.h" #include @@ -98,7 +99,7 @@ namespace snmalloc SNMALLOC_FAST_PATH bool is_empty() { static_assert( - proxy::is_same_v().node)>, + proxy::is_same_v().node)>, "T->node must be Node for T"); head.invariant(); return head.next == &head; diff --git a/src/snmalloc/mem/backend_concept.h b/src/snmalloc/mem/backend_concept.h index f72f575d7..4fbb16b7d 100644 --- a/src/snmalloc/mem/backend_concept.h +++ b/src/snmalloc/mem/backend_concept.h @@ -19,11 +19,11 @@ namespace snmalloc requires(address_t addr, size_t sz, const typename Pagemap::Entry& t) { { Pagemap::template get_metaentry(addr) - } -> ConceptSame; + } -> ConceptSame; { Pagemap::template get_metaentry(addr) - } -> ConceptSame; + } -> ConceptSame; }; /** @@ -39,15 +39,13 @@ namespace snmalloc requires(address_t addr, size_t sz, const typename Pagemap::Entry& t) { { Pagemap::template get_metaentry_mut(addr) - } -> ConceptSame; + } -> ConceptSame; { Pagemap::template get_metaentry_mut(addr) - } -> ConceptSame; + } -> ConceptSame; - { - Pagemap::set_metaentry(addr, sz, t) - } -> ConceptSame; + { Pagemap::set_metaentry(addr, sz, t) } -> ConceptSame; }; /** @@ -59,10 +57,8 @@ namespace snmalloc */ template concept IsPagemapWithRegister = requires(capptr::Arena p, size_t sz) { - { - Pagemap::register_range(p, sz) - } -> ConceptSame; - }; + { Pagemap::register_range(p, sz) } -> ConceptSame; + }; /** * The full pagemap accessor interface, with all of {get,set}_metadata and @@ -87,11 +83,11 @@ namespace snmalloc requires(typename Config::LocalState* ls, capptr::AllocWild ptr) { { Config::capptr_domesticate(ls, ptr) - } -> ConceptSame>; + } -> ConceptSame>; { Config::capptr_domesticate(ls, ptr.template as_static()) - } -> ConceptSame>; + } -> ConceptSame>; }; class CommonConfig; @@ -106,13 +102,13 @@ namespace snmalloc sizeclass_t sizeclass) { { Backend::alloc_chunk(local_state, size, ras, sizeclass) - } -> ConceptSame< - std::pair, typename Backend::SlabMetadata*>>; + } -> ConceptSame< + proxy::Pair, typename Backend::SlabMetadata*>>; } && requires(LocalState* local_state, size_t size) { { Backend::template alloc_meta_data(local_state, size) - } -> ConceptSame>; + } -> ConceptSame>; } && requires( LocalState& local_state, @@ -123,16 +119,16 @@ namespace snmalloc { Backend::dealloc_chunk( local_state, slab_metadata, alloc, size, sizeclass) - } -> ConceptSame; + } -> ConceptSame; } && requires(address_t p) { { Backend::template get_metaentry(p) - } -> ConceptSame; + } -> ConceptSame; { Backend::template get_metaentry(p) - } -> ConceptSame; + } -> ConceptSame; }; /** @@ -152,35 +148,30 @@ namespace snmalloc IsPAL && IsBackend && + typename Config::Backend>&& + requires() + { + typename Config::LocalState; + typename Config::Backend; + typename Config::PagemapEntry; + + { + Config::Options + } -> ConceptSameModRef; + } + &&( requires() { - typename Config::LocalState; - typename Config::Backend; - typename Config::PagemapEntry; - - { - Config::Options - } -> ConceptSameModRef; - } && - ( - requires() { - Config::Options.CoreAllocIsPoolAllocated == true; - typename Config::GlobalPoolState; - { - Config::pool() - } -> ConceptSame; - } || - requires() { - Config::Options.CoreAllocIsPoolAllocated == false; - }); + Config::Options.CoreAllocIsPoolAllocated == true; + typename Config::GlobalPoolState; + { Config::pool() } -> ConceptSame; + } || requires() { Config::Options.CoreAllocIsPoolAllocated == false; }); /** * The lazy version of the above; please see ds_core/concept.h and use * sparingly. */ template - concept IsConfigLazy = ! - is_type_complete_v || IsConfig; + concept IsConfigLazy = !is_type_complete_v || IsConfig; } // namespace snmalloc diff --git a/src/snmalloc/mem/backend_wrappers.h b/src/snmalloc/mem/backend_wrappers.h index 59ec407f1..0a2f597ea 100644 --- a/src/snmalloc/mem/backend_wrappers.h +++ b/src/snmalloc/mem/backend_wrappers.h @@ -17,8 +17,8 @@ namespace snmalloc * it if it exists. */ template - SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) - -> decltype(T::is_initialised()) + SNMALLOC_FAST_PATH auto + call_is_initialised(T*, int) -> decltype(T::is_initialised()) { return T::is_initialised(); } @@ -43,17 +43,17 @@ namespace snmalloc SNMALLOC_CONCEPT(IsConfigDomestication) Config, typename T, SNMALLOC_CONCEPT(capptr::IsBound) B> - constexpr SNMALLOC_FAST_PATH auto has_domesticate(int) - -> proxy::enable_if_t< - proxy::is_same_v< - decltype(Config::capptr_domesticate( - std::declval(), - std::declval>())), - CapPtr< - T, - typename B::template with_wildness< - capptr::dimension::Wildness::Tame>>>, - bool> + constexpr SNMALLOC_FAST_PATH auto + has_domesticate(int) -> proxy::enable_if_t< + proxy::is_same_v< + decltype(Config::capptr_domesticate( + proxy::declval(), + proxy::declval>())), + CapPtr< + T, + typename B::template with_wildness< + capptr::dimension::Wildness::Tame>>>, + bool> { return true; } diff --git a/src/snmalloc/mem/freelist.h b/src/snmalloc/mem/freelist.h index e62ef8ca1..6e47ee20f 100644 --- a/src/snmalloc/mem/freelist.h +++ b/src/snmalloc/mem/freelist.h @@ -906,7 +906,7 @@ namespace snmalloc template proxy::enable_if_t< !RANDOM_, - std::pair< + proxy::Pair< Object::BHeadPtr, Object::BHeadPtr>> extract_segment(const FreeListKey& key, address_t key_tweak) diff --git a/src/snmalloc/mem/localalloc.h b/src/snmalloc/mem/localalloc.h index 101ba7b9e..3681601ca 100644 --- a/src/snmalloc/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -21,8 +21,10 @@ # include "external_alloc.h" #endif +#include "snmalloc/proxy/utility.h" + #include -#include + namespace snmalloc { @@ -329,8 +331,8 @@ namespace snmalloc * if it exists. */ template - SNMALLOC_FAST_PATH auto call_ensure_init(T*, int) - -> decltype(T::ensure_init()) + SNMALLOC_FAST_PATH auto + call_ensure_init(T*, int) -> decltype(T::ensure_init()) { T::ensure_init(); } diff --git a/src/snmalloc/mem/metadata.h b/src/snmalloc/mem/metadata.h index c51f90a51..47aa80200 100644 --- a/src/snmalloc/mem/metadata.h +++ b/src/snmalloc/mem/metadata.h @@ -620,7 +620,7 @@ namespace snmalloc * available objects for this slab metadata. */ template - static SNMALLOC_FAST_PATH std::pair + static SNMALLOC_FAST_PATH proxy::Pair alloc_free_list( Domesticator domesticate, FrontendSlabMetadata* meta, diff --git a/src/snmalloc/mem/remoteallocator.h b/src/snmalloc/mem/remoteallocator.h index 4dbf5bfbe..885d6f264 100644 --- a/src/snmalloc/mem/remoteallocator.h +++ b/src/snmalloc/mem/remoteallocator.h @@ -104,7 +104,7 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH static std::pair + SNMALLOC_FAST_PATH static proxy::Pair open_free_ring( capptr::Alloc m, size_t objsize, @@ -242,7 +242,7 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH static std::pair + SNMALLOC_FAST_PATH static proxy::Pair open_free_ring( capptr::Alloc m, size_t, diff --git a/src/snmalloc/pal/pal.h b/src/snmalloc/pal/pal.h index f25cb6662..8015042bb 100644 --- a/src/snmalloc/pal/pal.h +++ b/src/snmalloc/pal/pal.h @@ -168,14 +168,14 @@ namespace snmalloc template [[noreturn]] inline void report_fatal_error(Args... args) { - MessageBuilder msg{std::forward(args)...}; + MessageBuilder msg{proxy::forward(args)...}; DefaultPal::error(msg.get_message()); } template inline void message(Args... args) { - MessageBuilder msg{std::forward(args)...}; + MessageBuilder msg{proxy::forward(args)...}; MessageBuilder msg_tid{ "{}: {}", DefaultPal::get_tid(), msg.get_message()}; DefaultPal::message(msg_tid.get_message()); diff --git a/src/snmalloc/pal/pal_posix.h b/src/snmalloc/pal/pal_posix.h index 73f174a68..6447724b2 100644 --- a/src/snmalloc/pal/pal_posix.h +++ b/src/snmalloc/pal/pal_posix.h @@ -6,6 +6,8 @@ #if defined(SNMALLOC_BACKTRACE_HEADER) # include SNMALLOC_BACKTRACE_HEADER #endif +#include "snmalloc/proxy/utility.h" + #include #include #include @@ -16,7 +18,7 @@ #include #include #include -#include + #if __has_include() # include diff --git a/src/snmalloc/proxy/type_traits.h b/src/snmalloc/proxy/type_traits.h index 76e1a87ee..1fcbd260f 100644 --- a/src/snmalloc/proxy/type_traits.h +++ b/src/snmalloc/proxy/type_traits.h @@ -403,6 +403,22 @@ namespace snmalloc template using add_const_t = const T; +# if __has_builtin(__is_lvalue_reference) + template + inline constexpr bool is_lvalue_reference_v = __is_lvalue_reference(T); +# else + template + struct is_lvalue_reference : public false_type + {}; + + template + struct is_lvalue_reference : public true_type + {}; + + template + inline constexpr bool is_lvalue_reference_v = is_lvalue_reference::value; +# endif + } // namespace proxy } // namespace snmalloc @@ -436,6 +452,7 @@ namespace snmalloc using std::is_function_v; using std::is_integral; using std::is_integral_v; + using std::is_lvalue_reference; using std::is_move_assignable_v; using std::is_move_constructible_v; using std::is_same; diff --git a/src/snmalloc/proxy/utility.h b/src/snmalloc/proxy/utility.h new file mode 100644 index 000000000..9d69babda --- /dev/null +++ b/src/snmalloc/proxy/utility.h @@ -0,0 +1,91 @@ +#pragma once + +#ifndef SNMALLOC_USE_SELF_VENDORED_STL +# define SNMALLOC_USE_SELF_VENDORED_STL 0 +#endif + +#if SNMALLOC_USE_SELF_VENDORED_STL + +# include "snmalloc/proxy/type_traits.h" + +# if !defined(__GNUC__) && !defined(__clang__) +# error "cannot use vendored STL without GNU/Clang extensions" +# endif + +# if __has_cpp_attribute(_Clang::__lifetimebound__) +# define SNMALLOC_LIFETIMEBOUND [[_Clang::__lifetimebound__]] +# else +# define SNMALLOC_LIFETIMEBOUND +# endif + +namespace snmalloc +{ + namespace proxy + { + template + [[nodiscard]] inline constexpr T&& + forward(remove_reference_t& ref) noexcept + { + return static_cast(ref); + } + + template + [[nodiscard]] inline constexpr T&& + forward(SNMALLOC_LIFETIMEBOUND remove_reference_t&& ref) noexcept + { + static_assert( + !is_lvalue_reference_v, "cannot forward an rvalue as an lvalue"); + return static_cast(ref); + } + + template + [[nodiscard]] inline constexpr remove_reference_t&& + move(SNMALLOC_LIFETIMEBOUND T&& ref) noexcept + { +# ifdef __clang__ + using U [[gnu::nodebug]] = remove_reference_t; +# else + using U = remove_reference_t; +# endif + return static_cast(ref); + } + +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wdeprecated" + template + T&& declval_impl(int); + template + T declval_impl(long); +# pragma GCC diagnostic pop + + template + constexpr inline decltype(declval_impl(0)) declval() noexcept + { + static_assert( + !is_same_v, "declval cannot be used in an evaluation context"); + } + + template + struct Pair + { + T1 first; + T2 second; + }; + } // namespace proxy +} // namespace snmalloc +#else + +# include + +namespace snmalloc +{ + namespace proxy + { + using std::declval; + using std::forward; + using std::move; + template + using Pair = std::pair; + } // namespace proxy +} // namespace snmalloc +#endif