diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 10edaea33..db0b7ca3f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ permissions: contents: write env: - RELEASE: Release v3.3.2 + RELEASE: Release v3.4.1 FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true name: Release @@ -19,7 +19,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - branch: [v1.9.10,v2.3.2,v3.3.2] # [dev,dev2,dev3] + branch: [v1.9.11,v2.4.1,v3.4.1] # [dev,dev2,dev3] # we build on the oldest ubuntu version for better binary compatibility. os: [windows-latest, macOS-latest, macos-15-intel, ubuntu-22.04, ubuntu-22.04-arm] @@ -45,7 +45,7 @@ jobs: # Create a release: https://github.com/softprops/action-gh-release (MIT license) - name: Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 if: ${{ env.RELEASE != '' && env.RELEASE != 'no' }} with: name: ${{ env.RELEASE }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 89d4170fe..df85b16cb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -59,7 +59,7 @@ jobs: ctest --test-dir out/debug-cxx --verbose --timeout 240 -C Debug - name: Release, C++ - if: matrix.tests == 'basic' && runner.os == 'Linux' # windows already uses C++ by default; macOS interpose does not work well with C++ + if: matrix.tests == 'basic' && runner.os != 'Windows' # windows already uses C++ by default run: | cmake . -B out/release-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON cmake --build out/release-cxx --parallel 8 --config Release diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e7ee8b3d..1dc1e782c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -381,7 +381,7 @@ endif() if(MI_USE_CXX) message(STATUS "Use the C++ compiler to compile (MI_USE_CXX=ON)") set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX ) - set_source_files_properties(src/static.c test/test-api.c test/test-api-fill test/test-stress PROPERTIES LANGUAGE CXX ) + set_source_files_properties(src/static.c test/test-api.c test/test-api-fill.c test/test-stress.c PROPERTIES LANGUAGE CXX ) if(CMAKE_CXX_COMPILER_ID MATCHES "AppleClang|Clang") list(APPEND mi_cflags -Wno-deprecated) endif() @@ -762,7 +762,7 @@ if (MI_BUILD_TESTS) endforeach() # dynamic override test - if(MI_BUILD_SHARED AND NOT (MI_TRACK_ASAN OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) + if(MI_BUILD_SHARED AND NOT (MI_TRACK_ASAN OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN) AND NOT (APPLE AND MI_USE_CXX)) add_executable(mimalloc-test-stress-dynamic test/test-stress.c) target_compile_definitions(mimalloc-test-stress-dynamic PRIVATE ${mi_defines} "USE_STD_MALLOC=1") target_compile_options(mimalloc-test-stress-dynamic PRIVATE ${mi_cflags}) diff --git a/bin/bundle.sh b/bin/bundle.sh index 954b135cc..bc864b359 100755 --- a/bin/bundle.sh +++ b/bin/bundle.sh @@ -24,7 +24,7 @@ warn() { } stop() { - warn $@ + warn "$@" exit 1 } @@ -103,7 +103,11 @@ process_options() { -p) shift PREFIX="$1";; -p=*|--prefix=*) - PREFIX=`eval echo $flag_arg`;; # no quotes so ~ gets expanded (issue #412) + case "$flag_arg" in + "~") PREFIX="$HOME" ;; + "~/"*) PREFIX="$HOME/${flag_arg#~/}" ;; + *) PREFIX="$flag_arg" ;; + esac;; -h|--help|-\?|help|\?) MODE="help";; *) case "$flag" in @@ -139,7 +143,7 @@ download_file() { # fi;; *) info "Copying: $1" - if ! cp $1 $2 ; then + if ! cp "$1" "$2" ; then stop "Unable to copy from $1" fi;; esac @@ -220,7 +224,7 @@ main_help() { main_start() { detect_osarch detect_git_tag - process_options $@ + process_options "$@" if [ "$MODE" = "help" ] ; then main_help else @@ -229,4 +233,4 @@ main_start() { } # note: only start executing commands now to guard against partial downloads -main_start $@ +main_start "$@" diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 649eb71d0..261112af1 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 2) -set(mi_version_minor 3) -set(mi_version_patch 2) +set(mi_version_minor 4) +set(mi_version_patch 1) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/contrib/vcpkg/portfile.cmake b/contrib/vcpkg/portfile.cmake index a1bce75d3..1cf6ee53d 100644 --- a/contrib/vcpkg/portfile.cmake +++ b/contrib/vcpkg/portfile.cmake @@ -3,7 +3,7 @@ vcpkg_from_github( REPO microsoft/mimalloc HEAD_REF master - # The "REF" can be a commit hash, branch name (dev3), or a version (v3.3.2). + # The "REF" can be a commit hash, branch name (dev3), or a version (v3.4.1). REF "v${VERSION}" # REF e2db21e9ba9fb9172b7b0aa0fe9b8742525e8774 diff --git a/contrib/vcpkg/vcpkg.json b/contrib/vcpkg/vcpkg.json index 84ae68f32..b797c18ff 100644 --- a/contrib/vcpkg/vcpkg.json +++ b/contrib/vcpkg/vcpkg.json @@ -1,6 +1,6 @@ { "name": "mimalloc", - "version": "3.3.2", + "version": "3.4.1", "port-version": 0, "description": "Compact general purpose allocator with excellent performance", "homepage": "https://github.com/microsoft/mimalloc", diff --git a/doc/release-notes.md b/doc/release-notes.md index b467c1468..1e59f4a53 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -10,6 +10,6 @@ Notes: - Generally it is recommended to download sources (or use `vcpkg` etc.) and build mimalloc as part of your project. - Source releases can also be downloaded directly from github by the tag. - For example . + For example . - Binary releases include a release-, debug-, and secure build. - Linux binaries are built on Ubuntu 22. diff --git a/include/mimalloc.h b/include/mimalloc.h index b61349ce6..aca8bd387 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 20302 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 20401 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes @@ -95,7 +95,7 @@ terms of the MIT license. A copy of the license can be found in the file // Includes // ------------------------------------------------------ -#include // size_t +#include // size_t, wchar_t #include // bool #include // INTPTR_MAX @@ -467,7 +467,7 @@ mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_ mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; -mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; +mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size); // mi_attr_noexcept; mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); @@ -484,7 +484,6 @@ mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; // wide characters -#include // wchar_t mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc; mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc; diff --git a/include/mimalloc/atomic.h b/include/mimalloc/atomic.h index 5ac8da6f9..618074b9b 100644 --- a/include/mimalloc/atomic.h +++ b/include/mimalloc/atomic.h @@ -416,7 +416,8 @@ static inline void mi_atomic_yield(void) { #elif (defined(__GNUC__) || defined(__clang__)) && \ (defined(__x86_64__) || defined(__i386__) || \ defined(__aarch64__) || defined(__arm__) || \ - defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__)) + defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__) || \ + defined(__riscv)) #if defined(__x86_64__) || defined(__i386__) static inline void mi_atomic_yield(void) { __asm__ volatile ("pause" ::: "memory"); @@ -445,6 +446,16 @@ static inline void mi_atomic_yield(void) { __asm__ __volatile__ ("or 27,27,27" ::: "memory"); } #endif +#elif defined(__riscv) +#if defined(__riscv_zihintpause) +static inline void mi_atomic_yield(void) { + __asm__ volatile("pause" ::: "memory"); +} +#else +static inline void mi_atomic_yield(void) { + __asm__ volatile("nop" ::: "memory"); +} +#endif #endif #elif defined(__sun) #include @@ -469,6 +480,32 @@ static inline void mi_atomic_yield(void) { } #endif +#if defined(_WIN32) +static inline void mi_sleep0(void) { + Sleep(0); +} +#else +#include +static inline void mi_sleep0(void) { + sleep(0); +} +#endif + +static inline void mi_atomic_yield_sleep( size_t* ticks, const size_t ticks_until_sleep ) { + const size_t n = *ticks; + if (n==0 || n > ticks_until_sleep) { + *ticks = ticks_until_sleep; // reset + } + else { + *ticks = n-1; + } + if (n>1) { + mi_atomic_yield(); + } + else { + mi_sleep0(); + } +} // ---------------------------------------------------------------------- // Locks @@ -589,9 +626,10 @@ static inline bool mi_lock_try_acquire(mi_lock_t* lock) { return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1); } static inline void mi_lock_acquire(mi_lock_t* lock) { + size_t ticks = 0; for (int i = 0; i < 10000; i++) { // for at most 10000 tries? if (mi_lock_try_acquire(lock)) return; - mi_atomic_yield(); + mi_atomic_yield_sleep(&ticks,100); } _mi_error_message(EFAULT, "internal error: lock cannot be acquired (due to lack of native lock primitives)\n"); } diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 4553c0fb3..06fe9f823 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -110,7 +110,7 @@ void _mi_strlcat(char* dest, const char* src, size_t dest_size); size_t _mi_strlen(const char* s); size_t _mi_strnlen(const char* s, size_t max_len); bool _mi_streq(const char* s, const char* t); -bool _mi_getenv(const char* name, char* result, size_t result_size); +int _mi_getenv(const char* name, char* result, size_t result_size); // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); @@ -320,6 +320,9 @@ bool _mi_page_is_valid(mi_page_t* page); #ifndef EOVERFLOW // count*size overflow #define EOVERFLOW (75) #endif +#ifndef ENOENT // environment variable not found +#define ENOENT (2) +#endif // ------------------------------------------------------ @@ -718,7 +721,7 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* #else MI_UNUSED(block); MI_UNUSED(p); return false; -#endif +#endif } #if MI_GUARDED diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index b963ad5dc..51ef18cc2 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -106,7 +106,9 @@ void _mi_prim_out_stderr( const char* msg ); // Get an environment variable. (only for options) // name != NULL, result != NULL, result_size >= 64 -bool _mi_prim_getenv(const char* name, char* result, size_t result_size); +// Return 1 for success, 0 if not found, +// and -1 on error (for example, if `getenv` cannot be called yet during preloading). +int _mi_prim_getenv(const char* name, char* result, size_t result_size); // Fill a buffer with strong randomness; return `false` on error or if @@ -139,7 +141,7 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); // but unfortunately we can not detect support reliably (see issue #883) // We also use it on Apple OS as we use a TLS slot for the default heap there. #if defined(__GNUC__) && ( \ - (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \ + (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__) || defined(__riscv))) \ || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \ || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \ || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ @@ -171,6 +173,10 @@ static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept { __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); #endif res = tcb[slot]; + #elif defined(__riscv) + void** tcb; MI_UNUSED(ofs); + __asm__ volatile ("mv %0, tp" : "=r" (tcb)); + res = tcb[slot]; #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781 MI_UNUSED(ofs); res = pthread_getspecific(slot); @@ -201,6 +207,10 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); #endif tcb[slot] = value; + #elif defined(__riscv) + void** tcb; MI_UNUSED(ofs); + __asm__ volatile ("mv %0, tp" : "=r" (tcb)); + tcb[slot] = value; #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781 MI_UNUSED(ofs); pthread_setspecific(slot, value); @@ -263,6 +273,7 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce && !defined(MI_LIBC_MUSL) \ && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot () */ #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \ + || (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__riscv)) \ || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \ || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__))) #define MI_USE_BUILTIN_THREAD_POINTER 1 diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index f13cd331a..dd390ef0b 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -465,7 +465,7 @@ typedef struct mi_subproc_s mi_subproc_t; typedef struct mi_segment_s { // constant fields mi_memid_t memid; // memory id for arena/OS allocation - bool allow_decommit; // can we decommmit the memory + bool allow_decommit; // can we decommit the memory bool allow_purge; // can we purge the memory (reset or decommit) size_t segment_size; mi_subproc_t* subproc; // segment belongs to sub process diff --git a/readme.md b/readme.md index bb788c307..5f75e7c35 100644 --- a/readme.md +++ b/readme.md @@ -15,9 +15,9 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.3.2` (2026-04-29) recommended. -Latest v2 release: `v2.3.2` (2026-04-29) stable. -Latest v1 release: `v1.9.10` (2026-04-29) legacy. +Latest release : `v3.4.1` (2026-07-14) recommended. +Latest v2 release: `v2.4.1` (2026-07-14) stable. +Latest v1 release: `v1.9.11` (2026-07-14) legacy. mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -88,6 +88,10 @@ New development is mostly on v3, while v1 and v2 are maintained with security an - __v1__: legacy version: initial design of mimalloc (release tags: `v1.9.x`, development branch `dev`). Send PR's against this version if possible. ### Releases +* 2026-07-14, `v1.9.11`, `v2.4.1`, `v3.4.1`: various bug and security fixes through LLM audit (by @Zoxc). + Fix issue with using OS memory instead of arenas for > 4GiB memory usage (v3), fix concurrency bug in concurrent + heap destroy (v3), detect riscV virtual address bits at runtime, add riscV TLS support, reduce spinlock waits (v2), + use TLS slots 126/127 on macOS (v3), and other small fixes. All metadata is now separated from heap objects in v3. * 2026-04-29, `v1.9.10`, `v2.3.2`, `v3.3.2`: various bug and security fixes through LLM audit (by @Zoxc). Only increase minimal purge size automatically if allow_thp is set to 2. Enable large OS alignment on all platforms (fixing OS large pages on Windows). Fix accounting of committed memory on Linux/macOS. diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 60639ff49..41135f38f 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -52,7 +52,7 @@ void mi_cfree(void* p) mi_attr_noexcept { } } -int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept { +int mi_posix_memalign(void** p, size_t alignment, size_t size) { // mi_attr_noexcept (issue #794) // Note: The spec dictates we should not modify `*p` on an error. (issue#27) // if (p == NULL) return EINVAL; diff --git a/src/arena.c b/src/arena.c index 0ba4f9985..f28113fcf 100644 --- a/src/arena.c +++ b/src/arena.c @@ -702,12 +702,12 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi // checks if (arena == NULL) { - _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memid: 0x%zx\n", p, size, memid); + _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind); return; } mi_assert_internal(arena->field_count > mi_bitmap_index_field(bitmap_idx)); if (arena->field_count <= mi_bitmap_index_field(bitmap_idx)) { - _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid); + _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memid: 0x%x\n", p, size, memid.memkind); return; } diff --git a/src/heap.c b/src/heap.c index 219fb8db4..9a40ebbe3 100644 --- a/src/heap.c +++ b/src/heap.c @@ -155,7 +155,11 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) // free all current thread delayed blocks. // (if abandoning, after this there are no more thread-delayed references into the pages.) - _mi_heap_delayed_free_all(heap); + if (collect >= MI_ABANDON) { + _mi_heap_delayed_free_all(heap); // free all + } else { + _mi_heap_delayed_free_partial(heap); // best-effort; contended blocks are retried next time + } // collect retired pages _mi_heap_collect_retired(heap, force); diff --git a/src/init.c b/src/init.c index 3cc32358b..d6977b370 100644 --- a/src/init.c +++ b/src/init.c @@ -673,8 +673,9 @@ void mi_process_init(void) mi_attr_noexcept { } } -// Called when the process is done (cdecl as it is used with `at_exit` on some platforms) -void mi_cdecl mi_process_done(void) mi_attr_noexcept { + +// Called when the process is done +static void mi_process_done_once(void) { // only shutdown if we were initialized if (!_mi_process_is_initialized) return; // ensure we are called once @@ -700,7 +701,7 @@ void mi_cdecl mi_process_done(void) mi_attr_noexcept { #endif // done with tracking tools - mi_track_done() + mi_track_done(); // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free // since after process_done there might still be other code running that calls `free` (like at_exit routines, @@ -720,6 +721,14 @@ void mi_cdecl mi_process_done(void) mi_attr_noexcept { os_preloading = true; // don't call the C runtime anymore } + +// Called when the process is done (cdecl as it is used with `at_exit` on some platforms) +void mi_cdecl mi_process_done(void) mi_attr_noexcept { + mi_atomic_do_once { + mi_process_done_once(); + } +} + void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept { if (_mi_option_get_fast(mi_option_destroy_on_exit)>1) return; mi_process_done(); diff --git a/src/libc.c b/src/libc.c index 3eefa52d0..41903cd08 100644 --- a/src/libc.c +++ b/src/libc.c @@ -64,7 +64,7 @@ void _mi_strlcat(char* dest, const char* src, size_t dest_size) { size_t _mi_strnlen(const char* s, size_t max_len) { if (s==NULL) return 0; size_t len = 0; - while(s[len] != 0 && len < max_len) { len++; } + while(len < max_len && s[len] != 0) { len++; } return len; } @@ -73,16 +73,18 @@ size_t _mi_strlen(const char* s) { } #ifdef MI_NO_GETENV -bool _mi_getenv(const char* name, char* result, size_t result_size) { +int _mi_getenv(const char* name, char* result, size_t result_size) { MI_UNUSED(name); MI_UNUSED(result); MI_UNUSED(result_size); - return false; + return ENOENT; } #else -bool _mi_getenv(const char* name, char* result, size_t result_size) { +int _mi_getenv(const char* name, char* result, size_t result_size) { if (name==NULL || result == NULL || result_size < 64) return false; - return _mi_prim_getenv(name,result,result_size); + // change the result of _mi_prim_getenv to an errno result + const int res = _mi_prim_getenv(name,result,result_size); + return (res > 0 ? 0 : (res == 0 ? ENOENT : EAGAIN)); } #endif @@ -102,13 +104,13 @@ bool _mi_atomic_once_enter(mi_atomic_once_t* once) { const mi_threadid_t current_tid = _mi_thread_id(); if (once_tid == current_tid) { return false; // recursive invocation; we need this for process_init for example - } + } mi_lock_acquire(&once->lock); uintptr_t expected = 0; if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) { // could use atomic_load/store as well return true; // should execute and release - } + } else { mi_lock_release(&once->lock); return false; // already another thread entered and released @@ -119,7 +121,7 @@ void _mi_atomic_once_release(mi_atomic_once_t* once) { if (mi_atomic_load_acquire(&once->tid)>1) { // paranoia mi_atomic_store_release(&once->tid,1); // done executing mi_lock_release(&once->lock); - } + } } diff --git a/src/options.c b/src/options.c index b02e011b7..98f0790ee 100644 --- a/src/options.c +++ b/src/options.c @@ -168,7 +168,7 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(guarded_sample_seed)}, { 0, UNINIT, MI_OPTION(target_segments_per_thread) }, // abandon segments beyond this point, or 0 to disable. { 10000, UNINIT, MI_OPTION(generic_collect) }, // collect heaps every N (=10000) generic allocation calls - { MI_DEFAULT_ALLOW_THP, + { MI_DEFAULT_ALLOW_THP, UNINIT, MI_OPTION(allow_thp) } // allow transparent huge pages? }; @@ -611,17 +611,17 @@ static void mi_option_init(mi_option_desc_t* desc) { char buf[64+1]; _mi_strlcpy(buf, "mimalloc_", sizeof(buf)); _mi_strlcat(buf, desc->name, sizeof(buf)); - bool found = _mi_getenv(buf, s, sizeof(s)); - if (!found && desc->legacy_name != NULL) { + int err = _mi_getenv(buf, s, sizeof(s)); + if (err==ENOENT && desc->legacy_name != NULL) { _mi_strlcpy(buf, "mimalloc_", sizeof(buf)); _mi_strlcat(buf, desc->legacy_name, sizeof(buf)); - found = _mi_getenv(buf, s, sizeof(s)); - if (found) { + err = _mi_getenv(buf, s, sizeof(s)); + if (err==0) { _mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name); } } - if (found) { + if (err==0) { size_t len = _mi_strnlen(s, sizeof(buf) - 1); for (size_t i = 0; i < len; i++) { buf[i] = _mi_toupper(s[i]); @@ -631,7 +631,7 @@ static void mi_option_init(mi_option_desc_t* desc) { desc->value = 1; desc->init = INITIALIZED; } - else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) { + else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) { desc->value = 0; desc->init = INITIALIZED; } @@ -673,7 +673,8 @@ static void mi_option_init(mi_option_desc_t* desc) { } mi_assert_internal(desc->init != UNINIT); } - else if (!_mi_preloading()) { + else if (err==ENOENT) { desc->init = DEFAULTED; } + // and on another error, keep unitialized to try again (can happen during preloading if getenv is not available) } diff --git a/src/os.c b/src/os.c index b60080a7f..08d0f672d 100644 --- a/src/os.c +++ b/src/os.c @@ -105,7 +105,7 @@ static mi_decl_cache_align _Atomic(uintptr_t)aligned_base; // Return a MI_SEGMENT_SIZE aligned address that is probably available. // If this returns NULL, the OS will determine the address but on some OS's that may not be // properly aligned which can be more costly as it needs to be adjusted afterwards. -// For a size > 1GiB this always returns NULL in order to guarantee good ASLR randomization; +// In secure mode, for a size > 1GiB this always returns NULL in order to guarantee good ASLR randomization. // (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses // in the middle of the 2TiB - 6TiB address range (see issue #372)) @@ -118,7 +118,9 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL; if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space size = _mi_align_up(size, MI_SEGMENT_SIZE); + #if (MI_SECURE>0) if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096. + #endif size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); diff --git a/src/page-queue.c b/src/page-queue.c index bfe9f4fdc..c8f9e245a 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -107,10 +107,10 @@ size_t _mi_bin_size(size_t bin) { // Good size for allocation size_t mi_good_size(size_t size) mi_attr_noexcept { - if (size <= MI_MEDIUM_OBJ_SIZE_MAX) { + if (size <= MI_MEDIUM_OBJ_SIZE_MAX - MI_PADDING_SIZE) { return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE)); } - else if (size <= MI_MAX_ALLOC_SIZE) { + else if (size <= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size()); } else { diff --git a/src/page.c b/src/page.c index 9a8aac0e3..71e1724ae 100644 --- a/src/page.c +++ b/src/page.c @@ -161,8 +161,9 @@ bool _mi_page_is_valid(mi_page_t* page) { #endif void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) { + size_t ticks = 0; while (!_mi_page_try_use_delayed_free(page, delay, override_never)) { - mi_atomic_yield(); + mi_atomic_yield_sleep(&ticks,10); } } @@ -342,8 +343,9 @@ static mi_page_t* mi_page_fresh(mi_heap_t* heap, mi_page_queue_t* pq) { (put there by other threads if they deallocated in a full page) ----------------------------------------------------------- */ void _mi_heap_delayed_free_all(mi_heap_t* heap) { + size_t ticks = 0; while (!_mi_heap_delayed_free_partial(heap)) { - mi_atomic_yield(); + mi_atomic_yield_sleep(&ticks,10); } } diff --git a/src/prim/emscripten/prim.c b/src/prim/emscripten/prim.c index 93c76757d..a9b9266a9 100644 --- a/src/prim/emscripten/prim.c +++ b/src/prim/emscripten/prim.c @@ -184,12 +184,12 @@ void _mi_prim_out_stderr( const char* msg) { // Environment //---------------------------------------------------------------- -bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { +int _mi_prim_getenv(const char* name, char* result, size_t result_size) { // For code size reasons, do not support environ customization for now. MI_UNUSED(name); MI_UNUSED(result); MI_UNUSED(result_size); - return false; + return 0; // not found } diff --git a/src/prim/osx/alloc-override-zone.c b/src/prim/osx/alloc-override-zone.c index 3b1d10706..0aa6cc7d2 100644 --- a/src/prim/osx/alloc-override-zone.c +++ b/src/prim/osx/alloc-override-zone.c @@ -41,10 +41,19 @@ extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_im malloc zone members ------------------------------------------------------ */ +static bool is_mimalloc_zone( malloc_zone_t* zone ); + static size_t zone_size(malloc_zone_t* zone, const void* p) { MI_UNUSED(zone); - if (!mi_is_in_heap_region(p)){ return 0; } // not our pointer, bail out - return mi_usable_size(p); + if (mi_is_in_heap_region(p)) { + return mi_usable_size(p); + } + else if (!is_mimalloc_zone(zone)) { // can happen due to interpose + return zone->size(zone,p); + } + else { + return 0; + } } static void* zone_malloc(malloc_zone_t* zone, size_t size) { @@ -63,13 +72,24 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) { } static void zone_free(malloc_zone_t* zone, void* p) { - MI_UNUSED(zone); - mi_cfree(p); + if (mi_is_in_heap_region(p)) { + mi_free(p); + } + else if (!is_mimalloc_zone(zone)) { // can happen due to interpose + zone->free(zone,p); + } } static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) { - MI_UNUSED(zone); - return mi_realloc(p, newsize); + if (p == NULL || mi_is_in_heap_region(p)) { + return mi_realloc(p, newsize); + } + else if (!is_mimalloc_zone(zone)) { // can happen due to interpose + return zone->realloc(zone,p,newsize); + } + else { + return NULL; + } } static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) { @@ -78,8 +98,9 @@ static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) { } static void zone_destroy(malloc_zone_t* zone) { - MI_UNUSED(zone); - // todo: ignore for now? + if (!is_mimalloc_zone(zone)) { + zone->destroy(zone); + } } static unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) { @@ -114,7 +135,6 @@ static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) { return mi_is_in_heap_region(p); } - /* ------------------------------------------------------ Introspection members ------------------------------------------------------ */ @@ -174,6 +194,14 @@ static boolean_t intro_zone_locked(malloc_zone_t* zone) { return false; } +// Required whenever the zone advertises version >= 9: macOS calls this from the +// atfork_child handler (_malloc_fork_child) without a NULL check. mimalloc keeps +// no zone-level locks that need reinitializing after fork, so a no-op is safe. +// Leaving it NULL makes the forked child jump to address 0 and crash in fork(). +static void intro_reinit_lock(malloc_zone_t* zone) { + MI_UNUSED(zone); +} + /* ------------------------------------------------------ At process start, override the default allocator @@ -198,6 +226,7 @@ static malloc_introspection_t mi_introspect = { #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) .statistics = &intro_statistics, .zone_locked = &intro_zone_locked, + .reinit_lock = &intro_reinit_lock, #endif }; @@ -236,6 +265,11 @@ static malloc_zone_t mi_malloc_zone = { #endif }; +static bool is_mimalloc_zone( malloc_zone_t* zone ) { + return (zone==NULL || zone==&mi_malloc_zone); +} + + #ifdef __cplusplus } #endif diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 3bffd2912..7252d1688 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -349,8 +349,8 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec lflags |= MAP_HUGETLB; #endif #ifdef MAP_HUGE_1GB - static bool mi_huge_pages_available = true; - if (large_only && (size % MI_GiB) == 0 && mi_huge_pages_available) { + static _Atomic(size_t) mi_huge_1gib_pages_unavailable; + if (large_only && (size % MI_GiB) == 0 && (mi_atomic_load_relaxed(&mi_huge_1gib_pages_unavailable)==0)) { lflags |= MAP_HUGE_1GB; } else @@ -369,7 +369,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd); #ifdef MAP_HUGE_1GB if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) { - mi_huge_pages_available = false; // don't try huge 1GiB pages again + mi_atomic_store_relaxed(&mi_huge_1gib_pages_unavailable,1); // don't try huge 1GiB pages again if (large_only) { _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno); } @@ -389,7 +389,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec *is_large = false; p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd); #if !defined(MI_NO_THP) - if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) { + if (p != NULL && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) { #if defined(MADV_HUGEPAGE) // Many Linux systems don't allow MAP_HUGETLB but they support instead // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE @@ -562,7 +562,7 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); if (err != 0) { err = errno; - _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err); + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %ld (0x%lx))\n", numa_node, err, err); } } return (*addr != NULL ? 0 : errno); @@ -661,35 +661,39 @@ size_t _mi_prim_numa_node_count(void) { #include -#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) - -mi_msecs_t _mi_prim_clock_now(void) { - struct timespec t; - #ifdef CLOCK_MONOTONIC - clock_gettime(CLOCK_MONOTONIC, &t); +// low resolution timer +static mi_msecs_t mi_prim_clock_now_lowres(void) { + const int64_t ticks = (int64_t)clock(); + #if !defined(CLOCKS_PER_SEC) + return ticks; #else - clock_gettime(CLOCK_REALTIME, &t); + if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) { + return ticks; + } + else if (CLOCKS_PER_SEC > 0 && CLOCKS_PER_SEC < 1000) { + return ticks * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + } + else { + return ticks / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); + } #endif - return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000); } -#else - -// low resolution timer mi_msecs_t _mi_prim_clock_now(void) { - #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) - return (mi_msecs_t)clock(); - #elif (CLOCKS_PER_SEC < 1000) - return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); - #else - return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); - #endif + #if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) + #ifdef CLOCK_MONOTONIC + const clockid_t clockid = CLOCK_MONOTONIC; + #else + const clockid_t clockid = CLOCK_REALTIME; + #endif + struct timespec t; + if (clock_gettime(clockid,&t) == 0) { + return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L); + } + #endif + return mi_prim_clock_now_lowres(); } -#endif - - - //---------------------------------------------------------------- // Process info @@ -796,28 +800,28 @@ static char** mi_get_environ(void) { return environ; } #endif -bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { - if (name==NULL) return false; +int _mi_prim_getenv(const char* name, char* result, size_t result_size) { + if (name==NULL) return -1; const size_t len = _mi_strlen(name); - if (len == 0) return false; + if (len == 0) return -1; char** env = mi_get_environ(); - if (env == NULL) return false; + if (env == NULL) return -1; // compare up to 10000 entries for (int i = 0; i < 10000 && env[i] != NULL; i++) { const char* s = env[i]; if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive // found it _mi_strlcpy(result, s + len + 1, result_size); - return true; + return 1; // success } } - return false; + return 0; // not found } #else // fallback: use standard C `getenv` but this cannot be used while initializing the C runtime -bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { +int _mi_prim_getenv(const char* name, char* result, size_t result_size) { // cannot call getenv() when still initializing the C runtime. - if (_mi_preloading()) return false; + if (_mi_preloading()) return -1; // error, try again later const char* s = getenv(name); if (s == NULL) { // we check the upper case name too. @@ -829,9 +833,9 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { buf[len] = 0; s = getenv(buf); } - if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false; + if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found _mi_strlcpy(result, s, result_size); - return true; + return 1; // success } #endif // !MI_USE_ENVIRON @@ -930,7 +934,11 @@ static void mi_pthread_done(void* value) { void _mi_prim_thread_init_auto_done(void) { mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1)); - pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); + const int err = pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); + if (err!=0) { + _mi_error_message(err,"unable to create a pthread thread local key (error %d (0x%x))", err, err); + _mi_heap_default_key = (pthread_key_t)(-1); + }; } void _mi_prim_thread_done_auto_done(void) { diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index 8fff0ea84..2d19846d8 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -186,33 +186,39 @@ size_t _mi_prim_numa_node_count(void) { #include -#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) - -mi_msecs_t _mi_prim_clock_now(void) { - struct timespec t; - #ifdef CLOCK_MONOTONIC - clock_gettime(CLOCK_MONOTONIC, &t); +// low resolution timer +static mi_msecs_t mi_prim_clock_now_lowres(void) { + const int64_t ticks = (int64_t)clock(); + #if !defined(CLOCKS_PER_SEC) + return ticks; #else - clock_gettime(CLOCK_REALTIME, &t); + if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) { + return ticks; + } + else if (CLOCKS_PER_SEC > 0 && CLOCKS_PER_SEC < 1000) { + return ticks * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + } + else { + return ticks / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); + } #endif - return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000); } -#else - -// low resolution timer mi_msecs_t _mi_prim_clock_now(void) { - #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) - return (mi_msecs_t)clock(); - #elif (CLOCKS_PER_SEC < 1000) - return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); - #else - return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); - #endif + #if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) + #ifdef CLOCK_MONOTONIC + const clockid_t clockid = CLOCK_MONOTONIC; + #else + const clockid_t clockid = CLOCK_REALTIME; + #endif + struct timespec t; + if (clock_gettime(clockid,&t) == 0) { + return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L); + } + #endif + return mi_prim_clock_now_lowres(); } -#endif - //---------------------------------------------------------------- // Process info @@ -238,9 +244,9 @@ void _mi_prim_out_stderr( const char* msg ) { // Environment //---------------------------------------------------------------- -bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { +int _mi_prim_getenv(const char* name, char* result, size_t result_size) { // cannot call getenv() when still initializing the C runtime. - if (_mi_preloading()) return false; + if (_mi_preloading()) return -1; // error, try again later const char* s = getenv(name); if (s == NULL) { // we check the upper case name too. @@ -252,9 +258,9 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { buf[len] = 0; s = getenv(buf); } - if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false; + if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found _mi_strlcpy(result, s, result_size); - return true; + return 1; // found } diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 0d9bb2ac9..1ecde738e 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -611,10 +611,10 @@ void _mi_prim_out_stderr( const char* msg ) // reliably even when this is invoked before the C runtime is initialized. // i.e. when `_mi_preloading() == true`. // Note: on windows, environment names are not case sensitive. -bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { +int _mi_prim_getenv(const char* name, char* result, size_t result_size) { result[0] = 0; const size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size); - return (len > 0 && len < result_size); + return (len < result_size ? (len > 0 ? 1 /* success */ : 0 /* not found */) : -1 /* error */); } @@ -722,9 +722,9 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { ------------------------------------------------------------------------- */ #if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS) #if !defined(__INTEL_LLVM_COMPILER) && !defined(__INTEL_COMPILER) - #define MI_WIN_INIT_USE_CRT_TLS 1 + #define MI_WIN_INIT_USE_CRT_TLS 1 #else - #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* default for Intel ICX, see issue #1268 */ + #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* default for Intel ICX, see issue #1268 */ #endif #endif diff --git a/src/segment.c b/src/segment.c index 7ae84616f..75bf81f69 100644 --- a/src/segment.c +++ b/src/segment.c @@ -92,9 +92,7 @@ size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) count += MI_COMMIT_MASK_FIELD_BITS; } else { - for (; mask != 0; mask >>= 1) { // todo: use popcount - if ((mask&1)!=0) count++; - } + count += mi_popcount(mask); } } // we use total since for huge segments each commit bit may represent a larger size