[GR-74145] C API raw allocation fixes and debugging helpers.

ci/python-gate.libsonnet

@@ -142,6 +142,7 @@
             BISECT_EMAIL_TO_PATTERN: ".*@oracle.com",
             TRUFFLE_STRICT_OPTION_DEPRECATION: "true",
             npm_config_registry: $.overlay_imports.npm_config_registry,
+            CFLAGS: "-ggdb",
         },
         linux: {
             common: ENV_POSIX + {},
@@ -241,6 +242,8 @@
         "graal_dumps/*/*",
         "bench-results.json",
         "raw-results.json",
+        "mxbuild/*/libpythonvm/libpythonvm.so.debug",
+        "mxbuild/*/GRAALPY_STANDALONE_COMMON/lib/graalpy*/libpython-native.so",
     ],
 
     //------------------------------------------------------------------------------------------------------------------

docs/contributor/CONTRIBUTING.md

@@ -287,6 +287,9 @@ mx benchmark meso:nbody3 \
     -Dgraal.MethodFilter=*measure*
 ```
 
+For debugging native problems in benchmark runs, there's `BENCHMARK_DEBUG_ARGS` in `mx_graalpython_benchmark.py` to log more stuff for debugging, at the cost of performance.
+This is intended for focused reproducer runs on a branch.
+
 ### A note on terminology
 
 Note that there may be a little confusion about the configuration names of benchmarks.

graalpython/com.oracle.graal.python.cext/CMakeLists.txt

@@ -162,7 +162,7 @@ set(SRC_FILES ${CAPI_SRC}/codecs.c ${CAPI_SRC}/setobject.c ${CAPI_SRC}/compile.c
               ${CAPI_SRC}/fileobject.c ${CAPI_SRC}/pystrcmp.c ${CAPI_SRC}/getversion.c
               ${CAPI_SRC}/genobject.c ${CAPI_SRC}/methodobject.c ${CAPI_SRC}/boolobject.c ${CAPI_SRC}/pylifecycle.c
               ${CAPI_SRC}/errors.c ${CAPI_SRC}/signals.c ${CAPI_SRC}/datetime.c ${CAPI_SRC}/call.c
-              ${CAPI_SRC}/getargs.c ${CAPI_SRC}/tracemalloc.c ${CAPI_SRC}/initconfig.c
+              ${CAPI_SRC}/getargs.c ${CAPI_SRC}/tracemalloc.c ${CAPI_SRC}/initconfig.c ${CAPI_SRC}/graalpy_stacktrace.c
 )
 
 file(GLOB_RECURSE ACTUAL_SRC_FILES
@@ -454,6 +454,7 @@ if(WIN32)
     if (NOT MSVC)
         target_compile_options(${TARGET_LIBPYTHON} PRIVATE "-fmsc-version=1920")
     endif()
+    target_link_libraries(${TARGET_LIBPYTHON} dbghelp)
 else()
     # Link to math library; required for functions like 'hypot' or similar
     target_link_libraries(${TARGET_LIBPYTHON} m)

graalpython/com.oracle.graal.python.cext/src/capi.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -96,6 +96,8 @@
 #define PY_TRUFFLE_LOG_FINEST 0x20
 #define PY_TRUFFLE_DEBUG_CAPI 0x40
 #define PY_TRUFFLE_PYTHON_GC 0x80
+#define PY_TRUFFLE_POISON_NATIVE_MEMORY_ON_FREE 0x100
+#define PY_TRUFFLE_SAMPLE_NATIVE_MEMORY_ALLOC_SITES 0x200
 
 typedef struct mmap_object mmap_object;
 typedef struct _gc_runtime_state GCState; // originally in 'gcmodule.c'
@@ -169,16 +171,7 @@ extern Py_LOCAL_SYMBOL int8_t *_graalpy_finalizing;
 #if (__linux__ && __GNU_LIBRARY__)
 #include <stdlib.h>
 #include <string.h>
-#include <execinfo.h>
 #include <unistd.h>
-static void print_c_stacktrace() {
-    fprintf(stderr, "Native stacktrace:\n");
-    intptr_t stack[16];
-    size_t stack_size = backtrace((void *)stack, sizeof(stack) / sizeof(stack[0]));
-    backtrace_symbols_fd((void *)stack, stack_size, STDERR_FILENO);
-    fflush(stderr);
-}
-
 static void attach_gdb() {
     pid_t my_pid = getpid();
     char* pathname = "/bin/sh";
@@ -197,15 +190,20 @@ static void attach_gdb() {
     }
 }
 #else
-static void print_c_stacktrace() {
-    // not supported
-}
-
 static void attach_gdb() {
     // not supported
 }
 #endif
 
+size_t GraalPyPrivate_CaptureStacktrace(void **frames, size_t max_depth, size_t skip);
+void GraalPyPrivate_PrintCapturedStacktrace(FILE *file, const char *header, void *const *frames, size_t depth);
+void GraalPyPrivate_PrintCurrentStacktrace(FILE *file, const char *header, size_t max_depth, size_t skip);
+void GraalPyPrivate_LogCapturedStacktrace(int level, const char *prefix, void *const *frames, size_t depth);
+
+static void print_c_stacktrace() {
+    GraalPyPrivate_PrintCurrentStacktrace(stderr, "Native stacktrace:\n", 16, 0);
+}
+
 /* Flags definitions representing global (debug) options. */
 static MUST_INLINE int GraalPyPrivate_Trace_Memory() {
     return Py_Truffle_Options & PY_TRUFFLE_TRACE_MEM;
@@ -232,6 +230,12 @@ static MUST_INLINE int GraalPyPrivate_Debug_CAPI() {
 static MUST_INLINE int GraalPyPrivate_PythonGC() {
     return Py_Truffle_Options & PY_TRUFFLE_PYTHON_GC;
 }
+static MUST_INLINE int GraalPyPrivate_PoisonNativeMemoryOnFree() {
+    return Py_Truffle_Options & PY_TRUFFLE_POISON_NATIVE_MEMORY_ON_FREE;
+}
+static MUST_INLINE int GraalPyPrivate_SampleNativeMemoryAllocSites() {
+    return Py_Truffle_Options & PY_TRUFFLE_SAMPLE_NATIVE_MEMORY_ALLOC_SITES;
+}
 
 static void
 GraalPyPrivate_Log(int level, const char *format, ...)

graalpython/com.oracle.graal.python.cext/src/graalpy_stacktrace.c

@@ -0,0 +1,279 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "capi.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if defined(MS_WINDOWS)
+#include <windows.h>
+#include <dbghelp.h>
+#elif (defined(__linux__) && defined(__GNU_LIBRARY__)) || defined(__APPLE__)
+#include <execinfo.h>
+#endif
+
+#define GRAALPY_NATIVE_STACK_MAX_NAME 1024
+#define GRAALPY_NATIVE_STACK_LINE_BUFFER 2048
+
+typedef void (*GraalPyStacktraceWriter)(void *ctx, const char *line);
+
+static void
+render_unavailable_stacktrace(GraalPyStacktraceWriter writer, void *ctx)
+{
+    writer(ctx, "<native stacktrace unavailable>");
+}
+
+#if defined(MS_WINDOWS)
+
+static int
+ensure_windows_symbols_initialized(void)
+{
+    static int initialized = 0;
+    if (!initialized) {
+        HANDLE process = GetCurrentProcess();
+        SymSetOptions(SymGetOptions() | SYMOPT_LOAD_LINES | SYMOPT_UNDNAME);
+        if (!SymInitialize(process, NULL, TRUE)) {
+            return 0;
+        }
+        initialized = 1;
+    }
+    return 1;
+}
+
+static const char *
+windows_basename(const char *path)
+{
+    const char *slash = strrchr(path, '\\');
+    const char *alt = strrchr(path, '/');
+    const char *base = slash != NULL ? slash + 1 : path;
+    if (alt != NULL && (slash == NULL || alt > slash)) {
+        base = alt + 1;
+    }
+    return base;
+}
+
+static void
+render_windows_stacktrace(GraalPyStacktraceWriter writer, void *ctx, void *const *frames, size_t depth)
+{
+    HANDLE process = GetCurrentProcess();
+    char line[GRAALPY_NATIVE_STACK_LINE_BUFFER];
+    char symbol_buffer[sizeof(SYMBOL_INFO) + GRAALPY_NATIVE_STACK_MAX_NAME];
+    PSYMBOL_INFO symbol = (PSYMBOL_INFO) symbol_buffer;
+
+    memset(symbol_buffer, 0, sizeof(symbol_buffer));
+    symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+    symbol->MaxNameLen = GRAALPY_NATIVE_STACK_MAX_NAME - 1;
+
+    if (!ensure_windows_symbols_initialized()) {
+        for (size_t i = 0; i < depth; i++) {
+            snprintf(line, sizeof(line), "frame[%lu]: %p",
+                            (unsigned long) i, (void *) frames[i]);
+            writer(ctx, line);
+        }
+        return;
+    }
+
+    for (size_t i = 0; i < depth; i++) {
+        DWORD64 address = (DWORD64) (uintptr_t) frames[i];
+        DWORD64 displacement = 0;
+        IMAGEHLP_LINE64 source_line;
+        DWORD source_displacement = 0;
+        char module_path[MAX_PATH] = {'\0'};
+        const char *module_name = NULL;
+        HMODULE module = NULL;
+
+        memset(&source_line, 0, sizeof(source_line));
+        source_line.SizeOfStruct = sizeof(source_line);
+
+        if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+                        (LPCSTR) frames[i], &module) && GetModuleFileNameA(module, module_path, MAX_PATH) > 0) {
+            module_name = windows_basename(module_path);
+        }
+
+        if (SymFromAddr(process, address, &displacement, symbol)) {
+            if (SymGetLineFromAddr64(process, address, &source_displacement, &source_line)) {
+                if (module_name != NULL) {
+                    snprintf(line, sizeof(line), "frame[%lu]: %s!%s+0x%llx (%s:%lu) [%p]",
+                                    (unsigned long) i, module_name, symbol->Name, (unsigned long long) displacement,
+                                    source_line.FileName, (unsigned long) source_line.LineNumber, (void *) frames[i]);
+                } else {
+                    snprintf(line, sizeof(line), "frame[%lu]: %s+0x%llx (%s:%lu) [%p]",
+                                    (unsigned long) i, symbol->Name, (unsigned long long) displacement,
+                                    source_line.FileName, (unsigned long) source_line.LineNumber, (void *) frames[i]);
+                }
+            } else if (module_name != NULL) {
+                snprintf(line, sizeof(line), "frame[%lu]: %s!%s+0x%llx [%p]",
+                                (unsigned long) i, module_name, symbol->Name, (unsigned long long) displacement, (void *) frames[i]);
+            } else {
+                snprintf(line, sizeof(line), "frame[%lu]: %s+0x%llx [%p]",
+                                (unsigned long) i, symbol->Name, (unsigned long long) displacement, (void *) frames[i]);
+            }
+        } else if (module_name != NULL) {
+            snprintf(line, sizeof(line), "frame[%lu]: %s [%p]",
+                            (unsigned long) i, module_name, (void *) frames[i]);
+        } else {
+            snprintf(line, sizeof(line), "frame[%lu]: %p",
+                            (unsigned long) i, (void *) frames[i]);
+        }
+        writer(ctx, line);
+    }
+}
+
+#elif (defined(__linux__) && defined(__GNU_LIBRARY__)) || defined(__APPLE__)
+
+static void
+render_execinfo_stacktrace(GraalPyStacktraceWriter writer, void *ctx, void *const *frames, size_t depth)
+{
+    char **symbols = backtrace_symbols((void *const *) frames, (int) depth);
+    char line[GRAALPY_NATIVE_STACK_LINE_BUFFER];
+    if (symbols == NULL) {
+        for (size_t i = 0; i < depth; i++) {
+            snprintf(line, sizeof(line), "frame[%lu]: %p",
+                            (unsigned long) i, (void *) frames[i]);
+            writer(ctx, line);
+        }
+        return;
+    }
+
+    for (size_t i = 0; i < depth; i++) {
+        snprintf(line, sizeof(line), "frame[%lu]: %s", (unsigned long) i, symbols[i]);
+        writer(ctx, line);
+    }
+    free(symbols);
+}
+
+#endif
+
+size_t
+GraalPyPrivate_CaptureStacktrace(void **frames, size_t max_depth, size_t skip)
+{
+    if (frames == NULL || max_depth == 0) {
+        return 0;
+    }
+#if defined(MS_WINDOWS)
+    return (size_t) CaptureStackBackTrace((ULONG) (skip + 1), (ULONG) max_depth, frames, NULL);
+#elif (defined(__linux__) && defined(__GNU_LIBRARY__)) || defined(__APPLE__)
+    int raw_depth = backtrace(frames, (int) max_depth);
+    size_t depth = raw_depth > 0 ? (size_t) raw_depth : 0;
+    size_t start = depth > (skip + 1) ? (skip + 1) : depth;
+    size_t usable_depth = depth - start;
+    if (usable_depth > 0) {
+        memmove(frames, frames + start, usable_depth * sizeof(void *));
+    }
+    return usable_depth;
+#else
+    return 0;
+#endif
+}
+
+static void
+render_stacktrace(GraalPyStacktraceWriter writer, void *ctx, void *const *frames, size_t depth)
+{
+    if (depth == 0) {
+        render_unavailable_stacktrace(writer, ctx);
+        return;
+    }
+#if defined(MS_WINDOWS)
+    render_windows_stacktrace(writer, ctx, frames, depth);
+#elif (defined(__linux__) && defined(__GNU_LIBRARY__)) || defined(__APPLE__)
+    render_execinfo_stacktrace(writer, ctx, frames, depth);
+#else
+    (void) frames;
+    render_unavailable_stacktrace(writer, ctx);
+#endif
+}
+
+static void
+file_writer(void *ctx, const char *line)
+{
+    fprintf((FILE *) ctx, "%s\n", line);
+}
+
+void
+GraalPyPrivate_PrintCapturedStacktrace(FILE *file, const char *header, void *const *frames, size_t depth)
+{
+    if (header != NULL) {
+        fputs(header, file);
+    }
+    render_stacktrace(file_writer, file, frames, depth);
+    fflush(file);
+}
+
+void
+GraalPyPrivate_PrintCurrentStacktrace(FILE *file, const char *header, size_t max_depth, size_t skip)
+{
+    void *frames[64];
+    size_t depth = max_depth;
+    if (depth > (sizeof(frames) / sizeof(frames[0]))) {
+        depth = sizeof(frames) / sizeof(frames[0]);
+    }
+    depth = GraalPyPrivate_CaptureStacktrace(frames, depth, skip + 1);
+    GraalPyPrivate_PrintCapturedStacktrace(file, header, frames, depth);
+}
+
+typedef struct {
+    int level;
+    const char *prefix;
+} LogWriterCtx;
+
+static void
+log_writer(void *ctx, const char *line)
+{
+    LogWriterCtx *log_ctx = (LogWriterCtx *) ctx;
+    if (log_ctx->prefix != NULL) {
+        GraalPyPrivate_Log(log_ctx->level, "%s%s\n", log_ctx->prefix, line);
+    } else {
+        GraalPyPrivate_Log(log_ctx->level, "%s\n", line);
+    }
+}
+
+void
+GraalPyPrivate_LogCapturedStacktrace(int level, const char *prefix, void *const *frames, size_t depth)
+{
+    if ((Py_Truffle_Options & level) == 0) {
+        return;
+    }
+    LogWriterCtx log_ctx = {level, prefix};
+    render_stacktrace(log_writer, &log_ctx, frames, depth);
+}