diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 854cd4eae429..33a9d980f437 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -2904,6 +2904,14 @@ impl FuncEnvironment<'_> { gc::translate_exn_throw_ref(self, builder, exnref) } + pub fn translate_drop_exnref( + &mut self, + builder: &mut FunctionBuilder<'_>, + exnref: ir::Value, + ) -> WasmResult<()> { + gc::translate_drop_exnref(self, builder, exnref) + } + pub fn translate_array_new( &mut self, builder: &mut FunctionBuilder, diff --git a/crates/cranelift/src/func_environ/gc/disabled.rs b/crates/cranelift/src/func_environ/gc/disabled.rs index 13c41f27a125..584652aa8c24 100644 --- a/crates/cranelift/src/func_environ/gc/disabled.rs +++ b/crates/cranelift/src/func_environ/gc/disabled.rs @@ -84,6 +84,14 @@ pub fn translate_exn_throw_ref( disabled() } +pub fn translate_drop_exnref( + _func_env: &mut FuncEnvironment<'_>, + _builder: &mut FunctionBuilder<'_>, + _exnref: ir::Value, +) -> WasmResult<()> { + disabled() +} + pub fn translate_array_new( _func_env: &mut FuncEnvironment<'_>, _builder: &mut FunctionBuilder, diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs index 06b44b513281..fff2c809f400 100644 --- a/crates/cranelift/src/func_environ/gc/enabled.rs +++ b/crates/cranelift/src/func_environ/gc/enabled.rs @@ -462,6 +462,32 @@ pub fn translate_exn_unbox( Ok(result) } +/// Drop a GC reference to an exception object, decrementing its ref count. +/// +/// This should be called when a `catch` (non-ref) handler has finished +/// extracting fields from the caught exnref and no longer needs it. +pub fn translate_drop_exnref( + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder<'_>, + exnref: ir::Value, +) -> WasmResult<()> { + log::trace!("translate_drop_exnref({exnref:?})"); + match func_env.tunables.collector { + #[cfg(feature = "gc-drc")] + Some(Collector::DeferredReferenceCounting) => { + let drop_gc_ref_libcall = func_env.builtin_functions.drop_gc_ref(builder.func); + let vmctx = func_env.vmctx_val(&mut builder.cursor()); + builder.ins().call(drop_gc_ref_libcall, &[vmctx, exnref]); + } + // The null collector doesn't track ref counts, so nothing to do. + _ => { + // Avoid unused-parameter warning. + let _ = builder; + } + } + Ok(()) +} + pub fn translate_exn_throw( func_env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index 494bb275c71c..0345cd545057 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -4474,6 +4474,10 @@ fn create_catch_block( } if is_ref { params.push(exn_ref); + } else { + // For non-ref catches, the exnref is consumed here and not + // passed to the branch target. + environ.translate_drop_exnref(builder, exn_ref)?; } // Generate the branch itself. diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 4733dc639390..ba79b9995d0b 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -1016,6 +1016,13 @@ impl Store { StoreContextMut(&mut self.inner).gc(why) } + /// Returns the current size of the GC heap in bytes, or 0 if the GC heap + /// has not been allocated yet. + #[cfg(feature = "gc")] + pub fn gc_heap_size(&self) -> usize { + self.inner.vm_store_context.gc_heap.current_length() + } + /// Returns the amount fuel in this [`Store`]. When fuel is enabled, it must /// be configured via [`Store::set_fuel`]. /// @@ -1327,6 +1334,14 @@ impl<'a, T> StoreContext<'a, T> { pub fn get_fuel(&self) -> Result { self.0.get_fuel() } + + /// Returns the current size of the GC heap in bytes. + /// + /// Same as [`Store::gc_heap_size`]. + #[cfg(feature = "gc")] + pub fn gc_heap_size(&self) -> usize { + self.0.vm_store_context.gc_heap.current_length() + } } impl<'a, T> StoreContextMut<'a, T> { diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index c547f95a7518..cd927431f770 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -1744,7 +1744,10 @@ fn throw_ref( exnref: u32, ) -> Result<(), TrapReason> { let exnref = VMGcRef::from_raw_u32(exnref).ok_or_else(|| Trap::NullReference)?; - let exnref = store.unwrap_gc_store_mut().clone_gc_ref(&exnref); + // Transfer ownership of the GC ref from Wasm to the pending + // exception slot without cloning. The `throw` / `throw_ref` + // instruction consumes the exnref operand, so Wasm no longer + // holds a reference. let exnref = exnref .into_exnref(&*store.unwrap_gc_store().gc_heap) .expect("gc ref should be an exception object"); diff --git a/tests/all/exceptions.rs b/tests/all/exceptions.rs index 987011c4f481..c1b41e1ac938 100644 --- a/tests/all/exceptions.rs +++ b/tests/all/exceptions.rs @@ -242,3 +242,70 @@ fn gc_with_exnref_global(config: &mut Config) -> Result<()> { Ok(()) } + +#[wasmtime_test(wasm_features(exceptions))] +#[cfg_attr(miri, ignore)] +fn throw_catch_many_times(config: &mut Config) -> Result<()> { + // The GC heap is allocated in 64 KiB pages, so the minimum heap + // size is one page. With at most one live exception at a time, the + // heap should never need to grow beyond that initial page. + const HEAP_SIZE_LIMIT: usize = 64 * 1024; + + let engine = Engine::new(config)?; + let mut store = Store::new(&engine, ()); + + // Host function that checks the GC heap size mid-loop. + let check_heap = Func::wrap(&mut store, |caller: Caller<'_, ()>| { + let heap_size = caller.as_context().gc_heap_size(); + assert!( + heap_size <= HEAP_SIZE_LIMIT, + "GC heap grew to {heap_size} bytes mid-loop (limit {HEAP_SIZE_LIMIT})" + ); + }); + + let module = Module::new( + &engine, + r#" + (module + (import "" "check_heap" (func $check_heap)) + (tag $e (param i32)) + + (func (export "run") (result i32) + (local $i i32) + (local $sum i32) + (local.set $i (i32.const 0)) + (local.set $sum (i32.const 0)) + (block $done + (loop $loop + ;; throw and catch an exception + (local.set $sum + (i32.add (local.get $sum) + (block $handler (result i32) + (try_table (catch $e $handler) + (throw $e (local.get $i))) + (unreachable)))) + ;; check heap size from the host + (call $check_heap) + ;; increment and check loop counter + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br_if $done (i32.ge_u (local.get $i) (i32.const 100_000))) + (br $loop))) + (local.get $sum))) + "#, + )?; + + let instance = Instance::new(&mut store, &module, &[check_heap.into()])?; + let run = instance.get_typed_func::<(), i32>(&mut store, "run")?; + let result = run.call(&mut store, ())?; + + let expected: i32 = (0i64..100_000i64).sum::() as i32; + assert_eq!(result, expected); + + let heap_size = store.gc_heap_size(); + assert!( + heap_size <= HEAP_SIZE_LIMIT, + "GC heap grew to {heap_size} bytes after loop (limit {HEAP_SIZE_LIMIT})" + ); + + Ok(()) +} diff --git a/tests/all/gc.rs b/tests/all/gc.rs index 55b6e9e8b5b6..7a182e850bfb 100644 --- a/tests/all/gc.rs +++ b/tests/all/gc.rs @@ -1800,3 +1800,66 @@ fn array_init_elem_oom() -> Result<()> { Ok(()) } + +#[test] +#[cfg_attr(miri, ignore)] +fn alloc_free_many_times() -> Result<()> { + // The GC heap is allocated in 64 KiB pages, so the minimum heap + // size is one page. With at most one live exception at a time, the + // heap should never need to grow beyond that initial page. + const HEAP_SIZE_LIMIT: usize = 64 * 1024; + + let mut config = Config::new(); + config.wasm_gc(true); + let engine = Engine::new(&config)?; + let mut store = Store::new(&engine, ()); + + // Host function that checks the GC heap size mid-loop. + let check_heap = Func::wrap(&mut store, |caller: Caller<'_, ()>| { + let heap_size = caller.as_context().gc_heap_size(); + assert!( + heap_size <= HEAP_SIZE_LIMIT, + "GC heap grew to {heap_size} bytes mid-loop (limit {HEAP_SIZE_LIMIT})" + ); + }); + + let module = Module::new( + &engine, + r#" + (module + (import "" "check_heap" (func $check_heap)) + (type $t (struct (field i32))) + + (func (export "run") (result i32) + (local $i i32) + (local $sum i32) + (local.set $i (i32.const 0)) + (local.set $sum (i32.const 0)) + (block $done + (loop $loop + (drop (struct.new $t (i32.const 42))) + ;; check heap size from the host + (call $check_heap) + ;; increment and check loop counter + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br_if $done (i32.ge_u (local.get $i) (i32.const 100_000))) + (br $loop))) + (local.get $sum))) + "#, + )?; + + let instance = Instance::new(&mut store, &module, &[check_heap.into()])?; + let run = instance.get_typed_func::<(), i32>(&mut store, "run")?; + let result = run.call(&mut store, ())?; + + let expected: i32 = (0i64..100_000i64).sum::() as i32; + assert_eq!(result, expected); + + let heap_size = store.gc_heap_size(); + assert!( + heap_size <= HEAP_SIZE_LIMIT, + "GC heap grew to {heap_size} bytes after loop (limit {HEAP_SIZE_LIMIT})" + ); + + Ok(()) +} diff --git a/tests/disas/debug-exceptions.wat b/tests/disas/debug-exceptions.wat index 440b63fdae59..cd9eb2cb4b79 100644 --- a/tests/disas/debug-exceptions.wat +++ b/tests/disas/debug-exceptions.wat @@ -30,42 +30,42 @@ ;; ldr x0, [x0, #0x18] ;; mov x1, sp ;; cmp x1, x0 -;; b.lo #0x194 +;; b.lo #0x19c ;; 44: stur x2, [sp] ;; mov x0, x2 ;; stur x2, [sp, #0x10] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 53, slot at FP-0xc0, locals , stack -;; ╰─╼ breakpoint patch: wasm PC 53, patch bytes [38, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 53, patch bytes [54, 1, 0, 148] ;; ldur x0, [sp, #0x10] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 55, slot at FP-0xc0, locals , stack -;; ╰─╼ breakpoint patch: wasm PC 55, patch bytes [36, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 55, patch bytes [52, 1, 0, 148] ;; ldur x0, [sp, #0x10] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 61, slot at FP-0xc0, locals , stack -;; ╰─╼ breakpoint patch: wasm PC 61, patch bytes [34, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 61, patch bytes [50, 1, 0, 148] ;; mov w19, #0x2a ;; stur w19, [sp, #8] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 63, slot at FP-0xc0, locals , stack I32 @ slot+0x8 -;; ╰─╼ breakpoint patch: wasm PC 63, patch bytes [31, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 63, patch bytes [47, 1, 0, 148] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 64, slot at FP-0xc0, locals , stack -;; ╰─╼ breakpoint patch: wasm PC 64, patch bytes [30, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 64, patch bytes [46, 1, 0, 148] ;; stur w19, [sp, #8] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 66, slot at FP-0xc0, locals , stack I32 @ slot+0x8 -;; ╰─╼ breakpoint patch: wasm PC 66, patch bytes [28, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 66, patch bytes [44, 1, 0, 148] ;; ldur x2, [sp, #0x10] -;; bl #0x448 +;; bl #0x488 ;; 84: mov x20, x2 ;; mov w3, #0x4000000 ;; mov w4, #2 ;; mov w5, #0x28 ;; mov w6, #8 ;; ldur x2, [sp, #0x10] -;; bl #0x368 +;; bl #0x3a8 ;; a0: ldur x0, [sp, #0x10] ;; ldr x5, [x0, #8] ;; ldr x6, [x5, #0x20] @@ -81,50 +81,52 @@ ;; str w3, [x4, w2, uxtw] ;; mov x3, x2 ;; ldur x2, [sp, #0x10] -;; bl #0x480 +;; bl #0x4c0 ;; ├─╼ exception frame offset: SP = FP - 0xc0 ;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0xf8 ;; e0: mov w3, #9 ;; ldur x2, [sp, #0x10] -;; bl #0x3dc +;; bl #0x41c ;; ec: ldur x2, [sp, #0x10] -;; bl #0x414 +;; bl #0x454 ;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 66, slot at FP-0xc0, locals , stack I32 @ slot+0x8 ;; f4: .byte 0x1f, 0xc1, 0x00, 0x00 -;; mov x2, x0 -;; mov w3, w2 +;; mov x3, x0 +;; mov w2, w3 ;; mov x4, #0x28 -;; adds x3, x3, x4 +;; adds x2, x2, x4 ;; cset x4, hs ;; uxtb w4, w4 -;; cbnz x4, #0x1ac +;; cbnz x4, #0x1b4 ;; 114: ldur x5, [sp, #0x20] ;; ldr x1, [x5, #0x28] -;; cmp x3, x1 +;; cmp x2, x1 ;; cset x1, hi ;; uxtb w1, w1 -;; cbnz x1, #0x1b0 +;; cbnz x1, #0x1b8 ;; 12c: ldur x6, [sp, #0x18] ;; add x0, x6, #0x20 -;; ldr w0, [x0, w2, uxtw] -;; stur w0, [sp, #8] +;; ldr w19, [x0, w3, uxtw] +;; ldur x2, [sp, #0x10] +;; bl #0x370 +;; 140: stur w19, [sp, #8] ;; ldur x0, [sp, #0x10] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 72, slot at FP-0xc0, locals , stack I32 @ slot+0x8 -;; ╰─╼ breakpoint patch: wasm PC 72, patch bytes [234, 0, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 72, patch bytes [248, 0, 0, 148] ;; ldur x1, [sp, #0x10] ;; ldr x0, [x1, #0x30] ;; ldr x2, [x1, #0x40] ;; ldur x3, [sp, #0x10] ;; blr x0 ;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 74, slot at FP-0xc0, locals , stack I32 @ slot+0x8 -;; 158: ldur x0, [sp, #0x10] +;; 160: ldur x0, [sp, #0x10] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 74, slot at FP-0xc0, locals , stack I32 @ slot+0x8 -;; ╰─╼ breakpoint patch: wasm PC 74, patch bytes [227, 0, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 74, patch bytes [241, 0, 0, 148] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 75, slot at FP-0xc0, locals , stack -;; ╰─╼ breakpoint patch: wasm PC 75, patch bytes [226, 0, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 75, patch bytes [240, 0, 0, 148] ;; add sp, sp, #0x30 ;; ldp d8, d9, [sp], #0x10 ;; ldp d10, d11, [sp], #0x10 @@ -137,12 +139,12 @@ ;; ldp x27, x28, [sp], #0x10 ;; ldp x29, x30, [sp], #0x10 ;; ret -;; 194: stur x2, [sp, #0x10] -;; 198: mov w3, #0 -;; 19c: bl #0x3dc -;; 1a0: ldur x2, [sp, #0x10] -;; 1a4: bl #0x414 +;; 19c: stur x2, [sp, #0x10] +;; 1a0: mov w3, #0 +;; 1a4: bl #0x41c +;; 1a8: ldur x2, [sp, #0x10] +;; 1ac: bl #0x454 ;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 52, slot at FP-0xc0, locals , stack -;; 1a8: .byte 0x1f, 0xc1, 0x00, 0x00 -;; 1ac: .byte 0x1f, 0xc1, 0x00, 0x00 ;; 1b0: .byte 0x1f, 0xc1, 0x00, 0x00 +;; 1b4: .byte 0x1f, 0xc1, 0x00, 0x00 +;; 1b8: .byte 0x1f, 0xc1, 0x00, 0x00 diff --git a/tests/disas/exceptions.wat b/tests/disas/exceptions.wat index bad9fd79703e..745c6945d53f 100644 --- a/tests/disas/exceptions.wat +++ b/tests/disas/exceptions.wat @@ -33,14 +33,14 @@ ;; movq %rdi, %rbx ;; movq %rcx, %r12 ;; movq %rdx, %r13 -;; callq 0x385 +;; callq 0x3c8 ;; movq %rax, %r14 ;; movl $0x4000000, %esi ;; movl $3, %edx ;; movl $0x30, %ecx ;; movl $8, %r8d ;; movq %rbx, %rdi -;; callq 0x322 +;; callq 0x365 ;; movq 8(%rbx), %rcx ;; movq 0x20(%rcx), %rcx ;; movl %eax, %edx @@ -54,7 +54,7 @@ ;; movq %rax, %rsi ;; movq %rbx, %rdi ;; movq %rbx, (%rsp) -;; callq 0x3b2 +;; callq 0x3f5 ;; ud2 ;; ud2 ;; @@ -65,7 +65,7 @@ ;; movq 0x18(%r10), %r10 ;; addq $0x50, %r10 ;; cmpq %rsp, %r10 -;; ja 0x135 +;; ja 0x149 ;; b9: subq $0x40, %rsp ;; movq %rbx, 0x10(%rsp) ;; movq %r12, 0x18(%rsp) @@ -77,18 +77,24 @@ ;; movq (%rsp), %rdi ;; callq 0 ;; ├─╼ exception frame offset: SP = FP - 0x40 -;; ╰─╼ exception handler: tag=0, context at [SP+0x0], handler=0xf6 +;; ╰─╼ exception handler: tag=0, context at [SP+0x0], handler=0xfc ;; movl $0x2a, %eax ;; movl $0x64, %ecx -;; jmp 0x113 -;; f6: movq %rax, %rdx -;; movq (%rsp), %rsi -;; movq 8(%rsi), %rax +;; movq %rax, %rbx +;; movq %rcx, %r12 +;; jmp 0x121 +;; fc: movq %rax, %rsi +;; movq (%rsp), %rdi +;; movq 8(%rdi), %rax ;; movq 0x20(%rax), %rcx -;; movq %rdx, %rax -;; movl %eax, %edx +;; movl %esi, %edx ;; movl 0x20(%rcx, %rdx), %eax +;; movq %rax, %rbx ;; movq 0x28(%rcx, %rdx), %rcx +;; movq %rcx, %r12 +;; callq 0x336 +;; movq %rbx, %rax +;; movq %r12, %rcx ;; movq 0x10(%rsp), %rbx ;; movq 0x18(%rsp), %r12 ;; movq 0x20(%rsp), %r13 @@ -98,4 +104,4 @@ ;; movq %rbp, %rsp ;; popq %rbp ;; retq -;; 135: ud2 +;; 149: ud2