diff --git a/gpu/third_party/suyu/shader_recompiler/CMakeLists.txt b/gpu/third_party/suyu/shader_recompiler/CMakeLists.txt index d02ee26..bd7cf93 100644 --- a/gpu/third_party/suyu/shader_recompiler/CMakeLists.txt +++ b/gpu/third_party/suyu/shader_recompiler/CMakeLists.txt @@ -227,6 +227,7 @@ add_library(shader_recompiler STATIC ir_opt/lower_fp16_to_fp32.cpp ir_opt/lower_fp64_to_fp32.cpp ir_opt/lower_int64_to_int32.cpp + ir_opt/native_cbuf_pointer_policy.h ir_opt/passes.h ir_opt/position_pass.cpp ir_opt/rescaling_pass.cpp diff --git a/gpu/vk/VULKAN.md b/gpu/vk/VULKAN.md index a730c32..d2a4897 100644 --- a/gpu/vk/VULKAN.md +++ b/gpu/vk/VULKAN.md @@ -17,7 +17,10 @@ Why Vulkan, after WebGPU: - **Guest memory in place.** `VK_EXT_external_memory_host` imports the game's pools as buffers. GPU writes land in guest memory and CPU writes are seen by the GPU, with no mirrored buffer cache, write-back syncs or upload streams (the WebGPU renderer's biggest costs). Hot read-only ranges can later be mirrored to device memory, as - totk-pc-recomp does. + totk-pc-recomp does. On Linux, a memfd plus `/dev/udmabuf` avoids re-pinning every host page at each AMD submit. + `RX_VK_SPARSE_GUEST=1` is an opt-in low-memory form of that path: when the device guarantees strict zero reads + from non-resident sparse-buffer blocks, only populated device-sized blocks are imported and untouched blocks stay + unbound. It falls back to the ordinary complete import when any required capability or import is unavailable. - **SPIR-V straight from the IR.** No WGSL, no naga, and none of the WebGPU workarounds the WGSL back end carries (texture swizzle and border emulation, d16 quantization, storage format casts, vertex pulling). - **No validation or usage tracking on every call**, and bindless descriptors like NVN's own pools. diff --git a/gpu/vk/eden/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp b/gpu/vk/eden/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp index 2c6760b..fe54e96 100644 --- a/gpu/vk/eden/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp +++ b/gpu/vk/eden/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp @@ -27,6 +27,7 @@ #include "shader_recompiler/frontend/ir/value.h" #include "shader_recompiler/host_translate_info.h" #include "shader_recompiler/ir_opt/passes.h" +#include "shader_recompiler/ir_opt/native_cbuf_pointer_policy.h" namespace Shader::Optimization { @@ -1309,7 +1310,8 @@ bool MixesLoadedPointer(const IR::Value& value) { } thread_local bool g_follow_stage = false; // (relink: the program being translated is compute, where the page table is) -thread_local bool g_graphics_stage = false; // (sharkiix: the program being translated is a geometry stage: vertex, tessellation, geometry, task, mesh) +thread_local bool g_graphics_stage = false; // (relink: any graphics stage, including fragment) +thread_local bool g_geometry_stage = false; // (sharkiix: vertex, tessellation, geometry, task or mesh; not fragment) /// Collects the storage buffer used by a global memory instruction and the instruction itself void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info, Environment& env, @@ -1492,10 +1494,19 @@ void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info, { // (sharkiix, from BernabeCR93 in upstream issue 667) A graphics stage's address tracked to another constant buffer than // c0 while part of it comes from memory: the far-foliage and mesh programs (95db514a35353e28, d905f52b7517c681, // 51183fe750d5d0c8...) took float constants for a buffer address ("storage buffer NOT bound (reads zeros)", c5+0x60), - // read zeros and drew the plants without colour (white/sky-blue "mirror" ferns, glowing flowers). Left to the run-time - // search. RX_VK_NO_CBUF_LOAD_RULE=1: off + // read zeros and drew the plants without colour (white/sky-blue "mirror" ferns, glowing flowers). A native argument + // table can also be mistaken directly for a pointer: da7a7ab28b8e73d9's fragment stage took c5+0's + // 0x4180000000000200 (16.0f and a 0x200 scalar). The environment validates candidate words against mapped GPU + // pools. The memory-origin rule remains geometry-only; native scalar rejection also covers fragment stages. + // Both cases stay on the run-time address search. RX_VK_NO_CBUF_LOAD_RULE=1: off static const bool cbuf_load_rule{std::getenv("RX_VK_NO_CBUF_LOAD_RULE") == nullptr}; - if (cbuf_load_rule && storage_buffer->index != 0 && g_graphics_stage && Origins(inst.Arg(0)).load) { + const bool native_program{env.IsNativeProgram()}; + const bool validate_pointer{native_program && g_graphics_stage && storage_buffer->index != 0}; + const bool mapped_pointer{!validate_pointer || + env.IsCbufStorageBufferPointer(storage_buffer->index, storage_buffer->offset)}; + if (cbuf_load_rule && native_cbuf_pointer_policy::leave_global( + native_program, g_graphics_stage, g_geometry_stage, storage_buffer->index, + Origins(inst.Arg(0)).load, mapped_pointer)) { leave_global(); return; } @@ -1660,8 +1671,10 @@ void GlobalMemoryToStorageBufferPass(Environment& env, IR::Program& program, // (relink) static const bool no_mesh{std::getenv("RX_VK_FOLLOW_MESH") == nullptr}; g_follow_stage = program.stage == Stage::Compute || (!no_mesh && (program.stage == Stage::Task || program.stage == Stage::Mesh)); - // (geometry stages only: with the fragment stage too, the sky drew black rectangles and the lighting went too bright at the castle) - g_graphics_stage = program.stage != Stage::Compute && program.stage != Stage::Fragment; + g_graphics_stage = program.stage != Stage::Compute; + // The older memory-origin workaround remains geometry-only: enabling it for fragment shaders made the sky draw + // black rectangles and lifted the castle lighting. Native unmapped-candidate rejection is independent and safe there. + g_geometry_stage = g_graphics_stage && program.stage != Stage::Fragment; g_origins.clear(); // (EvilOOT: Origins' walk, kept per program) const bool pairing{!GmLegacyFor(program.stage)}; std::optional conditions; diff --git a/gpu/vk/eden/shader_recompiler/ir_opt/native_cbuf_pointer_policy.h b/gpu/vk/eden/shader_recompiler/ir_opt/native_cbuf_pointer_policy.h new file mode 100644 index 0000000..a64cf5e --- /dev/null +++ b/gpu/vk/eden/shader_recompiler/ir_opt/native_cbuf_pointer_policy.h @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT +#pragma once + +#include + +namespace native_cbuf_pointer_policy { + +/// A geometry-stage global-memory access tracked to a non-driver constant bank must stay +/// on the run-time address search when the candidate is not a mapped GPU pointer. Native +/// argument tables mix pointers with handles, sizes, strides, and floating-point values. +/// +/// The memory-origin case predates the native validity check: rebasing an address that is +/// partly loaded through memory onto one constant-buffer candidate loses the loaded path. +inline bool leave_global(bool native_program, bool graphics_stage, bool geometry_stage, + uint32_t cbuf_index, bool address_loads_memory, + bool mapped_pointer) noexcept { + if (cbuf_index == 0) { + return false; + } + // The older memory-origin workaround is deliberately geometry-only: applying it to + // fragment shaders changed valid lighting paths. Rejecting an unmapped native scalar is + // a narrower invariant and applies to every graphics stage, including fragment shaders. + return (geometry_stage && address_loads_memory) || + (graphics_stage && native_program && !mapped_pointer); +} + +} // namespace native_cbuf_pointer_policy diff --git a/gpu/vk/sparse_guest_ranges.h b/gpu/vk/sparse_guest_ranges.h new file mode 100644 index 0000000..4fdaa6c --- /dev/null +++ b/gpu/vk/sparse_guest_ranges.h @@ -0,0 +1,197 @@ +#pragma once + +#include +#include +#include +#include + +namespace sparse_guest { + +struct Run { + uint64_t offset = 0, size = 0; +}; + +/// Append a sorted run, merging overlap and adjacency. False rejects wrapped +/// or out-of-order input instead of producing an unsafe backing range. +inline bool append_merged(std::vector& runs, Run r) { + if (!r.size) return true; + if (r.offset > UINT64_MAX - r.size) return false; + if (runs.empty()) { + runs.push_back(r); + return true; + } + Run& last = runs.back(); + if (last.offset > UINT64_MAX - last.size || r.offset < last.offset) return false; + const uint64_t last_hi = last.offset + last.size; + if (r.offset > last_hi) { + runs.push_back(r); + return true; + } + last.size = std::max(last_hi, r.offset + r.size) - last.offset; + return true; +} + +/// Number of bytes actually backed by a packed sparse allocation. This is +/// deliberately not the sparse resource's virtual span: holes have no dma-buf +/// pages and must not be credited as freed under memory pressure. +inline uint64_t bytes(const std::vector& runs) { + uint64_t result = 0; + for (const Run& r : runs) { + if (r.size > UINT64_MAX - result) return UINT64_MAX; + result += r.size; + } + return result; +} + +inline bool bound(const std::vector& words, uint64_t block) { + return block / 64 < words.size() && (words[block / 64] >> (block & 63) & 1); +} + +inline void set_bound(std::vector& words, uint64_t first, uint64_t count) { + for (uint64_t block = first; block < first + count; block++) + words[block / 64] |= 1ull << (block & 63); +} + +inline void clear_bound(std::vector& words, uint64_t first, uint64_t count) { + for (uint64_t block = first; block < first + count; block++) + if (block / 64 < words.size()) words[block / 64] &= ~(1ull << (block & 63)); +} + +/// Whether any byte backed by these runs touches [lo, hi). `resource_lo` is the guest address represented by offset 0. +inline bool touches(const std::vector& runs, uint64_t resource_lo, uint64_t lo, uint64_t hi) { + if (hi <= lo) return false; + for (const Run& r : runs) { + if (r.offset > UINT64_MAX - resource_lo) return true; + const uint64_t a = resource_lo + r.offset; + const uint64_t b = r.size > UINT64_MAX - a ? UINT64_MAX : a + r.size; + if (a < hi && b > lo) return true; + } + return false; +} + +/// Whether every byte backed by these runs is outside the sorted live ranges. A sparse allocation may pack several +/// disjoint runs into one VkDeviceMemory, so it can be freed only when none of them is live. +template +inline bool all_outside(const std::vector& runs, uint64_t resource_lo, const Ranges& live) { + for (const Run& r : runs) { + if (r.offset > UINT64_MAX - resource_lo) return false; + const uint64_t a = resource_lo + r.offset; + const uint64_t b = r.size > UINT64_MAX - a ? UINT64_MAX : a + r.size; + for (const auto& span : live) { + if (span.lo >= b) break; + if (span.lo < b && span.hi > a) return false; + } + } + return true; +} + +/// Whether [lo, hi) touches a block already bound to the sparse resource. The range may extend past the resource; +/// callers use this when deciding whether freed guest pages are still visible to the GPU. +inline bool overlaps_bound(const std::vector& words, uint64_t resource_lo, uint64_t block_size, + uint64_t lo, uint64_t hi) { + if (!block_size || hi <= lo || hi <= resource_lo || words.empty()) return false; + lo = std::max(lo, resource_lo); + const uint64_t first = (lo - resource_lo) / block_size; + const uint64_t bytes = hi - resource_lo; + const uint64_t limit = words.size() > UINT64_MAX / 64 ? UINT64_MAX : words.size() * 64; + const uint64_t last = std::min(limit, bytes / block_size + (bytes % block_size != 0)); + for (uint64_t block = first; block < last; block++) + if (bound(words, block)) return true; + return false; +} + +/// Mark every sparse block touched by [lo, hi). The caller supplies ranges already clamped to the resource. +inline void mark(std::vector& wanted, uint64_t resource_lo, uint64_t block_size, uint64_t lo, uint64_t hi) { + if (!block_size || hi <= lo || lo < resource_lo) return; + const uint64_t first = (lo - resource_lo) / block_size; + const uint64_t bytes = hi - resource_lo; + const uint64_t last = std::min(wanted.size(), bytes / block_size + (bytes % block_size != 0)); + for (uint64_t block = first; block < last; block++) wanted[(size_t)block] = 1; +} + +/// Merge consecutive wanted, not-yet-bound blocks into Vulkan sparse binding runs. +inline std::vector runs(const std::vector& wanted, const std::vector& bound_words, + uint64_t block_size) { + std::vector result; + for (uint64_t block = 0; block < wanted.size();) { + while (block < wanted.size() && (!wanted[(size_t)block] || bound(bound_words, block))) block++; + const uint64_t first = block; + while (block < wanted.size() && wanted[(size_t)block] && !bound(bound_words, block)) block++; + if (block > first) result.push_back({first * block_size, (block - first) * block_size}); + } + return result; +} + +/// Merge the not-yet-bound blocks touched by [lo, hi) into runs. Offsets are +/// relative to the sparse resource. This lets callers avoid querying their +/// backing store again for blocks whose Vulkan bindings already exist. +inline std::vector unbound_runs(const std::vector& bound_words, uint64_t block_size, + uint64_t resource_size, uint64_t lo, uint64_t hi) { + std::vector result; + if (!block_size || hi <= lo || lo >= resource_size) return result; + hi = std::min(hi, resource_size); + const uint64_t first = lo / block_size; + const uint64_t last = (hi - 1) / block_size + 1; + for (uint64_t block = first; block < last;) { + while (block < last && bound(bound_words, block)) block++; + const uint64_t begin = block; + while (block < last && !bound(bound_words, block)) block++; + if (block > begin) result.push_back({begin * block_size, (block - begin) * block_size}); + } + return result; +} + +/// Continue an extent scan after the sparse block containing `data`. Once any +/// byte in a block is allocated, that whole block is wanted; visiting further +/// file extents inside it cannot change the resulting Vulkan binding. +inline uint64_t scan_advance(uint64_t resource_lo, uint64_t block_size, uint64_t data, + uint64_t hole, uint64_t limit) { + if (!block_size || data < resource_lo || hole < data) return limit; + const uint64_t block = (data - resource_lo) / block_size; + uint64_t block_hi = UINT64_MAX; + if (block < (UINT64_MAX - resource_lo) / block_size) + block_hi = resource_lo + (block + 1) * block_size; + return std::min(limit, std::max(hole, block_hi)); +} + +/// Split aligned runs into the packed allocations accepted by UDMABUF_CREATE_LIST. Both the packed byte total and +/// extent count are per ioctl; a single contiguous run may therefore become several allocations. An empty result for +/// non-empty input means the limits or the input alignment are invalid. +inline std::vector> batches(const std::vector& input, uint64_t max_bytes, + uint64_t max_runs, uint64_t alignment) { + std::vector> result; + if (input.empty()) return result; + if (!max_bytes || !max_runs || !alignment) return result; + const uint64_t byte_limit = max_bytes / alignment * alignment; + if (!byte_limit) return result; + std::vector batch; + uint64_t bytes = 0; + const auto finish = [&] { + if (!batch.empty()) result.push_back(std::move(batch)); + batch.clear(); + bytes = 0; + }; + for (const Run& r : input) { + if (!r.size || r.offset % alignment || r.size % alignment || r.size > UINT64_MAX - r.offset) { + result.clear(); + return result; + } + uint64_t offset = r.offset, left = r.size; + while (left) { + if (bytes == byte_limit || batch.size() >= max_runs) finish(); + const uint64_t take = std::min(left, byte_limit - bytes); + if (!take || take % alignment) { + result.clear(); + return result; + } + batch.push_back({offset, take}); + bytes += take; + offset += take; + left -= take; + } + } + finish(); + return result; +} + +} // namespace sparse_guest diff --git a/gpu/vk/texture_memory_policy.h b/gpu/vk/texture_memory_policy.h new file mode 100644 index 0000000..244c791 --- /dev/null +++ b/gpu/vk/texture_memory_policy.h @@ -0,0 +1,23 @@ +// Effective texture-memory allowance for policies that choose image formats and staging retention. +// nx2/test_texture_memory_policy.py +#pragma once +#include +#include + +namespace texture_memory_policy { + +constexpr uint64_t MiB = 1ull << 20; +constexpr uint64_t IntegratedDefault = 2560ull * MiB; + +// A discrete heap is dedicated VRAM. An integrated heap is shared system RAM, so the driver's heap allowance is not +// what textures may consume beside the game. An explicit texture budget is a still tighter statement of that limit. +inline uint64_t effective_allowance(uint64_t driver_allowed, bool integrated, uint64_t configured_budget = 0) { + const uint64_t cap = configured_budget ? configured_budget : integrated ? IntegratedDefault : 0; + return cap ? (driver_allowed ? std::min(driver_allowed, cap) : cap) : driver_allowed; +} + +inline uint64_t staging_keep(uint64_t effective_allowed) { + return effective_allowed && effective_allowed <= 6656ull * MiB ? 64ull * MiB : 256ull * MiB; +} + +} // namespace texture_memory_policy diff --git a/gpu/vk/vk_internal.h b/gpu/vk/vk_internal.h index ad61e67..c0df9e6 100644 --- a/gpu/vk/vk_internal.h +++ b/gpu/vk/vk_internal.h @@ -55,6 +55,7 @@ struct Device { device_fault = false, // (VK_EXT_device_fault: what a lost device faulted on) sparse_images = false, // (sparse resident 2D images on the queue: the terrain's huge arrays) sparse_buffers = false, // (sparse-bound buffers on the queue: a pool imported as several dma-bufs) + sparse_zero_buffers = false, // (partly unbound buffers read zero: sparse guest-memory holes) view_2d_of_3d = false, // (VK_EXT_image_2d_view_of_3d: a 3D texture a 2D sample instruction reads) coop_matrix = false, // (VK_KHR_cooperative_matrix with its feature) max_reconvergence = false, // (relink, EDEN_NX2_MAXRECONV=1: VK_KHR_shader_maximal_reconvergence with its feature) diff --git a/gpu/vk/vk_main.cpp b/gpu/vk/vk_main.cpp index 27d2aa0..2cdd3e6 100644 --- a/gpu/vk/vk_main.cpp +++ b/gpu/vk/vk_main.cpp @@ -436,6 +436,8 @@ bool create_device() { dev.sparse_images = (families[dev.queue_family].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) && f.features.sparseBinding && f.features.sparseResidencyImage2D && !(getenv("RX_VK_SPARSE") && !atoi(getenv("RX_VK_SPARSE"))); dev.sparse_buffers = (families[dev.queue_family].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) && f.features.sparseBinding; + dev.sparse_zero_buffers = dev.sparse_buffers && f.features.sparseResidencyBuffer && + dev.props.sparseProperties.residencyNonResidentStrict; VkDeviceCreateInfo ci{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; ci.pNext = &f; ci.queueCreateInfoCount = compute_family != UINT32_MAX ? 2 : 1; diff --git a/gpu/vk/vk_memory.cpp b/gpu/vk/vk_memory.cpp index 98f39f5..ee74092 100644 --- a/gpu/vk/vk_memory.cpp +++ b/gpu/vk/vk_memory.cpp @@ -31,12 +31,14 @@ #include "import_hold.h" #include "mirror_inherit.h" #include "mirror_trim.h" +#include "sparse_guest_ranges.h" #ifdef __linux__ #include #include #include #include #include +#include #include #endif #include "shaders/wb_diff_comp_spv.h" @@ -45,6 +47,14 @@ namespace mkwvk { namespace { +#ifdef __linux__ +struct SparseGuestAllocation { + VkDeviceMemory memory = VK_NULL_HANDLE; + std::vector runs; + bool keep = false; // the one-block primary allocation keeps Pool::memory valid for the pool's lifetime +}; +#endif + struct Pool { uint64_t lo = 0, hi = 0; // guest range imported (aligned outwards) uint64_t range_lo = 0, range_hi = 0; // (the game's pool range it was found for: pool_of's last hit) @@ -52,6 +62,12 @@ struct Pool { std::vector pieces; // (a pool imported as several dma-bufs: the ones after `memory`) VkBuffer buffer = VK_NULL_HANDLE; VkDeviceAddress address = 0; +#ifdef __linux__ + bool sparse_guest = false; // (RX_VK_SPARSE_GUEST: untouched guest blocks stay unbound and read as zero) + uint64_t sparse_align = 0, sparse_size = 0; + std::vector sparse_bound; // (one bit per sparse block already backed by the guest memfd) + std::vector sparse_allocations; // the exact runs owned by each imported dma-buf +#endif // its VRAM mirror (see "the VRAM mirror" below) enum : uint8_t { MIRROR_NONE, MIRROR_WANTED, MIRROR_ON, MIRROR_REFUSED } mstate = MIRROR_NONE; VkBuffer mirror = VK_NULL_HANDLE; @@ -87,6 +103,21 @@ struct Pool { std::vector wb_flush_words; // (the words of wb_flushing that are not zero) uint32_t wb_pending_n = 0; }; + +uint64_t pool_imported_bytes(const Pool& p) { +#ifdef __linux__ + if (p.sparse_guest) { + uint64_t bytes = 0; + for (const SparseGuestAllocation& allocation : p.sparse_allocations) { + const uint64_t allocation_bytes = sparse_guest::bytes(allocation.runs); + if (allocation_bytes > UINT64_MAX - bytes) return UINT64_MAX; + bytes += allocation_bytes; + } + return bytes; + } +#endif + return p.hi - p.lo; +} uint64_t mem_rec_value = 0, mem_completed = 0; // (the recording being made, and the GPU's progress) struct GuestPool { @@ -145,35 +176,73 @@ constexpr VkBufferUsageFlags kUsage = // pool is still one buffer with one device address and nobody has to raise the limit. RX_VK_NO_DMABUF=1: host pointers only, // as before; RX_VK_DMABUF_PIECE_MB=: pieces of at most n MB (to try the sparse path with the limit raised) struct UdmabufCreate { uint32_t memfd, flags; uint64_t offset, size; }; +struct UdmabufCreateItem { uint32_t memfd, pad; uint64_t offset, size; }; +struct UdmabufCreateList { uint32_t flags, count; }; constexpr unsigned long kUdmabufCreate = (1ul << 30) | (sizeof(UdmabufCreate) << 16) | ('u' << 8) | 0x42; // (_IOW('u', 0x42, ...)) +constexpr unsigned long kUdmabufCreateList = + (1ul << 30) | (sizeof(UdmabufCreateList) << 16) | ('u' << 8) | 0x43; // (_IOW('u', 0x43, ...)) int dmabuf_memfd = -2, dmabuf_dev = -1, dmabuf_failures = 0; -uint64_t dmabuf_limit = 0; +uint64_t dmabuf_limit = 0, dmabuf_piece = 0, dmabuf_list_limit = 1024; +std::atomic dmabuf_seen_blocks{-1}; -/// The memory of guest [lo, lo + size) as an imported dma-buf; VK_NULL_HANDLE when udmabuf or the import refused it -VkDeviceMemory dma_buf_memory(uint64_t lo, uint64_t size) { - UdmabufCreate c{(uint32_t)dmabuf_memfd, 1 /* UDMABUF_FLAGS_CLOEXEC */, lo, size}; - const int fd = ioctl(dmabuf_dev, kUdmabufCreate, &c); - if (fd < 0) { - if (dmabuf_failures++ < 8) - logf("vk: guest %#llx..%#llx: no udmabuf (errno %d)", (unsigned long long)lo, (unsigned long long)(lo + size), errno); - return VK_NULL_HANDLE; - } - static uint32_t type = UINT32_MAX; // (the same for every dma-buf of the window: asked once) - if (type == UINT32_MAX) { - VkMemoryFdPropertiesKHR fp{VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR}; - if (vkGetMemoryFdPropertiesKHR(dev.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, fd, &fp) == VK_SUCCESS) { - for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) // (host visible, coherent, first) - if ((fp.memoryTypeBits >> i & 1) && - (dev.memory.memoryTypes[i].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) == - (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) - type = i; - for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) - if (fp.memoryTypeBits >> i & 1) type = i; +bool dma_buf_init() { + static std::mutex init_mu; + std::lock_guard lk(init_mu); + if (dmabuf_memfd != -2) return dmabuf_memfd >= 0; + dmabuf_memfd = -1; + typedef int (*MemfdFn)(void); + const MemfdFn fn = getenv("RX_VK_NO_DMABUF") ? nullptr : (MemfdFn)dlsym(RTLD_DEFAULT, "hos_posix_guest_memfd"); + const int fd = fn ? fn() : -1; + if (fd >= 0 && dev.dma_buf_import && vkGetMemoryFdPropertiesKHR) dmabuf_dev = open("/dev/udmabuf", O_RDWR | O_CLOEXEC); + if (fd >= 0 && dmabuf_dev >= 0) { + dmabuf_memfd = fd; + dmabuf_limit = 64ull << 20; + if (FILE* f = fopen("/sys/module/udmabuf/parameters/size_limit_mb", "r")) { + unsigned long long mb = 0; + if (fscanf(f, "%llu", &mb) == 1 && mb) dmabuf_limit = mb << 20; + fclose(f); + } + if (FILE* f = fopen("/sys/module/udmabuf/parameters/list_limit", "r")) { + unsigned long long n = 0; + if (fscanf(f, "%llu", &n) == 1 && n) dmabuf_list_limit = n; + fclose(f); } + dmabuf_piece = dmabuf_limit; + if (const char* e = getenv("RX_VK_DMABUF_PIECE_MB")) + if (atoll(e) > 0) dmabuf_piece = std::min(dmabuf_piece, (uint64_t)atoll(e) << 20); + } + logf("vk: guest memory as dma-bufs: %s", dmabuf_memfd >= 0 ? "yes" : !fn ? "no (no memfd window, or RX_VK_NO_DMABUF)" + : fd < 0 ? "no (the window is anonymous memory)" : !dev.dma_buf_import ? "no (VK_EXT_external_memory_dma_buf)" + : "no (/dev/udmabuf cannot be opened)"); + if (dmabuf_memfd >= 0) + logf("vk: dma-bufs of at most %llu MB (udmabuf's limit %llu MB); larger pools %s", (unsigned long long)(dmabuf_piece >> 20), + (unsigned long long)(dmabuf_limit >> 20), + dev.sparse_buffers ? "are several, in a sparse buffer" : "are host-pointer imports (no sparse binding)"); + return dmabuf_memfd >= 0; +} + +/// Import an already-created dma-buf. Vulkan consumes fd on success and leaves it to the caller on failure. +VkDeviceMemory dma_buf_fd_memory(int fd, uint64_t size, uint64_t guest_lo) { + static uint32_t type = UINT32_MAX; // (the same for every dma-buf of the window: asked once) + static std::mutex type_mu; + { + std::lock_guard lk(type_mu); if (type == UINT32_MAX) { - close(fd); - if (dmabuf_failures++ < 8) logf("vk: a dma-buf of guest memory has no memory type"); - return VK_NULL_HANDLE; + VkMemoryFdPropertiesKHR fp{VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR}; + if (vkGetMemoryFdPropertiesKHR(dev.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, fd, &fp) == VK_SUCCESS) { + for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) // (host visible, coherent, first) + if ((fp.memoryTypeBits >> i & 1) && + (dev.memory.memoryTypes[i].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) == + (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) + type = i; + for (uint32_t i = 0; i < dev.memory.memoryTypeCount && type == UINT32_MAX; i++) + if (fp.memoryTypeBits >> i & 1) type = i; + } + if (type == UINT32_MAX) { + close(fd); + if (dmabuf_failures++ < 8) logf("vk: a dma-buf of guest memory has no memory type"); + return VK_NULL_HANDLE; + } } } VkImportMemoryFdInfoKHR im{VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR}; @@ -191,13 +260,366 @@ VkDeviceMemory dma_buf_memory(uint64_t lo, uint64_t size) { if (r != VK_SUCCESS) { // (the fd is the caller's again when the import fails) close(fd); if (dmabuf_failures++ < 8) - logf("vk: guest %#llx..%#llx: its dma-buf not imported (VkResult %d)", (unsigned long long)lo, - (unsigned long long)(lo + size), (int)r); + logf("vk: guest %#llx: its %llu-byte dma-buf not imported (VkResult %d)", (unsigned long long)guest_lo, + (unsigned long long)size, (int)r); return VK_NULL_HANDLE; } return memory; } +/// The memory of guest [lo, lo + size) as an imported dma-buf; VK_NULL_HANDLE when udmabuf or the import refused it +VkDeviceMemory dma_buf_memory(uint64_t lo, uint64_t size) { + UdmabufCreate c{(uint32_t)dmabuf_memfd, 1 /* UDMABUF_FLAGS_CLOEXEC */, lo, size}; + const int fd = ioctl(dmabuf_dev, kUdmabufCreate, &c); + if (fd < 0) { + if (dmabuf_failures++ < 8) + logf("vk: guest %#llx..%#llx: no udmabuf (errno %d)", (unsigned long long)lo, (unsigned long long)(lo + size), errno); + return VK_NULL_HANDLE; + } + return dma_buf_fd_memory(fd, size, lo); +} + +VkResult bind_sparse_buffers(const std::vector& buffers, bool drain_before) { + if (buffers.empty()) return VK_SUCCESS; + VkBindSparseInfo si{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO}; + si.bufferBindCount = (uint32_t)buffers.size(); + si.pBufferBinds = buffers.data(); + static VkFence fence = VK_NULL_HANDLE; // (the background importer and the recording thread can both import) + static std::mutex fence_mu; + std::lock_guard fk(fence_mu); + if (!fence) { + VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + const VkResult r = vkCreateFence(dev.device, &fi, nullptr, &fence); + if (r != VK_SUCCESS) return r; + } + VkResult res; + { // Sparse binds are not implicitly ordered against command-buffer execution, even on one queue. A mapping change on + // a published buffer drains earlier access; a brand-new buffer has none. The fence completes either change before + // any later submit can use it. + std::lock_guard qk(dev.present_mu); + queue_flush_locked(); + res = drain_before ? vkQueueWaitIdle(dev.queue) : VK_SUCCESS; + if (res == VK_SUCCESS) res = vkQueueBindSparse(dev.queue, 1, &si, fence); + } + if (res == VK_SUCCESS) { + res = vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX); + vkResetFences(dev.device, 1, &fence); + } + return res; +} + +VkResult bind_sparse_buffer(VkBuffer buffer, const std::vector& binds, bool drain_before = false) { + if (binds.empty()) return VK_SUCCESS; + const std::vector buffers{{buffer, (uint32_t)binds.size(), binds.data()}}; + return bind_sparse_buffers(buffers, drain_before); +} + +/// Import and bind one allocation's bounded runs. The primary allocation is deliberately one block: Pool::memory remains +/// valid even when every other allocation is released, while at most one sparse block stays pinned for that bookkeeping rule. +bool sparse_guest_bind_batch(Pool& p, const std::vector& runs, bool keep, bool drain_before) { + if (runs.empty()) return true; + uint64_t packed_size = 0; + for (const sparse_guest::Run& r : runs) packed_size += r.size; + if (!packed_size || packed_size > dmabuf_limit || runs.size() > dmabuf_list_limit || runs.size() > UINT32_MAX) { + if (dmabuf_failures++ < 8) + logf("vk: pool %#llx: %llu MB of sparse guest blocks exceed one udmabuf list", (unsigned long long)p.lo, + (unsigned long long)(packed_size >> 20)); + return false; + } + std::vector storage(sizeof(UdmabufCreateList) + runs.size() * sizeof(UdmabufCreateItem)); + auto* create = reinterpret_cast(storage.data()); + auto* items = reinterpret_cast(storage.data() + sizeof(UdmabufCreateList)); + create->flags = 1 /* UDMABUF_FLAGS_CLOEXEC */; + create->count = (uint32_t)runs.size(); + for (size_t i = 0; i < runs.size(); i++) + items[i] = {(uint32_t)dmabuf_memfd, 0, p.lo + runs[i].offset, runs[i].size}; + const int fd = ioctl(dmabuf_dev, kUdmabufCreateList, create); + if (fd < 0) { + if (dmabuf_failures++ < 8) + logf("vk: pool %#llx: UDMABUF_CREATE_LIST refused %zu extents (errno %d)", (unsigned long long)p.lo, + runs.size(), errno); + return false; + } + const VkDeviceMemory memory = dma_buf_fd_memory(fd, packed_size, p.lo); + if (!memory) return false; + std::vector binds; + uint64_t memory_offset = 0; + for (const sparse_guest::Run& r : runs) { + binds.push_back({r.offset, r.size, memory, memory_offset, 0}); + memory_offset += r.size; + } + const VkResult res = bind_sparse_buffer(p.buffer, binds, drain_before); + if (res != VK_SUCCESS) { + vkFreeMemory(dev.device, memory, nullptr); + if (dmabuf_failures++ < 8) + logf("vk: pool %#llx: %zu sparse guest extents not bound (VkResult %d)", (unsigned long long)p.lo, + runs.size(), (int)res); + return false; + } + if (keep) p.memory = memory; + else p.pieces.push_back(memory); + p.sparse_allocations.push_back({memory, runs, keep}); + for (const sparse_guest::Run& r : runs) + sparse_guest::set_bound(p.sparse_bound, r.offset / p.sparse_align, r.size / p.sparse_align); + static uint64_t bound_bytes; + static int64_t bound_calls; + bound_bytes += packed_size, bound_calls++; + if (bound_calls <= 8 || bound_calls % 64 == 0) + logf("vk: sparse guest pool %#llx: bound %zu runs / %llu KB (%lld binds, %llu MB in all)", + (unsigned long long)p.lo, runs.size(), (unsigned long long)(packed_size >> 10), (long long)bound_calls, + (unsigned long long)(bound_bytes >> 20)); + return true; +} + +/// Import any number of runs as as many bounded UDMABUF_CREATE_LIST allocations as the kernel's byte and extent limits +/// require. A run itself may be larger than size_limit_mb; splitting it preserves the sparse resource instead of falling +/// back to a dense import that pins all of its holes. +bool sparse_guest_bind_runs(Pool& p, const std::vector& runs, bool keep, bool drain_before) { + if (runs.empty()) return true; + const uint64_t list_limit = std::min(dmabuf_list_limit, UINT32_MAX); + const auto batches = sparse_guest::batches(runs, dmabuf_limit, list_limit, p.sparse_align); + if (batches.empty()) { + if (dmabuf_failures++ < 8) + logf("vk: pool %#llx: sparse guest runs cannot fit udmabuf's %llu MB / %llu-extent lists", + (unsigned long long)p.lo, (unsigned long long)(dmabuf_limit >> 20), + (unsigned long long)dmabuf_list_limit); + return false; + } + static int split_logged; + if (batches.size() > 1 && split_logged++ < 16) + logf("vk: pool %#llx: %zu sparse guest runs split across %zu udmabuf lists", + (unsigned long long)p.lo, runs.size(), batches.size()); + for (size_t i = 0; i < batches.size(); i++) + if (!sparse_guest_bind_batch(p, batches[i], keep && i == 0, drain_before && i == 0)) return false; + return true; +} + +/// Back the data blocks in sorted resource-relative ranges, or every block there when `force` is true. The latter is +/// used before GPU writes, so their results always land in the guest memfd. `ensure_one` keeps Pool::memory non-null. +bool sparse_guest_bind_ranges(Pool& p, const std::vector& scan_ranges, bool force, bool ensure_one) { + if (!p.sparse_guest || !p.sparse_align || !p.sparse_size) return true; + const uint64_t blocks = (p.sparse_size + p.sparse_align - 1) / p.sparse_align; + std::vector wanted((size_t)blocks); + if (force) { + for (const sparse_guest::Run& r : scan_ranges) + sparse_guest::mark(wanted, p.lo, p.sparse_align, p.lo + r.offset, p.lo + r.offset + r.size); + } else if (!scan_ranges.empty()) { + // A changed memfd block count refreshes every live sparse import. Most + // of those blocks are normally already bound; walking their often + // fragmented SEEK_DATA map again made an area load spend whole frames + // in lseek. Only an unbound block can need a new Vulkan binding. + std::vector unbound; + for (const sparse_guest::Run& scan : scan_ranges) { + const auto found = sparse_guest::unbound_runs(p.sparse_bound, p.sparse_align, p.sparse_size, + scan.offset, scan.offset + scan.size); + for (const sparse_guest::Run& r : found) + if (!sparse_guest::append_merged(unbound, r)) return false; + } + if (!unbound.empty()) { + const auto scan_started = std::chrono::steady_clock::now(); + uint64_t scan_seeks = 0; + const int scanfd = fcntl(dmabuf_memfd, F_DUPFD_CLOEXEC, 0); + if (scanfd < 0) return false; + for (const sparse_guest::Run& r : unbound) { + const uint64_t run_lo = p.lo + r.offset; + const uint64_t run_hi = run_lo + r.size; + for (uint64_t at = run_lo; at < run_hi;) { + const off_t data = lseek(scanfd, (off_t)at, SEEK_DATA); + scan_seeks++; + if (data < 0) { + const int e = errno; + if (e == ENXIO) goto sparse_scan_done; + close(scanfd); + if (dmabuf_failures++ < 8) logf("vk: guest sparse scan failed (errno %d)", e); + return false; + } + if ((uint64_t)data >= run_hi) break; + const off_t hole = lseek(scanfd, data, SEEK_HOLE); + scan_seeks++; + if (hole < 0) { + const int e = errno; + close(scanfd); + if (dmabuf_failures++ < 8) logf("vk: guest sparse hole scan failed (errno %d)", e); + return false; + } + const uint64_t hi = std::min((uint64_t)hole, run_hi); + sparse_guest::mark(wanted, p.lo, p.sparse_align, (uint64_t)data, hi); + // A shmem file can have many page-sized extents inside one + // Vulkan sparse block. The first already makes the complete + // block wanted, so do not issue two more seeks per fragment. + at = sparse_guest::scan_advance(p.lo, p.sparse_align, (uint64_t)data, hi, run_hi); + } + } +sparse_scan_done: + close(scanfd); + const double scan_ms = std::chrono::duration( + std::chrono::steady_clock::now() - scan_started) + .count(); + if (scan_ms >= 20.0) { + static int64_t slow_scans; + slow_scans++; + if (slow_scans <= 32 || slow_scans % 64 == 0) + logf("vk: sparse guest pool %#llx: %.1f ms scanning %llu KB, %zu live / %zu unbound runs, %llu seeks", + (unsigned long long)p.lo, scan_ms, + (unsigned long long)(sparse_guest::bytes(scan_ranges) >> 10), scan_ranges.size(), unbound.size(), + (unsigned long long)scan_seeks); + } + } + } + bool any = false; + for (uint8_t x : wanted) any |= x != 0; + // The generic Pool lifecycle owns its primary allocation through p.memory. Always include block zero in the first + // sparse import, even when other populated blocks were found, so the split below can establish that invariant. + if (ensure_one && blocks) wanted[0] = 1, any = true; + if (!any) return true; + std::vector runs = sparse_guest::runs(wanted, p.sparse_bound, p.sparse_align); + if (runs.empty()) return true; + if (!p.memory && ensure_one) { + // wanted[0] made the first run start at zero. Split one block out so later releases never have to keep a large, + // mixed allocation merely because Pool::memory is the primary handle used by the generic cleanup paths. + const sparse_guest::Run primary{0, p.sparse_align}; + if (runs.front().offset != 0 || runs.front().size < p.sparse_align || + !sparse_guest_bind_runs(p, {primary}, true, false)) + return false; + runs.front().offset += p.sparse_align; + runs.front().size -= p.sparse_align; + if (!runs.front().size) runs.erase(runs.begin()); + return sparse_guest_bind_runs(p, runs, false, false); + } + return sparse_guest_bind_runs(p, runs, false, true); +} + +bool sparse_guest_bind(Pool& p, uint64_t guest_lo, uint64_t guest_hi, bool force, bool ensure_one) { + std::vector ranges; + const uint64_t scan_lo = std::max(guest_lo, p.lo); + const uint64_t scan_hi = std::min(guest_hi, p.lo + p.sparse_size); + if (scan_hi > scan_lo) ranges.push_back({scan_lo - p.lo, scan_hi - scan_lo}); + return sparse_guest_bind_ranges(p, ranges, force, ensure_one); +} + +struct SparseGuestRelease { + Pool* pool = nullptr; + std::vector allocations; + std::vector binds; +}; + +/// Plan whole imported allocations touched by any released range once none of their packed runs belongs to a current +/// game pool. Vulkan names one VkDeviceMemory for all of an allocation's runs, so freeing only part would leave the same +/// dma-buf (and all its pinned pages) alive. The one-block primary allocation is kept until Pool teardown. +void sparse_guest_release_plan(Pool& p, const std::vector>& released, + SparseGuestRelease& plan) { + if (!p.sparse_guest || !p.sparse_align) return; + plan.pool = &p; + for (size_t i = 0; i < p.sparse_allocations.size(); i++) { + const SparseGuestAllocation& a = p.sparse_allocations[i]; + bool touched = false; + for (const auto& [lo, hi] : released) + if (sparse_guest::touches(a.runs, p.lo, lo, hi)) { + touched = true; + break; + } + if (a.keep || !touched || !sparse_guest::all_outside(a.runs, p.lo, guest_pools)) continue; + plan.allocations.push_back(i); + for (const sparse_guest::Run& r : a.runs) + plan.binds.push_back({r.offset, r.size, VK_NULL_HANDLE, 0, 0}); + } +} + +void sparse_guest_release_finish(SparseGuestRelease& plan, std::vector>& freed) { + Pool& p = *plan.pool; + uint64_t bytes = 0; + const size_t allocations = plan.allocations.size(); + for (auto at = plan.allocations.rbegin(); at != plan.allocations.rend(); ++at) { + SparseGuestAllocation a = std::move(p.sparse_allocations[*at]); + for (const sparse_guest::Run& r : a.runs) { + sparse_guest::clear_bound(p.sparse_bound, r.offset / p.sparse_align, r.size / p.sparse_align); + const uint64_t lo = p.lo + r.offset; + freed.push_back({lo, lo + r.size}); + bytes += r.size; + } + const auto piece = std::find(p.pieces.begin(), p.pieces.end(), a.memory); + if (piece != p.pieces.end()) p.pieces.erase(piece); + vkFreeMemory(dev.device, a.memory, nullptr); + p.sparse_allocations.erase(p.sparse_allocations.begin() + (ptrdiff_t)*at); + } + static uint64_t released_bytes; + static int64_t released_allocations; + released_bytes += bytes, released_allocations += (int64_t)allocations; + logf("vk: sparse guest pool %#llx: released %zu allocations / %llu KB (%lld allocations, %llu MB in all)", + (unsigned long long)p.lo, allocations, (unsigned long long)(bytes >> 10), (long long)released_allocations, + (unsigned long long)(released_bytes >> 20)); +} + +bool import_sparse_guest(uint64_t lo, uint64_t hi, Pool& p) { + static const bool asked = getenv("RX_VK_SPARSE_GUEST") && atoi(getenv("RX_VK_SPARSE_GUEST")) != 0; + if (!asked) return false; + static bool reported; + if (!dev.sparse_zero_buffers || !dma_buf_init()) { + if (!reported) logf("vk: sparse guest memory: unavailable (needs dma-buf, sparse residency, strict zero reads)"), reported = true; + return false; + } + if (!reported) logf("vk: sparse guest memory: on (RX_VK_SPARSE_GUEST)"), reported = true; + VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; + eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; + VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bi.pNext = &eb; + bi.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT | VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT; + bi.size = hi - lo; + bi.usage = kUsage; + VkBuffer buffer = VK_NULL_HANDLE; + if (vkCreateBuffer(dev.device, &bi, nullptr, &buffer) != VK_SUCCESS) return false; + VkMemoryRequirements req{}; + vkGetBufferMemoryRequirements(dev.device, buffer, &req); + const uint64_t block = std::max(req.alignment, 4096); + if (!block || (block & (block - 1)) || (req.size % block) || lo + req.size < lo || lo + req.size > host->guest_size()) { + vkDestroyBuffer(dev.device, buffer, nullptr); + return false; + } + p.lo = lo, p.hi = hi, p.buffer = buffer, p.sparse_guest = true, p.sparse_align = block, p.sparse_size = req.size; + p.sparse_bound.assign((size_t)((req.size / block + 63) / 64), 0); + // Establish only the primary allocation here. The imported buffer is a + // chunk around the resource that caused the lookup, and scanning that + // whole chunk made a small texture upload pay SEEK_DATA over tens of MB + // (and occasionally seconds under shmem pressure). gpu_buffer[_bind] + // binds the exact range before returning it to a Vulkan consumer; imports + // made ahead need no other blocks until their first binding. + if (!sparse_guest_bind(p, lo, lo, false, true)) { + vkDestroyBuffer(dev.device, buffer, nullptr); + if (p.memory) vkFreeMemory(dev.device, p.memory, nullptr); + for (VkDeviceMemory m : p.pieces) vkFreeMemory(dev.device, m, nullptr); + p = Pool{}; + return false; + } + VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; + da.buffer = p.buffer; + p.address = vkGetBufferDeviceAddress(dev.device, &da); + return true; +} + +/// Newly allocated memfd extents can appear after a pool was imported. A single fstat normally makes this free; only a +/// changed block count scans the parts of sparse imports that current game pools cover. Scanning an import's whole range +/// would immediately rebind CPU pages after the game released their old pool. Explicit bindings refresh their exact range. +void sparse_guest_refresh_all() { + if (dmabuf_memfd < 0) return; + struct stat st{}; + if (fstat(dmabuf_memfd, &st) != 0) return; + if (dmabuf_seen_blocks.load(std::memory_order_relaxed) == st.st_blocks) return; + bool ok = true; + for (auto& kv : imported) { + Pool& p = kv.second; + if (!p.sparse_guest) continue; + std::vector live; + for (const GuestPool& g : guest_pools) { + if (g.lo >= p.lo + p.sparse_size) break; + if (g.hi <= p.lo) continue; + const uint64_t lo = std::max(p.lo, g.lo), hi = std::min(p.lo + p.sparse_size, g.hi); + if (hi > lo && !sparse_guest::append_merged(live, {lo - p.lo, hi - lo})) ok = false; + } + if (!live.empty() && !sparse_guest_bind_ranges(p, live, false, false)) ok = false; + } + dmabuf_seen_blocks.store(ok ? st.st_blocks : -1, std::memory_order_relaxed); +} + /// A pool larger than a dma-buf may be: a sparse buffer, each `piece` bytes of it bound to its own dma-buf bool import_dma_buf_pieces(uint64_t lo, uint64_t hi, uint64_t piece, Pool& p) { VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; @@ -237,26 +659,7 @@ bool import_dma_buf_pieces(uint64_t lo, uint64_t hi, uint64_t piece, Pool& p) { } VkResult res = VK_ERROR_UNKNOWN; if (binds.size() == (req.size + piece - 1) / piece) { - VkSparseBufferMemoryBindInfo bb{buffer, (uint32_t)binds.size(), binds.data()}; - VkBindSparseInfo si{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO}; - si.bufferBindCount = 1; - si.pBufferBinds = &bb; - static VkFence fence = VK_NULL_HANDLE; // (the background importer and the recording thread can both import) - static std::mutex fence_mu; - std::lock_guard fk(fence_mu); - if (!fence) { - VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; - vkCreateFence(dev.device, &fi, nullptr, &fence); - } - { // (on the queue, as the sparse textures' binds: nothing uses the new buffer yet, so no wait for earlier work) - std::lock_guard qk(dev.present_mu); - queue_flush_locked(); - res = vkQueueBindSparse(dev.queue, 1, &si, fence); - } - if (res == VK_SUCCESS) { - res = vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX); - vkResetFences(dev.device, 1, &fence); - } + res = bind_sparse_buffer(buffer, binds); } if (res != VK_SUCCESS) { vkDestroyBuffer(dev.device, buffer, nullptr); @@ -272,36 +675,10 @@ bool import_dma_buf_pieces(uint64_t lo, uint64_t hi, uint64_t piece, Pool& p) { } bool import_dma_buf(uint64_t lo, uint64_t hi, Pool& p) { - static uint64_t piece = 0; - if (dmabuf_memfd == -2) { - dmabuf_memfd = -1; - typedef int (*MemfdFn)(void); - const MemfdFn fn = getenv("RX_VK_NO_DMABUF") ? nullptr : (MemfdFn)dlsym(RTLD_DEFAULT, "hos_posix_guest_memfd"); - const int fd = fn ? fn() : -1; - if (fd >= 0 && dev.dma_buf_import && vkGetMemoryFdPropertiesKHR) dmabuf_dev = open("/dev/udmabuf", O_RDWR | O_CLOEXEC); - if (fd >= 0 && dmabuf_dev >= 0) { - dmabuf_memfd = fd; - dmabuf_limit = 64ull << 20; - if (FILE* f = fopen("/sys/module/udmabuf/parameters/size_limit_mb", "r")) { - unsigned long long mb = 0; - if (fscanf(f, "%llu", &mb) == 1 && mb) dmabuf_limit = mb << 20; - fclose(f); - } - piece = dmabuf_limit; - if (const char* e = getenv("RX_VK_DMABUF_PIECE_MB")) - if (atoll(e) > 0) piece = std::min(piece, (uint64_t)atoll(e) << 20); - } - logf("vk: guest memory as dma-bufs: %s", dmabuf_memfd >= 0 ? "yes" : !fn ? "no (no memfd window, or RX_VK_NO_DMABUF)" - : fd < 0 ? "no (the window is anonymous memory)" : !dev.dma_buf_import ? "no (VK_EXT_external_memory_dma_buf)" - : "no (/dev/udmabuf cannot be opened)"); - if (dmabuf_memfd >= 0) - logf("vk: dma-bufs of at most %llu MB (udmabuf's limit %llu MB); larger pools %s", (unsigned long long)(piece >> 20), - (unsigned long long)(dmabuf_limit >> 20), - dev.sparse_buffers ? "are several, in a sparse buffer" : "are host-pointer imports (no sparse binding)"); - } + dma_buf_init(); if (dmabuf_memfd < 0 || (lo & 4095) || (hi & 4095)) return false; - if (hi - lo > piece) { - if (!dev.sparse_buffers || !import_dma_buf_pieces(lo, hi, piece, p)) return false; + if (hi - lo > dmabuf_piece) { + if (!dev.sparse_buffers || !import_dma_buf_pieces(lo, hi, dmabuf_piece, p)) return false; } else { VkExternalMemoryBufferCreateInfo eb{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; eb.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; @@ -365,6 +742,14 @@ bool import_pool(uint64_t lo, uint64_t hi, Pool& p) { total, ms(a, b), ms(b, c), ms(c, d), ms(d, e), (unsigned long long)(n >> 10), (unsigned long long)(com >> 10)); } } report{t0, t1, t2, t3, t4, committed, bytes}; +#ifdef __linux__ + // (opt-in: inspect the memfd before the ordinary path commits every reserved page. The sparse buffer leaves + // untouched blocks unbound; residencyNonResidentStrict makes their reads deterministic zero.) + if (import_sparse_guest(lo, hi, p)) { + t1 = t2 = t3 = t4 = clk::now(); + return true; + } +#endif // (pages the game has not touched yet may be only reserved: committed now, as the pool's memory is usable; a range // committed already is left alone: the call took milliseconds in the game's huge reservation) { @@ -586,6 +971,11 @@ void memory_sync() { std::sort(next.begin(), next.end(), [](const GuestPool& a, const GuestPool& b) { return a.lo < b.lo; }); std::lock_guard lk(pools_mu); guest_pools.swap(next); +#ifdef __linux__ + // The memfd's block count need not change when a game pool begins covering already-allocated pages. Make the next + // submission refresh the new live intersections even when fstat alone would otherwise skip the scan. + dmabuf_seen_blocks.store(-1, std::memory_order_relaxed); +#endif guest_pools_gen.fetch_add(1, std::memory_order_relaxed); last_pool = nullptr; std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr); @@ -3234,6 +3624,12 @@ BufferRef gpu_buffer(uint64_t gpu, uint64_t size) { std::lock_guard lk(pools_mu); Pool* p = pool_of(guest, size); if (!p) return {}; +#ifdef __linux__ + // A sparse import starts with only its primary block. Bind the exact + // direct-source span here instead of scanning the import chunk around it. + // gpu_buffer_bind does the same for descriptor-backed ranges. + if (p->sparse_guest && !sparse_guest_bind(*p, guest, guest + size, false, false)) return {}; +#endif if (p->last_value < mem_rec_value) p->last_value = mem_rec_value; // (in use: not given back under it, imports_give_back) return {p->buffer, guest - p->lo, size, p->address + (guest - p->lo)}; } @@ -3254,14 +3650,24 @@ void memory_pool_created(uint64_t gpu, uint64_t size) { int memory_guest_imported(uint64_t lo, uint64_t hi) { std::lock_guard lk(pools_mu); + auto overlaps_import = [&](const Pool& p) { + if (!p.memory || p.lo >= hi || lo >= p.hi) return false; +#ifdef __linux__ + // An unbound sparse block is not visible to the GPU and strict non-resident reads are zero. Let the host punch + // freed pages there out of the memfd instead of allocating them merely to write the same zeroes. + if (p.sparse_guest) + return sparse_guest::overlaps_bound(p.sparse_bound, p.lo, p.sparse_align, lo, hi); +#endif + return true; + }; auto it = imported.upper_bound(hi); // (pools by lo: those starting below hi, the last ones checked first) while (it != imported.begin()) { --it; - if (it->second.hi > lo) return 1; + if (overlaps_import(it->second)) return 1; if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo) } for (const auto& nh : retired_pools) - if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return 1; + if (overlaps_import(nh.mapped())) return 1; return 0; } @@ -3377,6 +3783,12 @@ BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { bind_memo[bind_slot(guest, size)] = BindMemo{guest, size, gpu_bind_stamp(), ref}; return ref; }; + auto remember_pool = [&](Pool* p, BufferRef ref) { +#ifdef __linux__ + if (p->sparse_guest) return ref; // (the range is checked for newly allocated blocks at every binding) +#endif + return remember(ref); + }; if (!written) { const BindMemo& m = bind_memo[bind_slot(guest, size)]; if (bind_memo_hit(guest, size, gpu_bind_stamp(), m.guest, m.size, m.stamp, m.ref.buffer != VK_NULL_HANDLE)) @@ -3387,15 +3799,19 @@ BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { // lookup under the lock was ~8% of the replay's time in a race); the rest under it, as below) const bool par = par_active.load(std::memory_order_relaxed); TlPool& tl = tl_pools[(guest >> 16) & 1023]; - if (!written && tl.p && tl.gen == pools_gen.load(std::memory_order_relaxed) && guest >= tl.p->lo && - guest + size <= tl.p->hi) { + bool tl_hit = !written && tl.p && tl.gen == pools_gen.load(std::memory_order_relaxed) && guest >= tl.p->lo && + guest + size <= tl.p->hi; +#ifdef __linux__ + if (tl_hit && tl.p->sparse_guest) tl_hit = false; +#endif + if (tl_hit) { Pool* p = tl.p; const uint64_t off = guest - p->lo; if (p->last_value < mem_rec_value) p->last_value = mem_rec_value; if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED || p->mstate == Pool::MIRROR_WANTED) - return remember({p->buffer, off, size, 0}); + return remember_pool(p, {p->buffer, off, size, 0}); if (p->mstate == Pool::MIRROR_ON && p->synced_epoch == mirror_epoch && p->sync_seg <= texture_segment()) - return remember({p->mirror, off, size, 0}); + return remember_pool(p, {p->mirror, off, size, 0}); } ParLock plk; // (before pools_mu, always: a mirror sync records into the shared upload command buffer) std::lock_guard lk(pools_mu); @@ -3404,9 +3820,12 @@ BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { tl = {p, pools_gen.load(std::memory_order_relaxed)}; if (p->last_value < mem_rec_value) p->last_value = mem_rec_value; const uint64_t off = guest - p->lo; +#ifdef __linux__ + if (p->sparse_guest && !sparse_guest_bind(*p, guest, guest + size, written, false)) return {}; +#endif if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED) { if (mirror_on()) { auto& e = direct_by_pool[p->lo]; e.first++, e.second = mirror_upload ? (int)p->mstate : 9; direct_bytes_by_pool[p->lo] += size; } - return remember({p->buffer, off, size, 0}); + return remember_pool(p, {p->buffer, off, size, 0}); } // (a pool's mirror is made at its first binding, in this part's uploads: bound in guest memory until the next // submission, the GPU read it there while the game's threads wrote it for their next frames, and a pass that drew @@ -3435,7 +3854,7 @@ BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { if (written) p->direct_write_epoch = mirror_epoch; if (written && size <= (1u << 20)) // (EvilBoyce: GPU results written in place; small ones: the readbacks are) wb_pages_note(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo), mem_rec_value); - return remember({p->buffer, off, size, 0}); + return remember_pool(p, {p->buffer, off, size, 0}); } if (p->synced_epoch != mirror_epoch || p->sync_seg > texture_segment()) mirror_sync(*p); if (written) { @@ -3445,7 +3864,7 @@ BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { if (wb_lazy_on() && p->shadow && texture_storage_overlaps(gpu, size)) wb_pages_shared(*p, off, std::min(off + size, p->hi - p->lo)); } - return remember({p->mirror, off, size, 0}); + return remember_pool(p, {p->mirror, off, size, 0}); } // ---- mirrors given back ---- @@ -3617,7 +4036,7 @@ int64_t st_imports_evicted = 0; uint64_t st_import_bytes_evicted = 0; constexpr uint64_t kImportIdle = 600; -// ---- (relink, night 10/11) the game's freed address space given back to Windows. The game frees an address region without +// ---- (relink, night 10/11) the game's freed address space given back to the host. The game frees an address region without // unmapping its pages, the host only discarded them (their content dropped, their commit kept) and whatever lay under an import // had to stay: the regions' commit stood at their whole span, 8 GB for the ~2.9 GB mapped at a time, and the process at 29 GB // after a few areas on a 16 GB PC (vmmap.ps1, hs491). Here, under pools_mu, where every import is made (the background thread's, @@ -3627,7 +4046,6 @@ constexpr uint64_t kImportIdle = 600; int64_t st_released_calls = 0; uint64_t st_released_bytes = 0; void release_uncovered(uint64_t lo, uint64_t hi) { // (pools_mu held) -#ifdef _WIN32 static const bool off = getenv("RX_VK_KEEP_FREED") || getenv("RX_VK_BG_IMPORT") || (getenv("RX_VK_PREFETCH_CHUNKS") && atoi(getenv("RX_VK_PREFETCH_CHUNKS")) > 0); if (off || !host->region_release_gaps || hi <= lo) return; @@ -3635,13 +4053,32 @@ void release_uncovered(uint64_t lo, uint64_t hi) { // (pools_mu held) auto note = [&](uint64_t a, uint64_t b) { if (a < hi && b > lo) held.push_back({std::max(a, lo), std::min(b, hi)}); }; + auto note_pool = [&](const Pool& p) { +#ifdef __linux__ + if (p.sparse_guest && p.sparse_align) { + const uint64_t first = std::max(lo, p.lo) > p.lo ? (std::max(lo, p.lo) - p.lo) / p.sparse_align : 0; + const uint64_t end = std::min(hi, p.hi) > p.lo ? std::min(hi, p.hi) - p.lo : 0; + const uint64_t last = end / p.sparse_align + (end % p.sparse_align != 0); + for (uint64_t block = first; block < last;) { + while (block < last && !sparse_guest::bound(p.sparse_bound, block)) block++; + const uint64_t begin = block; + while (block < last && sparse_guest::bound(p.sparse_bound, block)) block++; + if (block > begin) + note(p.lo + begin * p.sparse_align, + std::min(p.hi, p.lo + block * p.sparse_align)); + } + return; + } +#endif + note(p.lo, p.hi); + }; for (auto it = imported.lower_bound(hi); it != imported.begin();) { // (the pools starting below hi, the last ones first) --it; if (it->second.hi + (1ull << 30) < lo) break; // (imports are at most 1 GB: none further down reaches lo) - note(it->second.lo, it->second.hi); + note_pool(it->second); } for (const auto& nh : retired_pools) - if (nh.mapped().memory) note(nh.mapped().lo, nh.mapped().hi); + if (nh.mapped().memory) note_pool(nh.mapped()); for (const DeadImport& d : dead_imports) note(d.lo, d.lo + d.bytes); for (const auto& nh : evicted_pools) note(nh.mapped().lo, nh.mapped().hi); std::sort(held.begin(), held.end()); @@ -3655,12 +4092,9 @@ void release_uncovered(uint64_t lo, uint64_t hi) { // (pools_mu held) static uint64_t logged_at; if (st_released_bytes >> 30 != logged_at) { // (said at every GB) logged_at = st_released_bytes >> 30; - logf("vk: the game's freed memory given back to Windows: %llu MB in all, over %lld freed regions and imports", + logf("vk: the game's freed memory given back to the host: %llu MB in all, over %lld freed regions and imports", (unsigned long long)(st_released_bytes >> 20), (long long)st_released_calls); } -#else - (void)lo, (void)hi; -#endif } // (the game's thread only leaves the range: the replay holds pools_mu for much of a submission, and the game frees regions // by the thousand while it loads. Taken up at the next submission; a range handed out again meanwhile is no gap by then) @@ -3668,6 +4102,11 @@ std::mutex released_mu; std::vector> released_queue; void memory_guest_release(uint64_t lo, uint64_t hi) { if (hi <= lo) return; +#ifdef __linux__ + // A release can remove memfd blocks while another thread allocates the same number elsewhere. The total st_blocks + // would then be unchanged, so make the next submission scan even when that cheap aggregate happens to match. + dmabuf_seen_blocks.store(-1, std::memory_order_relaxed); +#endif std::lock_guard lk(released_mu); if (!released_queue.empty() && released_queue.back().second == lo) released_queue.back().second = hi; // (the next one up) else if (!released_queue.empty() && released_queue.back().first == hi) released_queue.back().first = lo; // (or down) @@ -3682,7 +4121,46 @@ void released_drain() { // (pools_mu held) take.assign(released_queue.begin(), released_queue.begin() + n); released_queue.erase(released_queue.begin(), released_queue.begin() + n); } - for (const auto& [lo, hi] : take) release_uncovered(lo, hi); + std::vector> gaps = take; +#ifdef __linux__ + // Unbind every affected buffer in one sparse submission. release_uncovered then sees the cleared bound bits and can + // let the host punch those pages out. An allocation can contain runs from earlier releases too, so add all its runs. + std::vector plans; + for (auto& kv : imported) { + Pool& p = kv.second; + bool overlaps = false; + for (const auto& [lo, hi] : take) + if (p.lo < hi && p.hi > lo) { + overlaps = true; + break; + } + if (!overlaps) continue; + SparseGuestRelease plan; + sparse_guest_release_plan(p, take, plan); + if (!plan.allocations.empty()) plans.push_back(std::move(plan)); + } + if (!plans.empty()) { + std::vector buffers; + size_t extents = 0; + for (const SparseGuestRelease& plan : plans) { + buffers.push_back({plan.pool->buffer, (uint32_t)plan.binds.size(), plan.binds.data()}); + extents += plan.binds.size(); + } + const VkResult res = bind_sparse_buffers(buffers, true); + if (res == VK_SUCCESS) { + for (SparseGuestRelease& plan : plans) sparse_guest_release_finish(plan, gaps); + } else if (dmabuf_failures++ < 8) { + logf("vk: %zu sparse guest extents in %zu pools not released (VkResult %d)", extents, plans.size(), (int)res); + } + } +#endif + std::sort(gaps.begin(), gaps.end()); + std::vector> merged; + for (const auto& gap : gaps) { + if (!merged.empty() && gap.first <= merged.back().second) merged.back().second = std::max(merged.back().second, gap.second); + else merged.push_back(gap); + } + for (const auto& [lo, hi] : merged) release_uncovered(lo, hi); } void imports_give_back() { // (pools_mu held) @@ -3742,7 +4220,7 @@ void imports_give_back() { // (pools_mu held) Pool& p = it->second; guest_pages_set(p.lo, p.hi, 0); p.retired_value = mem_rec_value; // (freed once this submission's recording is done on the GPU) - bytes += p.hi - p.lo; + bytes += pool_imported_bytes(p); evicted_pools.push_back(imported.extract(it)); } pools_gen++; @@ -3785,7 +4263,7 @@ void imports_give_back() { // (pools_mu held) Pool& p = it->second; guest_pages_set(p.lo, p.hi, 0); p.retired_value = mem_rec_value; // (this submission's recording is being made: freed once it is done) - freed += p.hi - p.lo; + freed += pool_imported_bytes(p); n++; evicted_pools.push_back(imported.extract(it)); } @@ -3952,6 +4430,9 @@ void memory_begin_submission() { writebacks.clear(); imports_give_back(); released_drain(); +#ifdef __linux__ + sparse_guest_refresh_all(); +#endif if (tex_capture_on()) publish_tex_spans(texture_frame_no()); if (!mirror_on()) return; mirrors_trim(); diff --git a/gpu/vk/vk_texture.cpp b/gpu/vk/vk_texture.cpp index 5d5be1c..75e3929 100644 --- a/gpu/vk/vk_texture.cpp +++ b/gpu/vk/vk_texture.cpp @@ -17,6 +17,7 @@ #include #include #if defined(__linux__) +#include #include #endif #include @@ -49,8 +50,10 @@ #include "host_shaders/bc_encoder_comp_spv.h" #include "bc_levels.h" #include "astc_bc_policy.h" +#include "texture_memory_policy.h" #include "keep_rendered.h" #include "target_overlap.h" +#include "../../host/sparse_file_ranges.h" #include "host_shaders/block_linear_unswizzle_2d_comp_spv.h" #include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) @@ -62,6 +65,15 @@ uint64_t mix(uint64_t h, uint64_t v) { return h * 0xff51afd7ed558ccdull; } +uint64_t texture_policy_allowance(uint64_t driver_allowed) { + static const uint64_t configured = [] { + const char* e = getenv("RX_VK_VRAM_MB"); + return e ? (uint64_t)std::max(0, atoi(e)) << 20 : 0; + }(); + return texture_memory_policy::effective_allowance( + driver_allowed, dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU, configured); +} + // ---- formats ---- std::mutex fmt_mu; std::unordered_map fmt_cache; @@ -1262,6 +1274,19 @@ bool upload_all_zero(Texture& tex, uint64_t total, const Ranges* dirty, bool wid if (!p || avail < total) return false; const auto t0 = std::chrono::steady_clock::now(); std::atomic found{false}; +#if defined(__linux__) + // The native Linux guest window is a sparse memfd. Fresh render targets + // can be hundreds of megabytes of holes: they already read as zero, while + // the byte scan below would fault every page into shmem merely to prove it. + typedef int (*MemfdFn)(void); + static const MemfdFn memfd_fn = (MemfdFn)dlsym(RTLD_DEFAULT, "hos_posix_guest_memfd"); + const int memfd = memfd_fn ? memfd_fn() : -1; + const uint64_t guest = host->gpu_guest(tex.gpu); + const bool hole = guest && guest < host->guest_size() && total <= host->guest_size() - guest && + hos_sparse_file_has_data(memfd, guest, guest + total) == 0; +#else + const bool hole = false; +#endif const uint32_t nthreads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u); const uint64_t per = ((total / nthreads) + 4095) & ~4095ull; auto part = [&](uint32_t t) { @@ -1279,7 +1304,9 @@ bool upload_all_zero(Texture& tex, uint64_t total, const Ranges* dirty, bool wid if (acc) found.store(true, std::memory_order_relaxed); } }; - if (nthreads > 1) { + if (hole) { + // Nothing to read: sparse-file holes are defined to contain zeroes. + } else if (nthreads > 1) { std::vector th; for (uint32_t t = 1; t < nthreads; t++) th.emplace_back(part, t); part(0); @@ -1306,8 +1333,8 @@ bool upload_all_zero(Texture& tex, uint64_t total, const Ranges* dirty, bool wid vkCmdClearColorImage(upload_cmd(), tex.image, VK_IMAGE_LAYOUT_GENERAL, &z, 1, &range); } if (told.fetch_add(1) < 4) - logf("vk: upload: %ux%u NVN %#x (%llu MB) is all zero: cleared on the GPU (checked in %.1f ms)", tex.width, tex.height, - tex.nvn_format, (unsigned long long)(total >> 20), ms); + logf("vk: upload: %ux%u NVN %#x (%llu MB) is all zero: cleared on the GPU (checked in %.1f ms%s)", tex.width, tex.height, + tex.nvn_format, (unsigned long long)(total >> 20), ms, hole ? ", sparse hole" : ""); return true; } bool upload(Texture& tex, const Ranges* dirty) { @@ -1756,8 +1783,12 @@ Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw static const bool all = [] { if (const char* e = getenv("RX_VK_ASTC_BC_ALL")) return e[0] == '1'; uint64_t used = 0, allowed = 0; - const bool small = vram_usage(&used, &allowed) && astc_bc_policy::all_by_vram(allowed); - if (small) logf("vk: %llu MB of VRAM allowed: all large ASTC textures kept as BC3", (unsigned long long)(allowed >> 20)); + const bool known = vram_usage(&used, &allowed); + const uint64_t effective = texture_policy_allowance(known ? allowed : 0); + const bool small = astc_bc_policy::all_by_vram(effective); + if (small) + logf("vk: %llu MB effective texture allowance (%llu MB driver): all large ASTC textures kept as BC3", + (unsigned long long)(effective >> 20), (unsigned long long)(allowed >> 20)); return small; }(); const uint64_t min_bytes = (bw * bh >= 36 || all) ? low : 64ull << 20; @@ -3528,7 +3559,8 @@ void texture_frame(uint64_t completed) { static const uint64_t keep = []() -> uint64_t { if (const char* e = getenv("RX_VK_STAGING_KEEP_MB")) return (uint64_t)atoi(e) << 20; uint64_t used = 0, allowed = 0; - return (vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20) ? 64ull : 256ull) << 20; + const bool known = vram_usage(&used, &allowed); + return texture_memory_policy::staging_keep(texture_policy_allowance(known ? allowed : 0)); }(); uint64_t kept = 0; for (auto it = arenas.begin(); it != arenas.end();) { diff --git a/host/hos.h b/host/hos.h index 3ebd84f..5a79a97 100644 --- a/host/hos.h +++ b/host/hos.h @@ -83,6 +83,7 @@ uint64_t hos_sys_alloc(uint64_t size, uint64_t align); /* zeroed, from the s uint64_t hos_region_alloc(uint64_t size, uint64_t align); /* address space above the heap */ void hos_region_free(uint64_t addr); void hos_mem_init(uint64_t image_end); +void hos_zero_guest_pages(void *p, size_t n); /* zero allocated extents, preserving sparse memfd holes */ extern uint64_t hos_heap_size, hos_heap_used, hos_total_memory, hos_program_size; /* ---- threads (hos_thread.c) */ diff --git a/host/hos_main.c b/host/hos_main.c index c7fff85..aa9dc81 100644 --- a/host/hos_main.c +++ b/host/hos_main.c @@ -34,6 +34,9 @@ #include #include #endif +#ifdef __linux__ +#include "sparse_file_ranges.h" +#endif /* (linux) the separator of the paths the executable's folder is joined with */ #ifdef _WIN32 #define SEP 92 @@ -227,6 +230,17 @@ int hos_posix_guest_memfd(void) { return hos_guest_memfd; } int hos_posix_guest_discard(void *p, size_t n) { return madvise(p, n, hos_guest_memfd >= 0 ? MADV_REMOVE : MADV_DONTNEED); } +int hos_posix_guest_zero_allocated(void *p, size_t n) { +#ifdef __linux__ + if (hos_guest_memfd < 0 || (uint8_t *)p < A64_MEM) return -1; + const uint64_t lo = (uint64_t)((uint8_t *)p - A64_MEM); + if (lo > GUEST_SPACE || (uint64_t)n > GUEST_SPACE - lo) return -1; + return hos_sparse_file_zero_data(hos_guest_memfd, A64_MEM, lo, lo + (uint64_t)n); +#else + (void)p, (void)n; + return -1; +#endif +} #endif /* The guest window, for the renderer (gpu/render/rx_buffer.cpp). */ diff --git a/host/hos_mem.c b/host/hos_mem.c index a91b312..8a03573 100644 --- a/host/hos_mem.c +++ b/host/hos_mem.c @@ -1,6 +1,7 @@ /* Guest memory map: the system region (our own objects), the heap, address regions. The whole 4 GB window is committed up front (hos_main.c), so "mapping" here is only bookkeeping and zeroing. */ #include "hos.h" +#include "region_gaps.h" #include #include #include @@ -22,7 +23,7 @@ uint64_t hos_heap_size, hos_heap_used, hos_total_memory, hos_program_size; static uint64_t sys_next = HOS_SYSTEM_BASE; /* Address regions are handed out top-down from HOS_SPACE_END; the heap grows up from HOS_HEAP_BASE. */ -typedef struct { uint64_t addr, size; } Range; +typedef HosRegionGap Range; static Range *regions; static int nregions, capregions; static uint64_t region_floor = ~0ULL; /* the lowest region (none yet: the window's end) */ @@ -143,6 +144,14 @@ static void pages_mark_used_locked(uint64_t addr, uint64_t size) { for (uint64_t p = addr >> 12; p < (addr + size + 0xFFF) >> 12 && p < (1ULL << 20); p++) pages_used[p >> 6] |= 1ULL << (p & 63); } +void hos_zero_guest_pages(void *p, size_t n) { +#if !defined(_WIN32) && !defined(__EMSCRIPTEN__) + extern int hos_posix_guest_zero_allocated(void *p, size_t n); + if (hos_posix_guest_zero_allocated(p, n) == 0) return; +#endif + memset(p, 0, n); +} + static void discard_pages(uint64_t addr, uint64_t size) { const uint64_t lo = (addr + 0xFFF) & ~0xFFFULL, hi = (addr + size) & ~0xFFFULL; static int keep = -1; @@ -165,11 +174,24 @@ static void discard_pages(uint64_t addr, uint64_t size) { extern int rx_guest_imported(uint64_t lo, uint64_t hi); /* (pages a dma-buf import holds stay where they are, zeroed: a hole punched under one would leave the GPU on the old pages and the game on new ones) */ - if (rx_guest_imported(lo, hi)) memset(GP(lo), 0, (size_t)(hi - lo)); + if (rx_guest_imported(lo, hi)) hos_zero_guest_pages(GP(lo), (size_t)(hi - lo)); else hos_posix_guest_discard(GP(lo), (size_t)(hi - lo)); #endif } +#if !defined(__EMSCRIPTEN__) +static uint64_t release_region_gap(uint64_t lo, uint64_t hi, void *opaque) { + (void)opaque; + if (hi <= lo) return 0; +#if defined(_WIN32) && !defined(__EMSCRIPTEN__) + return VirtualFree(GP(lo), (SIZE_T)(hi - lo), MEM_DECOMMIT) ? hi - lo : 0; +#else + extern int hos_posix_guest_discard(void *p, size_t n); /* (hos_main.c: MADV_REMOVE on the memfd window) */ + return hos_posix_guest_discard(GP(lo), (size_t)(hi - lo)) == 0 ? hi - lo : 0; +#endif +} +#endif + /* (relink, night 10/11) The parts of [lo, hi) that no region holds now, in the address regions' part of the window, decommitted: what the game mapped in regions it has freed since. The game frees a region without unmapping its pages first (20.9 GB mapped and 52 MB unmapped over six areas, hs493), and a freed region's pages were only discarded: the regions' @@ -178,25 +200,14 @@ static void discard_pages(uint64_t addr, uint64_t size) { made meanwhile; a region made since is not a gap, so a range freed and handed out again keeps its pages. Returns the bytes given back */ uint64_t hos_region_release_gaps(uint64_t lo, uint64_t hi) { -#if defined(_WIN32) && !defined(__EMSCRIPTEN__) +#if !defined(__EMSCRIPTEN__) uint64_t done = 0; LOCK(); const uint64_t floor = region_floor < HOS_SPACE_END ? region_floor : HOS_SPACE_END; if (lo < floor) lo = floor; if (hi > HOS_SPACE_END) hi = HOS_SPACE_END; lo = (lo + 0xFFF) & ~0xFFFULL, hi &= ~0xFFFULL; - /* (the regions are sorted by address, descending: the gaps of [lo, hi) from its top down) */ - uint64_t top = hi; - for (int i = 0; i < nregions && top > lo; i++) { - const uint64_t a = regions[i].addr, b = (a + regions[i].size + 0xFFF) & ~0xFFFULL; - if (a >= top) continue; /* (above what is left of the range) */ - if (b < top) { - const uint64_t g = b > lo ? b : lo; - if (top > g && VirtualFree(GP(g), (SIZE_T)(top - g), MEM_DECOMMIT)) done += top - g; - } - top = a & ~0xFFFULL; - } - if (top > lo && VirtualFree(GP(lo), (SIZE_T)(top - lo), MEM_DECOMMIT)) done += top - lo; + done = hos_region_gaps_release(regions, (size_t)nregions, lo, hi, release_region_gap, NULL); UNLOCK(); return done; #else @@ -366,7 +377,7 @@ HLE(_ZN2nn2os15FreeMemoryPagesEmm) { region_count(®ion_mapped, -(int64_t)ARG(1)), region_count(®ion_freed, (int64_t)ARG(1)); region_note(); if (hi > lo && rx_guest_imported(lo, hi)) { - memset(GP(lo), 0, (size_t)(hi - lo)); + hos_zero_guest_pages(GP(lo), (size_t)(hi - lo)); region_count(®ion_kept, (int64_t)(hi - lo)); } #if defined(_WIN32) diff --git a/host/hos_svc.c b/host/hos_svc.c index b57b87d..4d060f9 100644 --- a/host/hos_svc.c +++ b/host/hos_svc.c @@ -464,7 +464,7 @@ void a64_svc(Ctx *c, uint32_t imm) { case 0x2D: /* UnmapPhysicalMemory(address, size): the pages read as zero if mapped again */ { /* (pages under a Vulkan import stay committed, zeroed: hos_mem.c FreeMemoryPages) */ extern int rx_guest_imported(uint64_t lo, uint64_t hi); - if (rx_guest_imported(c->x[0], c->x[0] + c->x[1])) memset(GP(c->x[0]), 0, (size_t)c->x[1]); + if (rx_guest_imported(c->x[0], c->x[0] + c->x[1])) hos_zero_guest_pages(GP(c->x[0]), (size_t)c->x[1]); else VirtualFree(GP(c->x[0]), (SIZE_T)c->x[1], MEM_DECOMMIT); } c->x[0] = R_OK; diff --git a/host/names_native.c b/host/names_native.c index d08e7ce..c0aef79 100644 --- a/host/names_native.c +++ b/host/names_native.c @@ -18,9 +18,17 @@ #include #include +static inline void names_checkpoint(Ctx *c) { + /* The helper replaces lifted loop back edges, including their A64_POLL. Keep the common path to one relaxed load + and preserve the poll's clear-before-yield order when another guest thread has asked for this virtual core. */ + if (__builtin_expect(__atomic_load_n(&c->poll, __ATOMIC_RELAXED) != 0, 0)) { + __atomic_store_n(&c->poll, 0, __ATOMIC_RELAXED); + hos_vcore_checkpoint(); + } +} + static inline uint64_t ld64(uint64_t a) { uint64_t v; memcpy(&v, GP(a), 8); return v; } static inline uint32_t ld32(uint64_t a) { uint32_t v; memcpy(&v, GP(a), 4); return v; } -static inline uint32_t ld8(uint64_t a) { return *(const uint8_t *)GP(a); } typedef struct { uint64_t x[32]; @@ -34,20 +42,37 @@ typedef struct { /* The game's inlined strcmp (0x805afc64.. / 0x805b1e24..), from x8 = the element's key e and x9 = the name t: what it leaves in x8 (the result; the key's low word before any byte differs), x9-x14. 0: done (the latch); 1: no end in 32 MB (left to the lifted code) */ -static int str_cmp(St *s, uint64_t e, uint64_t t, uint32_t neg) { +static int str_cmp(Ctx *c, St *s, uint64_t e, uint64_t t, uint32_t neg) { uint64_t x9 = t + 3, x10 = e + 3, x11 = 0x400000; + const uint8_t *const pe = (const uint8_t *)GP(e); + const uint8_t *const pt = (const uint8_t *)GP(t); + uint64_t at = 0; uint32_t w8 = (uint32_t)e, w12 = 0, w13 = 0, w14 = 0; for (;;) { - for (int64_t j = -3; j <= 4; j++) { - w12 = ld8(x10 + (uint64_t)j); - if (w12 == 0) w8 = 0; /* (its handler: w8 = 0, then the same compare) */ - w13 = ld8(x9 + (uint64_t)j); - w14 = w12 >= w13 ? 0u - neg : neg; - if (w12 != w13) w8 = w14; - if (w12 == 0 || w12 != w13) goto out; + const uintptr_t ea = (uintptr_t)(pe + at), ta = (uintptr_t)(pt + at); + int inspect = 1; + /* Do not read beyond a terminator into an uncommitted guest page. The 4 KB boundary is conservative on hosts with + larger pages; aligned guest mappings make it a real boundary wherever the host can protect pages individually. */ + if ((ea & 4095) <= 4088 && (ta & 4095) <= 4088) { + uint64_t ew, tw; + memcpy(&ew, (const void *)ea, sizeof ew); + memcpy(&tw, (const void *)ta, sizeof tw); + const uint64_t zero = (ew - UINT64_C(0x0101010101010101)) & ~ew & UINT64_C(0x8080808080808080); + inspect = ew != tw || zero; + } + if (inspect) { + for (uint64_t j = 0; j < 8; j++) { + w12 = pe[at + j]; + if (w12 == 0) w8 = 0; /* (its handler: w8 = 0, then the same compare) */ + w13 = pt[at + j]; + w14 = w12 >= w13 ? 0u - neg : neg; + if (w12 != w13) w8 = w14; + if (w12 == 0 || w12 != w13) goto out; + } } - x11 -= 8, x9 += 8, x10 += 8; + x11 -= 8, x9 += 8, x10 += 8, at += 8; if (x11 == 0) return 1; + names_checkpoint(c); } out: s->x[8] = w8, s->x[9] = x9, s->x[10] = x10, s->x[11] = x11, s->x[12] = w12, s->x[13] = w13, s->x[14] = w14; @@ -81,14 +106,14 @@ static int element(const uint64_t *x, uint64_t mid, uint64_t vt_a, uint64_t *n0, } /* One iteration of the first loop, from 0x805afbd4: -1 left to the lifted code (s unchanged), 0 back at its head, 1 out */ -static int step_lower(St *s) { +static __attribute__((always_inline)) inline int step_lower(Ctx *c, St *s) { uint64_t *x = s->x; const uint64_t mid = (uint32_t)(x[25] + x[26]) >> 1; uint64_t n0, n1, n2, key, tgt; if (element(x, mid, x[7], &n0, &n1, &n2, &key, &tgt, (uint32_t)x[22])) return -1; - St t = *s; - if (str_cmp(&t, key, tgt, (uint32_t)x[27])) return -1; - *s = t; + /* str_cmp commits x8..x14 only at its normal exit; its too-long fallback leaves s untouched. Calling it on s avoids + copying this 288-byte register snapshot twice for every binary-search step. */ + if (str_cmp(c, s, key, tgt, (uint32_t)x[27])) return -1; x[0] = n0, x[1] = n1, x[2] = n2, x[28] = mid; s->slot_type = (uint32_t)x[22], s->slot_key = key, s->slots = 1; const int32_t r = (int32_t)(uint32_t)x[8]; /* (0x805afbc0) */ @@ -99,7 +124,7 @@ static int step_lower(St *s) { } /* One iteration of the second loop: from 0x805b1d8c (at_head) or from 0x805b1d98 (its first, without the bounds' check) */ -static int step_upper(St *s, int at_head) { +static __attribute__((always_inline)) inline int step_upper(Ctx *c, St *s, int at_head) { uint64_t *x = s->x; if (at_head) { x[3] = (uint32_t)x[26]; @@ -109,9 +134,7 @@ static int step_upper(St *s, int at_head) { const uint64_t mid = (uint32_t)(x[23] + x[3]) >> 1; uint64_t n0, n1, n2, key, tgt; if (element(x, mid, x[5], &n0, &n1, &n2, &key, &tgt, (uint32_t)x[7])) return -1; - St t = *s; - if (str_cmp(&t, key, tgt, (uint32_t)x[22])) return -1; - *s = t; + if (str_cmp(c, s, key, tgt, (uint32_t)x[22])) return -1; x[0] = n0, x[1] = n1, x[2] = n2, x[26] = mid; s->slot_type = (uint32_t)x[7], s->slot_key = key, s->slots = 1; const int32_t r = (int32_t)(uint32_t)x[8]; /* (0x805b1f64) */ @@ -148,13 +171,31 @@ static int names_mode(void) { static volatile long st_heads, st_steps, st_left; static void count(long steps, int left) { - __atomic_add_fetch(&st_steps, steps, __ATOMIC_RELAXED); - if (left) __atomic_add_fetch(&st_left, 1, __ATOMIC_RELAXED); - if (__atomic_add_fetch(&st_heads, 1, __ATOMIC_RELAXED) % 100000 == 0) - hos_log("names: %ld searches entered: %ld iterations native in all, %ld left to the lifted code", (long)st_heads, - (long)st_steps, (long)st_left); + /* These are diagnostics, not synchronization. OptimizeChangeBanc can enter this helper tens of millions of times + while an area loads; three global atomic read-modify-writes per search made the measurement part of the stall. + Accumulate on each guest thread and publish only once per million searches. */ + static __thread long local_heads, local_steps, local_left; + local_heads++, local_steps += steps, local_left += left != 0; + if (local_heads < 1000000) return; + const long batch_heads = local_heads, batch_steps = local_steps, batch_left = local_left; + local_heads = local_steps = local_left = 0; + __atomic_add_fetch(&st_steps, batch_steps, __ATOMIC_RELAXED); + __atomic_add_fetch(&st_left, batch_left, __ATOMIC_RELAXED); + const long heads = __atomic_add_fetch(&st_heads, batch_heads, __ATOMIC_RELAXED); + if (heads / 10000000 != (heads - batch_heads) / 10000000) + hos_log("names: %ld searches entered: %ld iterations native in all, %ld left to the lifted code", heads, + __atomic_load_n(&st_steps, __ATOMIC_RELAXED), __atomic_load_n(&st_left, __ATOMIC_RELAXED)); } +#ifdef HOS_NAMES_TEST +int hos_names_test_str_cmp(Ctx *c, uint64_t element, uint64_t target, uint32_t neg, uint64_t out[7]) { + St s = {0}; + const int r = str_cmp(c, &s, element, target, neg); + memcpy(out, &s.x[8], 7 * sizeof out[0]); + return r; +} +#endif + /* HOS_NAMES_NATIVE=2: one prediction a loop and guest thread, checked where the lifted loop goes next */ enum { TO_HEAD = 0, TO_OUT = 1 }; typedef struct { int pending, to; St s; } Pred; @@ -186,12 +227,13 @@ static void check(Ctx *c, int which, int reached) { static void predict(Ctx *c, int which, int at_head) { Pred *p = &pred[which]; load(&p->s, c); - const int r = which ? step_upper(&p->s, at_head) : step_lower(&p->s); + const int r = which ? step_upper(c, &p->s, at_head) : step_lower(c, &p->s); p->pending = r >= 0, p->to = r == 1 ? TO_OUT : TO_HEAD; } /* (the first loop's head, 0x805afbd4) 1: the loop ended natively, out at 0x805afe30 */ int hos_names_lower(Ctx *c) { + names_checkpoint(c); const int m = names_mode(); if (m == 0) return 0; if (m == 2) { @@ -203,7 +245,10 @@ int hos_names_lower(Ctx *c) { load(&s, c); long steps = 0; int r; - while ((r = step_lower(&s)) == 0) steps++; + while ((r = step_lower(c, &s)) == 0) { + steps++; + names_checkpoint(c); + } if (r == 1) steps++; commit(c, &s); count(steps, r < 0); @@ -212,6 +257,7 @@ int hos_names_lower(Ctx *c) { /* (the second loop: at_head 1 at 0x805b1d8c, 0 at 0x805b1d98) 1: ended natively, out at 0x805b0200 */ int hos_names_upper(Ctx *c, int at_head) { + names_checkpoint(c); const int m = names_mode(); if (m == 0) return 0; if (m == 2) { @@ -222,8 +268,12 @@ int hos_names_upper(Ctx *c, int at_head) { St s; load(&s, c); long steps = 0; - int r = step_upper(&s, at_head); - while (r == 0) steps++, r = step_upper(&s, 1); + int r = step_upper(c, &s, at_head); + while (r == 0) { + steps++; + names_checkpoint(c); + r = step_upper(c, &s, 1); + } commit(c, &s); count(steps, r < 0); return r == 1; diff --git a/host/nn_video.c b/host/nn_video.c index 09cc5cb..2eb087f 100644 --- a/host/nn_video.c +++ b/host/nn_video.c @@ -305,7 +305,13 @@ static void dec_finalize(Ctx *c) { static void queue_frame(Dec *d, const AVFrame *f) { const int w = f->width, h = f->height; if (w <= 0 || h <= 0) return; - if (f->format != AV_PIX_FMT_NV12 && (!d->sws || d->sws_w != w || d->sws_h != h || d->sws_fmt != f->format)) { + /* Media Foundation returns NV12. Linux's dav1d returns planar 4:2:0, + which is just as suitable for the NV12 path below and is what OoT + reads. Keep RGBA lazy for both instead of allocating and converting + an otherwise unused 8.3 MB 1080p image for every decoded frame. */ + const int direct_nv12 = f->format == AV_PIX_FMT_NV12 || f->format == AV_PIX_FMT_YUV420P || + f->format == AV_PIX_FMT_YUVA420P; + if (!direct_nv12 && (!d->sws || d->sws_w != w || d->sws_h != h || d->sws_fmt != f->format)) { if (d->sws) sws_freeContext(d->sws); d->sws = sws_getContext(w, h, (enum AVPixelFormat)f->format, w, h, AV_PIX_FMT_RGBA, SWS_BILINEAR, NULL, NULL, NULL); d->sws_w = w, d->sws_h = h, d->sws_fmt = f->format; @@ -313,9 +319,9 @@ static void queue_frame(Dec *d, const AVFrame *f) { Picture p = {0}; p.w = w, p.h = h, p.pts = f->pts; const int nv12 = f->format == AV_PIX_FMT_NV12; /* (relink: Media Foundation's pictures, kept as NV12 alone) */ - p.rgba = nv12 ? NULL : (uint8_t *)malloc((size_t)w * h * 4); - p.nv12 = nv12 ? (uint8_t *)malloc((size_t)w * h * 3 / 2) : (uint8_t *)calloc(1, (size_t)w * h * 3 / 2); - if (!p.nv12 || (!nv12 && !p.rgba)) { /* (out of memory: this frame is dropped, not written through NULL; issue 342) */ + p.rgba = direct_nv12 ? NULL : (uint8_t *)malloc((size_t)w * h * 4); + p.nv12 = direct_nv12 ? (uint8_t *)malloc((size_t)w * h * 3 / 2) : (uint8_t *)calloc(1, (size_t)w * h * 3 / 2); + if (!p.nv12 || (!direct_nv12 && !p.rgba)) { /* (out of memory: this frame is dropped, not written through NULL; issue 342) */ static volatile LONG told; if (InterlockedIncrement(&told) <= 4) hos_log("video: no memory for a %dx%d frame: dropped", w, h); free(p.rgba), free(p.nv12); @@ -535,10 +541,19 @@ static void dec_unregister(Ctx *c) { RET(0); } /* AcquireXDecoderOutputBuffer(u32* index, NativeSyncHandle*, decoder): the oldest picture into a free buffer */ +static void dec_acquire_return(Ctx *c, int away, uint64_t result) { + if (away) hos_vcore_back(); + RET(result); +} static void dec_acquire(Ctx *c) { + /* The console's decoder is separate hardware. Here libavcodec/dav1d does + that work synchronously (including its own worker threads), so the + calling guest thread must not keep a virtual CPU core while it waits for + a frame and copies the decoded picture. */ + const int away = hos_vcore_away(); Dec *d = (Dec *)host_of(ARG(2), DEC_MAGIC); if (ARG(1)) G64(ARG(1)) = 0; - if (!d) { RET(RESULT_NO_OUTPUT); return; } + if (!d) { dec_acquire_return(c, away, RESULT_NO_OUTPUT); return; } /* (the library aborts on any failure here: the hardware call waits for a picture. Up to ~100 ms for one decoded by another thread's submission; then a free buffer as it was, so the caller is never refused) */ AcquireSRWLockExclusive(&d->lock); @@ -554,7 +569,7 @@ static void dec_acquire(Ctx *c) { if (d->slots[i] && !d->busy[i] && host_of(d->slots[i], POOL_MAGIC)) slot = i; if (slot < 0) { ReleaseSRWLockExclusive(&d->lock); - RET(RESULT_NO_OUTPUT); + dec_acquire_return(c, away, RESULT_NO_OUTPUT); return; } d->acquires++; @@ -570,7 +585,7 @@ static void dec_acquire(Ctx *c) { hos_log("video: the first buffer went out before the first picture was decoded (shown black)"); ReleaseSRWLockExclusive(&d->lock); if (ARG(0)) G32(ARG(0)) = (uint32_t)slot; - RET(0); + dec_acquire_return(c, away, 0); return; } { /* (relink) The game takes one picture a frame, and the sound plays on: under 60 fps the movie fell behind its @@ -651,7 +666,7 @@ static void dec_acquire(Ctx *c) { d->qhead = (d->qhead + 1) % MAX_QUEUE, d->qn--; ReleaseSRWLockExclusive(&d->lock); if (ARG(0)) G32(ARG(0)) = (uint32_t)slot; - RET(0); + dec_acquire_return(c, away, 0); } /* ReleaseXDecoderOutputBuffer(decoder, u32 index, NativeSyncHandle) */ static void dec_release(Ctx *c) { diff --git a/host/posix/win32_posix.c b/host/posix/win32_posix.c index 7681cf6..ad6586c 100644 --- a/host/posix/win32_posix.c +++ b/host/posix/win32_posix.c @@ -201,6 +201,9 @@ static HThread *thread_self(void) { } return self_thread; } +static int nice_of_thread_priority(int p) { + return p <= THREAD_PRIORITY_IDLE ? 19 : p <= THREAD_PRIORITY_LOWEST ? 10 : p == THREAD_PRIORITY_BELOW_NORMAL ? 5 : 0; +} #define PSEUDO_THREAD ((HANDLE)(intptr_t)-2) static HThread *thread_of(HANDLE h) { if (h == PSEUDO_THREAD) return thread_self(); @@ -219,7 +222,12 @@ static void thread_finish(DWORD code) { static void *thread_entry(void *p) { HThread *t = (HThread *)p; self_thread = t; - t->tid = (pid_t)syscall(SYS_gettid); + const pid_t tid = (pid_t)syscall(SYS_gettid); + __atomic_store_n(&t->tid, tid, __ATOMIC_SEQ_CST); + /* SetThreadPriority can run before this thread publishes its tid. The two sequentially consistent handoff + operations guarantee that either it sees tid, or this side sees priority, so a requested lowering is not lost. */ + const int nice = nice_of_thread_priority(__atomic_load_n(&t->priority, __ATOMIC_SEQ_CST)); + if (nice) setpriority(PRIO_PROCESS, (id_t)tid, nice); thread_finish(t->fn(t->arg)); return NULL; } @@ -241,8 +249,9 @@ HANDLE CreateThread(void *attr, SIZE_T stack, LPTHREAD_START_ROUTINE fn, void *a return NULL; } if (id) { - while (!t->tid) sched_yield(); - *id = (DWORD)t->tid; + pid_t tid; + while (!(tid = __atomic_load_n(&t->tid, __ATOMIC_SEQ_CST))) sched_yield(); + *id = (DWORD)tid; } return t; } @@ -251,8 +260,9 @@ DWORD GetCurrentThreadId(void) { return (DWORD)thread_self()->tid; } DWORD GetThreadId(HANDLE h) { HThread *t = thread_of(h); if (!t) return 0; - while (!t->tid && !t->done) sched_yield(); - return (DWORD)t->tid; + pid_t tid; + while (!(tid = __atomic_load_n(&t->tid, __ATOMIC_SEQ_CST)) && !t->done) sched_yield(); + return (DWORD)tid; } DWORD GetProcessId(HANDLE p) { (void)p; return (DWORD)getpid(); } HANDLE OpenThread(DWORD access, BOOL inherit, DWORD id) { @@ -272,14 +282,15 @@ BOOL DuplicateHandle(HANDLE sp, HANDLE src, HANDLE tp, HANDLE *out, DWORD access BOOL SetThreadPriority(HANDLE h, int p) { HThread *t = thread_of(h); if (!t) { last_error = ERROR_INVALID_HANDLE; return FALSE; } - t->priority = p; - const int nice = p <= THREAD_PRIORITY_IDLE ? 19 : p <= THREAD_PRIORITY_LOWEST ? 10 : p == THREAD_PRIORITY_BELOW_NORMAL ? 5 : 0; - if (nice && t->tid) setpriority(PRIO_PROCESS, (id_t)t->tid, nice); + __atomic_store_n(&t->priority, p, __ATOMIC_SEQ_CST); + const int nice = nice_of_thread_priority(p); + const pid_t tid = __atomic_load_n(&t->tid, __ATOMIC_SEQ_CST); + if (nice && tid) setpriority(PRIO_PROCESS, (id_t)tid, nice); return TRUE; } int GetThreadPriority(HANDLE h) { HThread *t = thread_of(h); - return t ? t->priority : 0; + return t ? __atomic_load_n(&t->priority, __ATOMIC_SEQ_CST) : 0; } DWORD_PTR SetThreadAffinityMask(HANDLE h, DWORD_PTR mask) { HThread *t = thread_of(h); @@ -294,13 +305,22 @@ DWORD_PTR SetThreadAffinityMask(HANDLE h, DWORD_PTR mask) { for (int i = 0; i < 64; i++) if (CPU_ISSET(i, &old)) r |= (DWORD_PTR)1 << i; return r ? r : 1; } +static cpu_set_t initial_process_affinity; +static pthread_once_t initial_process_affinity_once = PTHREAD_ONCE_INIT; +static void remember_initial_process_affinity(void) { + CPU_ZERO(&initial_process_affinity); + sched_getaffinity(0, sizeof initial_process_affinity, &initial_process_affinity); +} +/* Run before main can pin itself. sched_getaffinity(0) reports the calling thread's mask, not a process-wide mask, + so sampling it later would make each pin progressively narrow the next one. */ +static void __attribute__((constructor)) remember_launch_affinity(void) { + pthread_once(&initial_process_affinity_once, remember_initial_process_affinity); +} BOOL GetProcessAffinityMask(HANDLE p, DWORD_PTR *proc, DWORD_PTR *sys) { (void)p; - cpu_set_t s; - CPU_ZERO(&s); - sched_getaffinity(0, sizeof s, &s); + pthread_once(&initial_process_affinity_once, remember_initial_process_affinity); DWORD_PTR m = 0; - for (int i = 0; i < 64; i++) if (CPU_ISSET(i, &s)) m |= (DWORD_PTR)1 << i; + for (int i = 0; i < 64; i++) if (CPU_ISSET(i, &initial_process_affinity)) m |= (DWORD_PTR)1 << i; long n = sysconf(_SC_NPROCESSORS_CONF); *proc = m; *sys = n >= 64 ? ~(DWORD_PTR)0 : (((DWORD_PTR)1 << n) - 1); diff --git a/host/region_gaps.h b/host/region_gaps.h new file mode 100644 index 0000000..76ab744 --- /dev/null +++ b/host/region_gaps.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +typedef struct HosRegionGap { + uint64_t addr, size; +} HosRegionGap; + +typedef uint64_t (*HosRegionGapRelease)(uint64_t lo, uint64_t hi, void *opaque); + +/* Visit the parts of [lo, hi) outside every active region. Regions are sorted by descending address. */ +static inline uint64_t hos_region_gaps_release(const HosRegionGap *regions, size_t count, uint64_t lo, uint64_t hi, + HosRegionGapRelease release, void *opaque) { + if (!release || hi <= lo) return 0; + uint64_t done = 0, top = hi; + for (size_t i = 0; i < count && top > lo; i++) { + const uint64_t a = regions[i].addr; + const uint64_t b = a + regions[i].size; + if (a >= top) continue; + if (b < top) { + const uint64_t gap = b > lo ? b : lo; + if (top > gap) done += release(gap, top, opaque); + } + top = a; + } + if (top > lo) done += release(lo, top, opaque); + return done; +} diff --git a/host/rx_backend.c b/host/rx_backend.c index db3d22d..4a45a15 100644 --- a/host/rx_backend.c +++ b/host/rx_backend.c @@ -380,6 +380,11 @@ static void forward(const early_call *c) { break; } } +/* These synchronous renderer calls may wait for replay work, GPU completion, or a renderer mutex held by replay. + That work runs no guest code, so it must not keep one of Horizon's virtual cores from the next guest thread. */ +static int renderer_wait_away(void) { return hos_vcore_away(); } +static void renderer_wait_back(int away) { if (away) hos_vcore_back(); } + void rx_nvn_register_texture(int id, const nvn_tex *t, const nvn_view *v) { if (!started && t) { early_call c = {0, id}; @@ -387,7 +392,9 @@ void rx_nvn_register_texture(int id, const nvn_tex *t, const nvn_view *v) { if (v) c.v = *v, c.has_view = 1; if (keep_early(&c)) return; } + const int away = renderer_wait_away(); vk ? vk->nvn_register_texture(id, t, v) : wgr_nvn_register_texture(id, t, v); + renderer_wait_back(away); } void rx_nvn_register_sampler(int id, const nvn_samp_par *p) { if (!started && p) { @@ -395,7 +402,9 @@ void rx_nvn_register_sampler(int id, const nvn_samp_par *p) { c.p = *p; if (keep_early(&c)) return; } + const int away = renderer_wait_away(); vk ? vk->nvn_register_sampler(id, p) : wgr_nvn_register_sampler(id, p); + renderer_wait_back(away); } void rx_nvn_set_texture_pool_size(int n) { if (!started) { @@ -410,7 +419,9 @@ void rx_nvn_texture_released(const nvn_tex *t) { c.t = *t; if (keep_early(&c)) return; } + const int away = renderer_wait_away(); vk ? vk->nvn_texture_released(t) : wgr_nvn_texture_released(t); + renderer_wait_back(away); } void rx_nvn_texture_created(const nvn_tex *t) { if (!started && t) { @@ -423,10 +434,16 @@ void rx_nvn_texture_created(const nvn_tex *t) { void rx_nvn_program_set(const nvn_prog *p) { if (vk && program_set_fn) program_set_fn(p); } -void rx_nvn_program_released(const nvn_prog *p) { vk ? vk->nvn_program_released(p) : wgr_nvn_program_released(p); } +void rx_nvn_program_released(const nvn_prog *p) { + const int away = renderer_wait_away(); + vk ? vk->nvn_program_released(p) : wgr_nvn_program_released(p); + renderer_wait_back(away); +} void rx_nvn_read_texels(const nvn_tex *t, unsigned level, unsigned layer0, const NVNcopyRegion *r, uint8_t *dst, size_t row, size_t img) { + const int away = renderer_wait_away(); vk ? vk->nvn_read_texels(t, level, layer0, r, dst, row, img) : wgr_nvn_read_texels(t, level, layer0, r, dst, row, img); + renderer_wait_back(away); } void rx_nvn_write_texels(const nvn_tex *t, unsigned level, unsigned layer0, const NVNcopyRegion *r, const uint8_t *src, size_t row, size_t img) { @@ -448,8 +465,10 @@ void rx_nvn_write_texels(const nvn_tex *t, unsigned level, unsigned layer0, cons } } } + const int away = renderer_wait_away(); vk ? vk->nvn_write_texels(t, level, layer0, r, src, row, img) : wgr_nvn_write_texels(t, level, layer0, r, src, row, img); + renderer_wait_back(away); } void rx_nvn_invalidate(uint64_t gpu, uint64_t size) { /* The callback can wait for replay or GPU write-back: let another guest thread use this core meanwhile. */ diff --git a/host/sparse_file_ranges.h b/host/sparse_file_ranges.h new file mode 100644 index 0000000..2fee794 --- /dev/null +++ b/host/sparse_file_ranges.h @@ -0,0 +1,63 @@ +#pragma once + +#if defined(__linux__) +#include +#include +#include +#include +#include +#include + +// Whether [lo, hi) contains an allocated extent in a sparse file. A hole +// reads as zero, but reading it through a shared mapping instantiates pages; +// callers that only need to recognize an all-zero hole can avoid that fault. +// Returns 1 for data, 0 for holes only, and -1 when the allocation map cannot +// be queried (the caller must then use its ordinary byte-wise path). +static inline int hos_sparse_file_has_data(int fd, uint64_t lo, uint64_t hi) { + if (fd < 0 || hi < lo || hi > (uint64_t)INT64_MAX) return -1; + if (hi == lo) return 0; + char path[64]; + const int path_n = snprintf(path, sizeof path, "/proc/self/fd/%d", fd); + if (path_n <= 0 || (size_t)path_n >= sizeof path) return -1; + const int scan = open(path, O_RDONLY | O_CLOEXEC); + if (scan < 0) return -1; + const off_t data = lseek(scan, (off_t)lo, SEEK_DATA); + const int saved = errno; + close(scan); + if (data < 0) return saved == ENXIO ? 0 : -1; + return (uint64_t)data < hi; +} + +// Zero only allocated extents of [lo, hi) in a sparse file's shared mapping. Holes already read as zero; writing +// across them would instantiate pages just as a guest region is being freed. Reopening through proc gives this scan +// an independent file offset, unlike dup(), whose offset is shared with concurrent SEEK_DATA users. +static inline int hos_sparse_file_zero_data(int fd, void *mapping, uint64_t lo, uint64_t hi) { + if (fd < 0 || !mapping || hi < lo || hi > (uint64_t)INT64_MAX) return -1; + if (hi == lo) return 0; + char path[64]; + const int path_n = snprintf(path, sizeof path, "/proc/self/fd/%d", fd); + if (path_n <= 0 || (size_t)path_n >= sizeof path) return -1; + const int scan = open(path, O_RDONLY | O_CLOEXEC); + if (scan < 0) return -1; + uint64_t at = lo; + while (at < hi) { + const off_t data = lseek(scan, (off_t)at, SEEK_DATA); + if (data < 0) { + if (errno == ENXIO) break; + close(scan); + return -1; + } + if ((uint64_t)data >= hi) break; + const off_t hole = lseek(scan, data, SEEK_HOLE); + if (hole <= data) { + close(scan); + return -1; + } + const uint64_t end = (uint64_t)hole < hi ? (uint64_t)hole : hi; + memset((uint8_t *)mapping + (uint64_t)data, 0, (size_t)(end - (uint64_t)data)); + at = end; + } + close(scan); + return 0; +} +#endif diff --git a/nx2/build.py b/nx2/build.py index 60cfb20..e96d11f 100644 --- a/nx2/build.py +++ b/nx2/build.py @@ -832,6 +832,7 @@ def main(): os.path.join(GPU, "compat")) vdeps = tree_headers(vk, os.path.join(GPU, "render"), os.path.join(GPU, "compat"), os.path.join(GPU, "third_party", "suyu", "video_core", "textures")) + sparse_file_deps = open(os.path.join(HOST, "sparse_file_ranges.h"), "rb").read() # (relink) RX_VK_PGO=gen: the back end's own files (not Eden's recompiler nor sirit) count their branches, and host/pgo_rt.c # in the DLL writes them (RX_VK_PGO_FILE=, .reset and .dump as HOS_PGO's); RX_VK_PGO=: # built with that profile. The replay on the game's render thread is the frame's limit in heavy places (Kokiri village) @@ -850,7 +851,8 @@ def main(): vflags + ([] if lib else vk_pgo_flags) + ([shader_stamp] if os.path.basename(src) == "vk_shader.cpp" else []) + ([stamp_def] if os.path.basename(src) == "vk_menu.cpp" else []) + (ngx_flags if os.path.basename(src) == "vk_dlss.cpp" else []), - vk_lib_deps if lib else vdeps + (logo_text if os.path.basename(src) == "vk_menu.cpp" else b""))) + vk_lib_deps if lib else vdeps + (logo_text if os.path.basename(src) == "vk_menu.cpp" else b"") + + (sparse_file_deps if os.path.basename(src) == "vk_texture.cpp" else b""))) if vk_pgo == "gen": # (the counters' writer, the DLL's own copy: its sections are the DLL's) vk_jobs.append((cc, os.path.join(HOST, "pgo_rt.c"), os.path.join(obj_dir, "vk_pgo_rt.o"), target() + ["-O2", "-g0", "-Wno-everything", '-DPGO_ENV="RX_VK_PGO_FILE"'], b"")) diff --git a/nx2/test_import_windows.py b/nx2/test_import_windows.py index 99f940a..a005bce 100644 --- a/nx2/test_import_windows.py +++ b/nx2/test_import_windows.py @@ -40,7 +40,8 @@ def main(): Pool *last_pool=nullptr,*pool_cache[4096]{}; uint64_t max_import_size=128ull<<20; int failures=0,imports=0,retires=0,bg_starts=0; -int64_t st_bg_waits=0,st_prefetch_used=0,st_prefetch_sent=0,st_prefetch_dropped=0; +int64_t st_bg_waits=0,st_bg_wait_ns=0,st_sync_imports=0,st_sync_import_ns=0; +int64_t st_prefetch_used=0,st_prefetch_sent=0,st_prefetch_dropped=0; struct { bool external_memory_host=true; } dev; std::mutex bg_mu;std::condition_variable bg_done_cv,bg_cv;std::vector bg_inflight; std::deque> bg_queue; diff --git a/nx2/test_names_native.py b/nx2/test_names_native.py new file mode 100644 index 0000000..26969cb --- /dev/null +++ b/nx2/test_names_native.py @@ -0,0 +1,149 @@ +"""Test native name-search comparison state and guest scheduler checkpoints. + +No game data, guest memory, renderer, or platform thread is used. + +python nx2/test_names_native.py [--cc clang] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + fixture = r''' +#include +#include +#include "host/hos.h" +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); exit(1); } } while (0) + +static int checkpoints; +void hos_vcore_checkpoint(void) { checkpoints++; } +void hos_log(const char *fmt, ...) { (void)fmt; } + +int hos_names_lower(Ctx *c); +int hos_names_upper(Ctx *c, int at_head); +int hos_names_test_str_cmp(Ctx *c, uint64_t element, uint64_t target, uint32_t neg, uint64_t out[7]); + +int main(void) { + Ctx c = {0}; + c.poll = 1; + CHECK(hos_names_lower(&c) == 0); + CHECK(c.poll == 0); + CHECK(checkpoints == 1); + CHECK(hos_names_lower(&c) == 0); + CHECK(checkpoints == 1); + c.poll = 1; + CHECK(hos_names_upper(&c, 1) == 0); + CHECK(c.poll == 0); + CHECK(checkpoints == 2); + CHECK(hos_names_upper(&c, 0) == 0); + CHECK(checkpoints == 2); + + uint64_t out[7] = {0}; + char equal_a[] = "abc", equal_b[] = "abc"; + CHECK(hos_names_test_str_cmp(&c, (uint64_t)(uintptr_t)equal_a, (uint64_t)(uintptr_t)equal_b, 1, out) == 0); + CHECK((uint32_t)out[0] == 0); + CHECK(out[1] == (uint64_t)(uintptr_t)equal_b + 3); + CHECK(out[2] == (uint64_t)(uintptr_t)equal_a + 3); + CHECK(out[3] == 0x400000); + CHECK((uint32_t)out[4] == 0 && (uint32_t)out[5] == 0 && (uint32_t)out[6] == UINT32_MAX); + + char lower[] = "abc", greater[] = "abd"; + CHECK(hos_names_test_str_cmp(&c, (uint64_t)(uintptr_t)lower, (uint64_t)(uintptr_t)greater, 1, out) == 0); + CHECK((uint32_t)out[0] == 1 && (uint32_t)out[4] == 'c' && (uint32_t)out[5] == 'd' && (uint32_t)out[6] == 1); + CHECK(hos_names_test_str_cmp(&c, (uint64_t)(uintptr_t)greater, (uint64_t)(uintptr_t)lower, 1, out) == 0); + CHECK((uint32_t)out[0] == UINT32_MAX && (uint32_t)out[4] == 'd' && (uint32_t)out[5] == 'c'); + + char long_a[] = "abcdefghX", long_b[] = "abcdefghY"; + c.poll = 1; + CHECK(hos_names_test_str_cmp(&c, (uint64_t)(uintptr_t)long_a, (uint64_t)(uintptr_t)long_b, 1, out) == 0); + CHECK(c.poll == 0 && checkpoints == 3); + CHECK(out[1] == (uint64_t)(uintptr_t)long_b + 11); + CHECK(out[2] == (uint64_t)(uintptr_t)long_a + 11); + CHECK(out[3] == 0x3ffff8); + CHECK((uint32_t)out[0] == 1 && (uint32_t)out[4] == 'X' && (uint32_t)out[5] == 'Y'); + + char scan_a[48], scan_b[48]; + for (int len = 0; len < 32; len++) { + for (int i = 0; i < len; i++) scan_a[i] = scan_b[i] = (char)('A' + i % 23); + scan_a[len] = scan_b[len] = 0; + CHECK(hos_names_test_str_cmp(&c, (uint64_t)(uintptr_t)scan_a, (uint64_t)(uintptr_t)scan_b, 1, out) == 0); + CHECK((uint32_t)out[0] == 0 && (uint32_t)out[4] == 0 && (uint32_t)out[5] == 0); + CHECK(out[1] == (uint64_t)(uintptr_t)scan_b + 3 + (uint64_t)(len / 8) * 8); + CHECK(out[2] == (uint64_t)(uintptr_t)scan_a + 3 + (uint64_t)(len / 8) * 8); + CHECK(out[3] == 0x400000 - (uint64_t)(len / 8) * 8); + for (int different = 0; different < len; different++) { + scan_b[different]++; + CHECK(hos_names_test_str_cmp(&c, (uint64_t)(uintptr_t)scan_a, (uint64_t)(uintptr_t)scan_b, 1, out) == 0); + CHECK((uint32_t)out[0] == 1); + CHECK((uint32_t)out[4] == (uint8_t)scan_a[different]); + CHECK((uint32_t)out[5] == (uint8_t)scan_b[different]); + CHECK(out[1] == (uint64_t)(uintptr_t)scan_b + 3 + (uint64_t)(different / 8) * 8); + CHECK(out[2] == (uint64_t)(uintptr_t)scan_a + 3 + (uint64_t)(different / 8) * 8); + CHECK(out[3] == 0x400000 - (uint64_t)(different / 8) * 8); + scan_b[different]--; + } + } + puts("PASS: native name-search comparisons preserve state and virtual-core preemption"); +} +''' + work = ROOT / "work" + work.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="names-native-", dir=work) as temp: + directory = Path(temp).resolve() + directory.relative_to(work.resolve()) + source = directory / "fixture.c" + executable = directory / "fixture" + # names_native includes the generated guest model through hos.h. Keep + # this scheduler test independent of a previous title lift while still + # modelling the registers and interrupt flag it touches. + (directory / "a64.h").write_text( + "#include \n" + "#include \n" + "typedef struct GuestContext {\n" + " uint64_t x[32];\n" + " uint8_t n, z, c, v;\n" + " uint32_t poll;\n" + "} GuestContext;\n" + "typedef GuestContext Ctx;\n" + "#define A64_PTR(a) ((uint8_t *)(uintptr_t)(a))\n", + encoding="utf-8", + ) + source.write_text(fixture, encoding="utf-8") + subprocess.run( + [ + args.cc, + "-std=c11", + "-O2", + "-Wall", + "-Wextra", + "-Werror", + "-DHOS_NAMES_TEST", + "-I", + str(ROOT), + "-I", + str(directory), + "-I", + str(ROOT / "third_party" / "a64-kit" / "rt"), + str(ROOT / "host" / "names_native.c"), + str(source), + "-o", + str(executable), + ], + check=True, + ) + env = os.environ.copy() + env["HOS_NAMES_NATIVE"] = "0" + subprocess.run([str(executable)], check=True, env=env) + + +if __name__ == "__main__": + main() diff --git a/nx2/test_native_cbuf_pointer_policy.py b/nx2/test_native_cbuf_pointer_policy.py new file mode 100644 index 0000000..6133d14 --- /dev/null +++ b/nx2/test_native_cbuf_pointer_policy.py @@ -0,0 +1,60 @@ +"""Test native non-c0 storage-candidate validation without a GPU or game data. + +python nx2/test_native_cbuf_pointer_policy.py [--cxx clang++] +""" +import argparse +from pathlib import Path +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + fixture = r''' +#include +#include +#include "gpu/vk/eden/shader_recompiler/ir_opt/native_cbuf_pointer_policy.h" +#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::exit(1); } } while (0) + +int main() { + using native_cbuf_pointer_policy::leave_global; + // Native vertex/task/mesh: reject an unmapped scalar; preserve a mapped pointer. + CHECK(leave_global(true, true, true, 5, false, false)); + CHECK(!leave_global(true, true, true, 5, false, true)); + // The pre-existing memory-origin rule remains active for geometry, native or not. + CHECK(leave_global(true, true, true, 5, true, true)); + CHECK(leave_global(false, true, true, 5, true, false)); + CHECK(!leave_global(false, true, true, 5, false, false)); + // Fragment rejects only the unmapped native scalar, not the older memory-origin case. + CHECK(leave_global(true, true, false, 5, false, false)); + CHECK(!leave_global(true, true, false, 5, false, true)); + CHECK(!leave_global(false, true, false, 5, true, false)); + // Compute and c0 keep their existing paths. + CHECK(!leave_global(true, false, false, 5, true, false)); + CHECK(!leave_global(true, true, true, 0, true, false)); + std::puts("PASS: native graphics rejects scalar non-c0 storage candidates without widening fragment memory-origin fallback"); +} +''' + work = ROOT / "work" + work.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="native-cbuf-pointer-", dir=work) as temp: + directory = Path(temp).resolve() + directory.relative_to(work.resolve()) + source = directory / "fixture.cpp" + executable = directory / "fixture" + source.write_text(fixture, encoding="utf-8") + subprocess.run( + [args.cxx, "-std=c++20", "-O2", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT), + str(source), "-o", str(executable)], + check=True, + ) + subprocess.run([str(executable)], check=True) + + +if __name__ == "__main__": + main() diff --git a/nx2/test_region_gaps.py b/nx2/test_region_gaps.py new file mode 100644 index 0000000..7cc9c92 --- /dev/null +++ b/nx2/test_region_gaps.py @@ -0,0 +1,74 @@ +"""Test the address-region gap planner used before Linux/Windows guest-page release. + +No game data, renderer, Vulkan device, or platform decommit call is used. + +python nx2/test_region_gaps.py [--cc clang] +""" +import argparse +from pathlib import Path +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + fixture = r''' +#include +#include +#include +#include "host/region_gaps.h" +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); exit(1); } } while (0) + +typedef struct Seen { uint64_t lo[8], hi[8]; size_t count; } Seen; + +static uint64_t note(uint64_t lo, uint64_t hi, void *opaque) { + Seen *seen = opaque; + CHECK(seen->count < 8 && hi > lo); + seen->lo[seen->count] = lo; + seen->hi[seen->count++] = hi; + return hi - lo; +} + +int main(void) { + const HosRegionGap regions[] = {{800, 100}, {500, 100}, {200, 100}}; + Seen seen = {0}; + uint64_t bytes = hos_region_gaps_release(regions, 3, 250, 850, note, &seen); + CHECK(bytes == 400 && seen.count == 2); + CHECK(seen.lo[0] == 600 && seen.hi[0] == 800); + CHECK(seen.lo[1] == 300 && seen.hi[1] == 500); + + seen = (Seen){0}; + bytes = hos_region_gaps_release(NULL, 0, 100, 200, note, &seen); + CHECK(bytes == 100 && seen.count == 1 && seen.lo[0] == 100 && seen.hi[0] == 200); + + seen = (Seen){0}; + bytes = hos_region_gaps_release(regions, 3, 520, 580, note, &seen); + CHECK(bytes == 0 && seen.count == 0); + CHECK(hos_region_gaps_release(regions, 3, 10, 10, note, &seen) == 0); + CHECK(hos_region_gaps_release(regions, 3, 10, 20, NULL, &seen) == 0); + + puts("PASS: active regions are excluded from descending address-space gaps"); +} +''' + work = ROOT / "work" + work.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="region-gaps-", dir=work) as temp: + directory = Path(temp).resolve() + directory.relative_to(work.resolve()) + source = directory / "fixture.c" + executable = directory / "fixture" + source.write_text(fixture, encoding="utf-8") + subprocess.run( + [args.cc, "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT), str(source), "-o", str(executable)], + check=True, + ) + subprocess.run([str(executable)], check=True) + + +if __name__ == "__main__": + main() diff --git a/nx2/test_sparse_file_ranges.py b/nx2/test_sparse_file_ranges.py new file mode 100644 index 0000000..af860d9 --- /dev/null +++ b/nx2/test_sparse_file_ranges.py @@ -0,0 +1,70 @@ +"""Test that freeing sparse memfd data zeros allocated extents without instantiating holes. + +No Vulkan device or game data is used. Linux memfd SEEK_DATA/SEEK_HOLE is the production mechanism. + +python nx2/test_sparse_file_ranges.py [--cc clang] +""" +import argparse +from pathlib import Path +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + fixture = r''' +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "host/sparse_file_ranges.h" +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); exit(1); } } while (0) + +int main(void) { + const size_t page = (size_t)sysconf(_SC_PAGESIZE), size = 4 * page; + const int fd = (int)syscall(SYS_memfd_create, "sparse-ranges-test", 1u); + CHECK(fd >= 0 && ftruncate(fd, (off_t)size) == 0); + uint8_t *map = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(map != MAP_FAILED); + CHECK(hos_sparse_file_has_data(fd, 0, size) == 0); + map[17] = 0xa5; + map[2 * page + 29] = 0x5a; + CHECK(hos_sparse_file_has_data(fd, 0, page) == 1); + CHECK(hos_sparse_file_has_data(fd, page, 2 * page) == 0); + CHECK(hos_sparse_file_has_data(fd, 2 * page, 3 * page) == 1); + CHECK(hos_sparse_file_has_data(-1, 0, page) == -1); + CHECK(lseek(fd, (off_t)page, SEEK_DATA) == (off_t)(2 * page)); + CHECK(hos_sparse_file_zero_data(fd, map, 0, 3 * page) == 0); + CHECK(map[17] == 0 && map[2 * page + 29] == 0); + CHECK(lseek(fd, (off_t)page, SEEK_DATA) == (off_t)(2 * page)); + CHECK(hos_sparse_file_zero_data(fd, map, page, page) == 0); + CHECK(hos_sparse_file_zero_data(-1, map, 0, page) == -1); + munmap(map, size); + close(fd); + puts("PASS: sparse guest zeroing preserves unallocated memfd holes"); +} +''' + work = ROOT / "work" + work.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="sparse-file-ranges-", dir=work) as temp: + source = Path(temp) / "fixture.c" + executable = Path(temp) / "fixture" + source.write_text(fixture, encoding="utf-8") + subprocess.run( + [args.cc, "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT), str(source), "-o", str(executable)], + check=True, + ) + subprocess.run([str(executable)], check=True) + + +if __name__ == "__main__": + main() diff --git a/nx2/test_sparse_guest_ranges.py b/nx2/test_sparse_guest_ranges.py new file mode 100644 index 0000000..44cedaa --- /dev/null +++ b/nx2/test_sparse_guest_ranges.py @@ -0,0 +1,141 @@ +"""Test sparse guest-memory block rounding, merging, accounting, and already-bound filtering. + +No Vulkan device, game data, or Linux udmabuf device is used. + +python nx2/test_sparse_guest_ranges.py [--cxx clang++] +""" +import argparse +from pathlib import Path +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + fixture = r''' +#include +#include +#include +#include "gpu/vk/sparse_guest_ranges.h" +#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::exit(1); } } while (0) + +int main() { + constexpr uint64_t base = 0x1000000000ull, block = 65536; + struct Live { uint64_t lo, hi; }; + std::vector wanted(8); + std::vector bound(1); + + std::vector merged; + CHECK(sparse_guest::append_merged(merged, {block, block})); + CHECK(sparse_guest::append_merged(merged, {2 * block, block})); + CHECK(sparse_guest::append_merged(merged, {2 * block + 1, 2 * block})); + CHECK(merged.size() == 1 && merged[0].offset == block && merged[0].size == 3 * block + 1); + CHECK(sparse_guest::append_merged(merged, {5 * block, block})); + CHECK(merged.size() == 2 && merged[1].offset == 5 * block && merged[1].size == block); + CHECK(!sparse_guest::append_merged(merged, {4 * block, block})); + CHECK(!sparse_guest::append_merged(merged, {UINT64_MAX, 2})); + + sparse_guest::mark(wanted, base, block, base + 4096, base + block + 4096); + sparse_guest::mark(wanted, base, block, base + 3 * block, base + 4 * block); + auto runs = sparse_guest::runs(wanted, bound, block); + CHECK(runs.size() == 2); + CHECK(runs[0].offset == 0 && runs[0].size == 2 * block); + CHECK(runs[1].offset == 3 * block && runs[1].size == block); + + sparse_guest::set_bound(bound, 1, 1); + runs = sparse_guest::runs(wanted, bound, block); + CHECK(runs.size() == 2); + CHECK(runs[0].offset == 0 && runs[0].size == block); + CHECK(runs[1].offset == 3 * block && runs[1].size == block); + CHECK(sparse_guest::bound(bound, 1) && !sparse_guest::bound(bound, 2)); + + sparse_guest::set_bound(bound, 0, 1); + sparse_guest::set_bound(bound, 3, 1); + CHECK(sparse_guest::runs(wanted, bound, block).empty()); + CHECK(sparse_guest::overlaps_bound(bound, base, block, base + 1, base + block)); + CHECK(!sparse_guest::overlaps_bound(bound, base, block, base + 2 * block, base + 3 * block)); + CHECK(sparse_guest::overlaps_bound(bound, base, block, base + 2 * block, base + 3 * block + 1)); + CHECK(!sparse_guest::overlaps_bound(bound, base, block, base - block, base)); + CHECK(!sparse_guest::overlaps_bound(bound, base, 0, base, base + 1)); + sparse_guest::clear_bound(bound, 3, 1); + CHECK(!sparse_guest::bound(bound, 3) && sparse_guest::bound(bound, 0)); + sparse_guest::clear_bound(bound, 99, 1); + + sparse_guest::clear_bound(bound, 1, 1); + sparse_guest::set_bound(bound, 3, 1); + auto unbound = sparse_guest::unbound_runs(bound, block, 8 * block, 1, 5 * block + 1); + CHECK(unbound.size() == 2); + CHECK(unbound[0].offset == block && unbound[0].size == 2 * block); + CHECK(unbound[1].offset == 4 * block && unbound[1].size == 2 * block); + sparse_guest::set_bound(bound, 1, 2); + sparse_guest::set_bound(bound, 4, 2); + CHECK(sparse_guest::unbound_runs(bound, block, 8 * block, 1, 5 * block + 1).empty()); + CHECK(sparse_guest::unbound_runs(bound, 0, 8 * block, 0, block).empty()); + CHECK(sparse_guest::unbound_runs(bound, block, 8 * block, 8 * block, 9 * block).empty()); + sparse_guest::clear_bound(bound, 1, 2); + sparse_guest::clear_bound(bound, 3, 1); + sparse_guest::clear_bound(bound, 4, 2); + CHECK(sparse_guest::scan_advance(base, block, base + 17, base + 4096, base + 8 * block) == base + block); + CHECK(sparse_guest::scan_advance(base, block, base + 17, base + 3 * block, base + 8 * block) == base + 3 * block); + CHECK(sparse_guest::scan_advance(base, block, base + 7 * block, base + 9 * block, base + 8 * block) == + base + 8 * block); + CHECK(sparse_guest::scan_advance(base, 0, base, base, base + block) == base + block); + + std::fill(wanted.begin(), wanted.end(), 0); + sparse_guest::mark(wanted, base, block, base + 7 * block + 1, base + 8 * block); + runs = sparse_guest::runs(wanted, bound, block); + CHECK(runs.size() == 1 && runs[0].offset == 7 * block && runs[0].size == block); + sparse_guest::mark(wanted, base, block, base, base); + sparse_guest::mark(wanted, base, 0, base, base + 1); + + const std::vector allocation{{block, block}, {4 * block, 2 * block}}; + CHECK(sparse_guest::bytes(allocation) == 3 * block); + CHECK(sparse_guest::bytes({{0, UINT64_MAX}, {0, 1}}) == UINT64_MAX); + CHECK(sparse_guest::touches(allocation, base, base + block, base + 2 * block)); + CHECK(sparse_guest::touches(allocation, base, base + 5 * block, base + 7 * block)); + CHECK(!sparse_guest::touches(allocation, base, base + 2 * block, base + 4 * block)); + CHECK(!sparse_guest::touches(allocation, base, base, base)); + CHECK(sparse_guest::all_outside(allocation, base, std::vector{{base, base + block}})); + CHECK(!sparse_guest::all_outside(allocation, base, std::vector{{base + 5 * block, base + 7 * block}})); + CHECK(!sparse_guest::all_outside(allocation, base, + std::vector{{base, base + block}, {base + block, base + 2 * block}})); + CHECK(sparse_guest::all_outside(allocation, base, + std::vector{{base, base + block}, {base + 2 * block, base + 4 * block}})); + + const auto batches = sparse_guest::batches( + std::vector{{0, 150 * block}, {200 * block, 3 * block}}, 64 * block, 2, block); + CHECK(batches.size() == 3); + CHECK(batches[0].size() == 1 && batches[0][0].offset == 0 && batches[0][0].size == 64 * block); + CHECK(batches[1].size() == 1 && batches[1][0].offset == 64 * block && batches[1][0].size == 64 * block); + CHECK(batches[2].size() == 2); + CHECK(batches[2][0].offset == 128 * block && batches[2][0].size == 22 * block); + CHECK(batches[2][1].offset == 200 * block && batches[2][1].size == 3 * block); + CHECK(sparse_guest::batches({{1, block}}, 64 * block, 2, block).empty()); + CHECK(sparse_guest::batches({{0, block}}, block - 1, 2, block).empty()); + CHECK(sparse_guest::batches({{UINT64_MAX - block + 1, block}}, 64 * block, 2, block).empty()); + + std::puts("PASS: sparse guest ranges round, merge, account, release, and exclude live blocks"); +} +''' + work = ROOT / "work" + work.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="sparse-guest-ranges-", dir=work) as temp: + directory = Path(temp).resolve() + directory.relative_to(work.resolve()) + cpp = directory / "fixture.cpp" + exe = directory / "fixture" + cpp.write_text(fixture, encoding="utf-8") + subprocess.run( + [args.cxx, "-std=c++20", "-O2", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT), str(cpp), "-o", str(exe)], + check=True, + ) + subprocess.run([str(exe)], check=True) + + +if __name__ == "__main__": + main() diff --git a/nx2/test_texture_memory_policy.py b/nx2/test_texture_memory_policy.py new file mode 100644 index 0000000..b32ebb3 --- /dev/null +++ b/nx2/test_texture_memory_policy.py @@ -0,0 +1,61 @@ +"""Test integrated/discrete texture-policy allowances without Vulkan or a device. + +python nx2/test_texture_memory_policy.py [--cxx clang++] +""" +import argparse +from pathlib import Path +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + fixture = r''' +#include +#include +#include "gpu/vk/astc_bc_policy.h" +#include "gpu/vk/texture_memory_policy.h" +#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::exit(1); } } while (0) + +int main() { + using namespace texture_memory_policy; + constexpr uint64_t raw = 9594ull * MiB; + constexpr uint64_t explicit_budget = 768ull * MiB; + CHECK(effective_allowance(raw, false) == raw); + CHECK(effective_allowance(raw, true) == IntegratedDefault); + CHECK(effective_allowance(2048ull * MiB, true) == 2048ull * MiB); + CHECK(effective_allowance(raw, false, explicit_budget) == explicit_budget); + CHECK(effective_allowance(raw, true, explicit_budget) == explicit_budget); + CHECK(effective_allowance(0, true) == IntegratedDefault); + CHECK(effective_allowance(0, false, explicit_budget) == explicit_budget); + CHECK(effective_allowance(0, false) == 0); + + CHECK(!astc_bc_policy::all_by_vram(raw)); + CHECK(astc_bc_policy::all_by_vram(effective_allowance(raw, true))); + CHECK(astc_bc_policy::all_by_vram(effective_allowance(raw, false, explicit_budget))); + CHECK(staging_keep(raw) == 256ull * MiB); + CHECK(staging_keep(effective_allowance(raw, true)) == 64ull * MiB); + CHECK(staging_keep(explicit_budget) == 64ull * MiB); + std::puts("PASS: integrated/configured texture allowance drives ASTC BC3 and 64 MB staging policies"); +} +''' + work = ROOT / "work" + work.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="texture-memory-policy-", dir=work) as temp: + source = Path(temp) / "fixture.cpp" + executable = Path(temp) / "fixture" + source.write_text(fixture, encoding="utf-8") + subprocess.run( + [args.cxx, "-std=c++20", "-O2", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT), str(source), "-o", str(executable)], + check=True, + ) + subprocess.run([str(executable)], check=True) + + +if __name__ == "__main__": + main() diff --git a/nx2/test_win32_posix_threading.py b/nx2/test_win32_posix_threading.py new file mode 100644 index 0000000..34e20be --- /dev/null +++ b/nx2/test_win32_posix_threading.py @@ -0,0 +1,112 @@ +"""Test the POSIX Win32 layer's process affinity and thread-priority handoff. + +No game data, guest memory, or renderer is used. + +python nx2/test_win32_posix_threading.py [--cc clang] +""" +import argparse +from pathlib import Path +import platform +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + if platform.machine().lower() not in {"x86_64", "amd64"}: + print("SKIP: win32_posix.c currently targets Linux/x86_64") + return + fixture = r''' +#define _GNU_SOURCE +#include + +#include +#include +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); exit(1); } } while (0) + +static DWORD record_nice(void *arg) { + *(int *)arg = getpriority(PRIO_PROCESS, (id_t)syscall(SYS_gettid)); + return 0; +} + +static DWORD_PTR mask64(const cpu_set_t *set) { + DWORD_PTR mask = 0; + for (int i = 0; i < 64; i++) + if (CPU_ISSET(i, set)) mask |= (DWORD_PTR)1 << i; + return mask; +} + +int main(void) { + cpu_set_t launch, one; + CHECK(sched_getaffinity(0, sizeof launch, &launch) == 0); + const DWORD_PTR launch_mask = mask64(&launch); + CHECK(launch_mask != 0); + + int cpu = 0; + while (cpu < 64 && !CPU_ISSET(cpu, &launch)) cpu++; + CHECK(cpu < 64); + CPU_ZERO(&one); + CPU_SET(cpu, &one); + CHECK(sched_setaffinity(0, sizeof one, &one) == 0); + + DWORD_PTR process_mask = 0, system_mask = 0; + CHECK(GetProcessAffinityMask(GetCurrentProcess(), &process_mask, &system_mask)); + CHECK(process_mask == launch_mask); + CHECK((system_mask & process_mask) == process_mask); + + /* The parent and children share one CPU, so CreateThread returns before the child can publish its tid. */ + for (int i = 0; i < 32; i++) { + int nice = -1; + HANDLE thread = CreateThread(NULL, 0, record_nice, &nice, 0, NULL); + CHECK(thread != NULL); + CHECK(SetThreadPriority(thread, THREAD_PRIORITY_LOWEST)); + CHECK(WaitForSingleObject(thread, INFINITE) == WAIT_OBJECT_0); + CHECK(nice == 10); + CHECK(CloseHandle(thread)); + } + + CHECK(sched_setaffinity(0, sizeof launch, &launch) == 0); + puts("PASS: launch affinity survives thread pinning and early priorities reach new threads"); +} +''' + work = ROOT / "work" + work.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="win32-posix-threading-", dir=work) as temp: + directory = Path(temp).resolve() + directory.relative_to(work.resolve()) + source = directory / "fixture.c" + executable = directory / "fixture" + source.write_text(fixture, encoding="utf-8") + subprocess.run( + [ + args.cc, + "-std=gnu11", + "-O2", + "-Wall", + "-Wextra", + "-I", + str(ROOT / "host" / "posix"), + str(ROOT / "host" / "posix" / "win32_posix.c"), + str(source), + "-ldl", + "-pthread", + "-o", + str(executable), + ], + check=True, + ) + subprocess.run([str(executable)], check=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/bc250/README.md b/scripts/bc250/README.md new file mode 100644 index 0000000..6360086 --- /dev/null +++ b/scripts/bc250/README.md @@ -0,0 +1,61 @@ +# BC-250 guarded launcher + +These scripts run an existing Linux Project ReLink build with an explicitly +selected RADV ICD. They do not download or contain title data. + +The launcher is intentionally conservative: + +- it asks Bazzite's installed `tuned-ppd` for the performance profile only for + the lifetime of the run, and verifies every CPU policy changed; +- the D-Bus hold is tied to its caller, so TuneD releases it when the holder is + killed as well as on an orderly exit; +- a systemd scope bounds memory, swap and elapsed time; +- a second guard stops before host free memory falls below 2 GiB or the AMD GPU + or Zen CPU reaches 90 C; +- it explicitly joins the existing Wayland session and aborts if ReLink does + not create a visible mailbox swapchain within 60 seconds; +- it returns the display to the active Wayland virtual terminal on exit; +- serial replay (`RX_VK_PARALLEL=1`) remains the correctness baseline. + +On Bazzite/Fedora, install the small runtime dependencies and prime sudo once: + +```sh +sudo dnf install python3-dbus tuned-ppd mangohud +sudo -v +``` + +Do **not** replace `tuned-ppd` with `power-profiles-daemon`; the Fedora packages +conflict, and Bazzite maps the same D-Bus API to its TuneD profiles. + +Copy the example profile somewhere writable, then give the launcher the build, +profile and development ICD paths: + +```sh +cp scripts/bc250/profile.env.example "$HOME/relink-bc250.env" + +scripts/bc250/run-relink-bc250.sh \ + "$HOME/games/relink-build" \ + "$HOME/relink-bc250.env" \ + "$HOME/mesa-directmesh/share/vulkan/icd.d/radeon_icd.x86_64.json" +``` + +Useful opt-ins are environment variables, not edits to the script: + +```sh +# Fullscreen, with MangoHud enabled by default. +BC250_FULLSCREEN=1 scripts/bc250/run-relink-bc250.sh BUILD PROFILE ICD + +# Five-minute validation route using an input script already in PROFILE. +BC250_RUNTIME_MAX_SEC=300 BC250_KEEP_INPUT_SCRIPT=1 \ + scripts/bc250/run-relink-bc250.sh BUILD PROFILE ICD +``` + +Each run writes a timestamped metrics file below `BUILD/logs/`. The defaults +are a 7.031-GiB soft limit, 8.30-GiB hard limit, 2-GiB cgroup swap limit and a +two-hour time limit. They can be changed with the documented `BC250_*_BYTES` +and `BC250_RUNTIME_MAX_SEC` environment variables, but a higher limit is not a +substitute for fixing allocation or reclaim growth. + +`BC250_REPLAY_THREADS=4` enables experimental parallel command recording. It +is deliberately not the default until the sparse-bind and mesh-program locking +paths are validated together on the guarded route. diff --git a/scripts/bc250/guard-bc250-relink.sh b/scripts/bc250/guard-bc250-relink.sh new file mode 100755 index 0000000..5a3b993 --- /dev/null +++ b/scripts/bc250/guard-bc250-relink.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# Stop a BC-250 Project ReLink run before host OOM or an unsafe GPU temperature. +set -euo pipefail + +if (( $# < 3 )); then + echo "usage: $0 METRICS_LOG GAME_LOG COMMAND [ARG ...]" >&2 + exit 2 +fi + +BC250_METRICS_LOG=$1 +BC250_GAME_LOG=$2 +shift 2 +BC250_MIN_AVAILABLE_MIB=${BC250_MIN_AVAILABLE_MIB:-2048} +BC250_SAMPLE_INTERVAL_SEC=${BC250_SAMPLE_INTERVAL_SEC:-1} +BC250_MAX_TEMP_MILLIC=${BC250_MAX_TEMP_MILLIC:-90000} +BC250_REQUIRE_SWAPCHAIN=${BC250_REQUIRE_SWAPCHAIN:-1} +BC250_SWAPCHAIN_TIMEOUT_SEC=${BC250_SWAPCHAIN_TIMEOUT_SEC:-60} + +[[ $BC250_MIN_AVAILABLE_MIB =~ ^[0-9]+$ ]] || { + echo "error: BC250_MIN_AVAILABLE_MIB must be an integer" >&2 + exit 2 +} +(( BC250_MIN_AVAILABLE_MIB >= 2048 )) || { + echo "error: BC250_MIN_AVAILABLE_MIB must be at least 2048" >&2 + exit 2 +} +[[ $BC250_MAX_TEMP_MILLIC =~ ^[0-9]+$ ]] || { + echo "error: BC250_MAX_TEMP_MILLIC must be an integer" >&2 + exit 2 +} +[[ $BC250_SAMPLE_INTERVAL_SEC =~ ^[0-9]+([.][0-9]+)?$ ]] || { + echo "error: BC250_SAMPLE_INTERVAL_SEC must be a positive number" >&2 + exit 2 +} +[[ $BC250_REQUIRE_SWAPCHAIN =~ ^[01]$ ]] || { + echo "error: BC250_REQUIRE_SWAPCHAIN must be 0 or 1" >&2 + exit 2 +} +[[ $BC250_SWAPCHAIN_TIMEOUT_SEC =~ ^[0-9]+$ ]] || { + echo "error: BC250_SWAPCHAIN_TIMEOUT_SEC must be an integer" >&2 + exit 2 +} +(( BC250_SWAPCHAIN_TIMEOUT_SEC >= 5 )) || { + echo "error: BC250_SWAPCHAIN_TIMEOUT_SEC must be at least 5 seconds" >&2 + exit 2 +} +awk -v value="$BC250_SAMPLE_INTERVAL_SEC" 'BEGIN { exit !(value > 0) }' || { + echo "error: BC250_SAMPLE_INTERVAL_SEC must be greater than zero" >&2 + exit 2 +} + +BC250_MIN_AVAILABLE_KIB=$((BC250_MIN_AVAILABLE_MIB * 1024)) +BC250_CHILD_PID=0 +BC250_STOPPING=0 +BC250_SWAPCHAIN_READY=0 +BC250_START_SECONDS=$SECONDS + +read_number() { + local path=$1 value + [[ -r $path ]] || { echo 0; return; } + value=$(<"$path") + [[ $value =~ ^[0-9]+$ ]] && echo "$value" || echo 0 +} + +stop_child() { + (( BC250_CHILD_PID > 0 )) || return 0 + kill -0 "$BC250_CHILD_PID" 2>/dev/null || return 0 + (( BC250_STOPPING == 0 )) || return 0 + BC250_STOPPING=1 + kill -TERM "$BC250_CHILD_PID" 2>/dev/null || true + for _ in {1..100}; do + kill -0 "$BC250_CHILD_PID" 2>/dev/null || return 0 + sleep 0.1 + done + kill -KILL "$BC250_CHILD_PID" 2>/dev/null || true +} + +on_signal() { + local status=$1 + stop_child + wait "$BC250_CHILD_PID" 2>/dev/null || true + exit "$status" +} + +trap 'on_signal 143' TERM +trap 'on_signal 130' INT +trap 'on_signal 129' HUP + +BC250_MEM_AVAILABLE_KIB=$(awk '/^MemAvailable:/ { print $2; exit }' /proc/meminfo) +if (( BC250_MEM_AVAILABLE_KIB < BC250_MIN_AVAILABLE_KIB )); then + echo "GUARD: MemAvailable is below ${BC250_MIN_AVAILABLE_MIB} MiB before launch" >&2 + exit 125 +fi + +if [[ -n ${BC250_DRM_DEVICE:-} ]]; then + BC250_GPU_DEVICE=$BC250_DRM_DEVICE +else + BC250_GPU_DEVICE= + shopt -s nullglob + for BC250_VENDOR_PATH in /sys/class/drm/card*/device/vendor; do + if [[ $(<"$BC250_VENDOR_PATH") == 0x1002 ]]; then + BC250_GPU_DEVICE=${BC250_VENDOR_PATH%/vendor} + break + fi + done + shopt -u nullglob +fi +if [[ -z $BC250_GPU_DEVICE || ! -d $BC250_GPU_DEVICE ]]; then + echo "error: could not find the BC-250 AMD DRM device" >&2 + exit 1 +fi + +mkdir -p "$(dirname -- "$BC250_METRICS_LOG")" +mkdir -p "$(dirname -- "$BC250_GAME_LOG")" +: >"$BC250_GAME_LOG" +printf '%s\n' \ + 'utc mem_available_kib cgroup_bytes cgroup_swap_bytes proc_rss_anon_kib proc_rss_file_kib proc_rss_shmem_kib cgroup_anon_bytes cgroup_file_bytes cgroup_shmem_bytes cgroup_kernel_bytes cgroup_high_events cgroup_psi_some_usec vram_bytes gtt_bytes sclk_mhz max_temp_millic' \ + >"$BC250_METRICS_LOG" + +"$@" >>"$BC250_GAME_LOG" 2>&1 & +BC250_CHILD_PID=$! + +while kill -0 "$BC250_CHILD_PID" 2>/dev/null; do + if (( BC250_REQUIRE_SWAPCHAIN == 1 && BC250_SWAPCHAIN_READY == 0 )); then + if grep -Eq 'vk: swapchain .* mailbox' "$BC250_GAME_LOG"; then + BC250_SWAPCHAIN_READY=1 + echo "GUARD: verified a visible mailbox swapchain" + elif grep -Eq 'SDL_Init: .*no window|vk: no window:' "$BC250_GAME_LOG"; then + echo "GUARD: ReLink fell back to offscreen rendering; stopping game" >&2 + stop_child + wait "$BC250_CHILD_PID" 2>/dev/null || true + exit 124 + elif (( SECONDS - BC250_START_SECONDS >= BC250_SWAPCHAIN_TIMEOUT_SEC )); then + echo "GUARD: no mailbox swapchain after ${BC250_SWAPCHAIN_TIMEOUT_SEC}s; stopping game" >&2 + stop_child + wait "$BC250_CHILD_PID" 2>/dev/null || true + exit 124 + fi + fi + BC250_MEM_AVAILABLE_KIB=$(awk '/^MemAvailable:/ { print $2; exit }' /proc/meminfo) + BC250_CGROUP_REL=$(awk -F: '$1 == "0" { print $3; exit }' \ + "/proc/$BC250_CHILD_PID/cgroup" 2>/dev/null || true) + BC250_CGROUP_ROOT=/sys/fs/cgroup$BC250_CGROUP_REL + BC250_CGROUP_BYTES=$(read_number "$BC250_CGROUP_ROOT/memory.current") + BC250_CGROUP_SWAP_BYTES=$(read_number "$BC250_CGROUP_ROOT/memory.swap.current") + read -r BC250_RSS_ANON_KIB BC250_RSS_FILE_KIB BC250_RSS_SHMEM_KIB < <( + awk 'BEGIN { a = f = s = 0 } + $1 == "RssAnon:" { a = $2 } + $1 == "RssFile:" { f = $2 } + $1 == "RssShmem:" { s = $2 } + END { print a, f, s }' "/proc/$BC250_CHILD_PID/status" 2>/dev/null || echo '0 0 0' + ) + read -r BC250_CGROUP_ANON BC250_CGROUP_FILE BC250_CGROUP_SHMEM BC250_CGROUP_KERNEL < <( + awk 'BEGIN { a = f = s = k = 0 } + $1 == "anon" { a = $2 } + $1 == "file" { f = $2 } + $1 == "shmem" { s = $2 } + $1 == "kernel" { k = $2 } + END { print a, f, s, k }' "$BC250_CGROUP_ROOT/memory.stat" 2>/dev/null || echo '0 0 0 0' + ) + BC250_CGROUP_HIGH=$(awk '$1 == "high" { print $2; found = 1; exit } + END { if (!found) print 0 }' "$BC250_CGROUP_ROOT/memory.events" 2>/dev/null || echo 0) + BC250_CGROUP_PSI=$(awk '$1 == "some" { + for (i = 2; i <= NF; i++) if ($i ~ /^total=/) { + sub(/^total=/, "", $i); print $i; found = 1; exit + }} END { if (!found) print 0 }' \ + "$BC250_CGROUP_ROOT/memory.pressure" 2>/dev/null || echo 0) + BC250_VRAM_BYTES=$(read_number "$BC250_GPU_DEVICE/mem_info_vram_used") + BC250_GTT_BYTES=$(read_number "$BC250_GPU_DEVICE/mem_info_gtt_used") + BC250_SCLK_MHZ=$(awk '/\*/ { sub(/Mhz/, "", $2); print $2 + 0; exit }' \ + "$BC250_GPU_DEVICE/pp_dpm_sclk" 2>/dev/null || echo 0) + BC250_MAX_SEEN_TEMP=0 + shopt -s nullglob + for BC250_SENSOR_PATH in "$BC250_GPU_DEVICE"/hwmon/hwmon*/temp*_input; do + BC250_SENSOR_VALUE=$(read_number "$BC250_SENSOR_PATH") + (( BC250_SENSOR_VALUE > BC250_MAX_SEEN_TEMP )) && BC250_MAX_SEEN_TEMP=$BC250_SENSOR_VALUE + done + for BC250_HWMON_PATH in /sys/class/hwmon/hwmon*; do + [[ -r $BC250_HWMON_PATH/name ]] || continue + [[ $(<"$BC250_HWMON_PATH/name") == k10temp ]] || continue + for BC250_SENSOR_PATH in "$BC250_HWMON_PATH"/temp*_input; do + BC250_SENSOR_VALUE=$(read_number "$BC250_SENSOR_PATH") + (( BC250_SENSOR_VALUE > BC250_MAX_SEEN_TEMP )) && BC250_MAX_SEEN_TEMP=$BC250_SENSOR_VALUE + done + done + shopt -u nullglob + + printf '%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s\n' \ + "$(date -u +%FT%TZ)" "$BC250_MEM_AVAILABLE_KIB" "$BC250_CGROUP_BYTES" \ + "$BC250_CGROUP_SWAP_BYTES" "$BC250_RSS_ANON_KIB" "$BC250_RSS_FILE_KIB" \ + "$BC250_RSS_SHMEM_KIB" "$BC250_CGROUP_ANON" "$BC250_CGROUP_FILE" \ + "$BC250_CGROUP_SHMEM" "$BC250_CGROUP_KERNEL" "$BC250_CGROUP_HIGH" \ + "$BC250_CGROUP_PSI" "$BC250_VRAM_BYTES" "$BC250_GTT_BYTES" \ + "$BC250_SCLK_MHZ" "$BC250_MAX_SEEN_TEMP" >>"$BC250_METRICS_LOG" + + if (( BC250_MEM_AVAILABLE_KIB < BC250_MIN_AVAILABLE_KIB )); then + echo "GUARD: MemAvailable fell below ${BC250_MIN_AVAILABLE_MIB} MiB; stopping game" >&2 + stop_child + wait "$BC250_CHILD_PID" 2>/dev/null || true + exit 125 + fi + if (( BC250_MAX_SEEN_TEMP >= BC250_MAX_TEMP_MILLIC )); then + echo "GUARD: GPU temperature reached ${BC250_MAX_SEEN_TEMP} millidegrees C; stopping game" >&2 + stop_child + wait "$BC250_CHILD_PID" 2>/dev/null || true + exit 126 + fi + sleep "$BC250_SAMPLE_INTERVAL_SEC" +done + +set +e +wait "$BC250_CHILD_PID" +BC250_CHILD_STATUS=$? +set -e +if (( BC250_REQUIRE_SWAPCHAIN == 1 && BC250_SWAPCHAIN_READY == 0 )); then + echo "GUARD: game exited before a mailbox swapchain was verified" >&2 + exit 124 +fi +exit "$BC250_CHILD_STATUS" diff --git a/scripts/bc250/profile.env.example b/scripts/bc250/profile.env.example new file mode 100644 index 0000000..42b73e6 --- /dev/null +++ b/scripts/bc250/profile.env.example @@ -0,0 +1,21 @@ +# Review and copy this file outside the source tree before editing it. +# It contains no title data, paths, input script, save, cache, or credentials. +RELINK_NO_UPDATE_CHECK=1 +RX_BACKEND=vulkan + +# BC-250 RADV DirectMesh and bounded sparse guest-memory profile. +RADV_BC250_TASK_PERSIST=1 +RX_VK_PRECOMPILE=0 +RX_VK_IMPORT_CHUNK_MB=2 +RX_VK_SPARSE_GUEST=1 +RX_VK_TEX_MAX_MB=32 +RX_VK_VRAM_MB=768 +RX_VK_EVICT_MIN_UNUSED=30 +RX_VK_DEAD_SWEEP_FAST=1 +RX_VK_DROP_DEAD_IMPORTS=1 + +# Window and controller defaults. The launcher owns resolution scale, +# framerate, fullscreen, query correctness and replay-worker defaults. +HOS_REGION_LOG=1 +HOS_WINDOW_SIZE=1280x720 +HOS_INPUT_EXCLUSIVE=1 diff --git a/scripts/bc250/run-relink-bc250.sh b/scripts/bc250/run-relink-bc250.sh new file mode 100755 index 0000000..732cbfb --- /dev/null +++ b/scripts/bc250/run-relink-bc250.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +# Launch an existing Project ReLink build on BC-250 with the validated guards. +set -euo pipefail + +if (( $# != 3 )); then + echo "usage: $0 RELINK_BUILD_DIR PROFILE_ENV RADV_ICD_JSON" >&2 + exit 2 +fi + +BC250_SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +BC250_BUILD_DIR=$1 +BC250_PROFILE_ENV=$2 +BC250_ICD_JSON=$3 +BC250_GAME_EXE=${BC250_GAME_EXE:-$BC250_BUILD_DIR/relink} + +for BC250_REQUIRED_PATH in \ + "$BC250_BUILD_DIR" "$BC250_PROFILE_ENV" "$BC250_ICD_JSON" "$BC250_GAME_EXE"; do + if [[ ! -e $BC250_REQUIRED_PATH ]]; then + echo "error: required path does not exist: $BC250_REQUIRED_PATH" >&2 + exit 1 + fi +done +if [[ ! -x $BC250_GAME_EXE ]]; then + echo "error: game executable is not executable: $BC250_GAME_EXE" >&2 + exit 1 +fi +if [[ $(uname -m) != x86_64 ]]; then + echo "error: the validated BC-250 Project ReLink build requires Linux/x86_64" >&2 + exit 1 +fi +if ! command -v systemd-run >/dev/null 2>&1; then + echo "error: systemd-run is required for the bounded launch scope" >&2 + exit 1 +fi + +set -a +# This is executable shell input and must be a trusted, locally reviewed file. +# shellcheck disable=SC1090 +source "$BC250_PROFILE_ENV" +set +a + +# Remove rejected/diagnostic test state inherited from older profiles. The +# caller may select replay workers explicitly through BC250_REPLAY_THREADS. +unset \ + RX_VK_ASTC_STAGE RX_VK_GPU_UNSWIZZLE RX_VK_ASYNC \ + HOS_VCORE_WAITS HOS_PROFILE HOS_THREAD_TIMES HOS_PRESENT_NOWAIT \ + HOS_FRAMES_AHEAD HOS_PIN_RENDER RX_VK_CHUNK +if [[ ${BC250_KEEP_INPUT_SCRIPT:-0} != 1 ]]; then + unset HOS_INPUT_SCRIPT HOS_INPUT_SCRIPT_BACKGROUND +fi + +export RELINK_NO_UPDATE_CHECK=1 +export RX_BACKEND=vulkan +export VK_DRIVER_FILES=$BC250_ICD_JSON +# An SSH-launched process does not inherit the graphical session environment. +# Keep SDL on the user's existing Wayland/X11 session; allowing its kmsdrm +# fallback can steal a new VT and leave the television on a blank console. +export XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-/run/user/$(id -u)} +export WAYLAND_DISPLAY=${WAYLAND_DISPLAY:-wayland-0} +export DISPLAY=${DISPLAY:-:0} +export SDL_VIDEODRIVER=${SDL_VIDEODRIVER:-wayland,x11} +export RADV_BC250_MESH_TASK_IDX_PASSES=${RADV_BC250_MESH_TASK_IDX_PASSES:-2} +export RADV_BC250_KEEP_IB_KB=${RADV_BC250_KEEP_IB_KB:-4096} +export HOS_FPS_LOG=${HOS_FPS_LOG:-1} +export HOS_FPS_TARGET=${HOS_FPS_TARGET:-60} +export HOS_FPS_MAX=${HOS_FPS_MAX:-60} +export HOS_MEMORY_MB=${HOS_MEMORY_MB:-9809} +export HOS_FULLSCREEN=${BC250_FULLSCREEN:-0} +export MANGOHUD=${MANGOHUD:-1} +export RX_VK_PARALLEL=${BC250_REPLAY_THREADS:-1} +export RX_VK_OCC_QUERIES=${BC250_OCC_QUERIES:-1} +export HOS_OUTPUT_RES=${HOS_OUTPUT_RES:-3} +export HOS_DYNRES=${HOS_DYNRES:-0} +export HOS_RES_SCALE=${HOS_RES_SCALE:-0.67} +export RX_VK_LEAN_PIPELINES=${RX_VK_LEAN_PIPELINES:-1} +export RX_VK_IMAGE_BLOCK_MB=${RX_VK_IMAGE_BLOCK_MB:-32} +export RX_VK_SPARE_MB=${RX_VK_SPARE_MB:-16} +export RX_VK_STAGING_KEEP_MB=${RX_VK_STAGING_KEEP_MB:-0} +export BC250_MIN_AVAILABLE_MIB=${BC250_MIN_AVAILABLE_MIB:-2048} + +[[ $HOS_MEMORY_MB =~ ^[0-9]+$ ]] && (( HOS_MEMORY_MB >= 9809 )) || { + echo "error: HOS_MEMORY_MB must be at least 9809 for this title" >&2 + exit 2 +} +[[ $HOS_FPS_MAX =~ ^[0-9]+$ ]] || { + echo "error: HOS_FPS_MAX must be an integer" >&2 + exit 2 +} +if (( HOS_FPS_MAX > 60 )); then + export HOS_FPS_UNLOCKED=${HOS_FPS_UNLOCKED:-1} +fi + +BC250_LOG_DIR=${BC250_LOG_DIR:-$BC250_BUILD_DIR/logs} +BC250_LOG_STAMP=$(date -u +%Y%m%dT%H%M%SZ) +BC250_METRICS_LOG=$BC250_LOG_DIR/relink-bc250-$BC250_LOG_STAMP.metrics +BC250_GAME_LOG=$BC250_LOG_DIR/relink-bc250-$BC250_LOG_STAMP.log +BC250_RUNTIME_MAX_SEC=${BC250_RUNTIME_MAX_SEC:-7200} +BC250_MEMORY_HIGH_BYTES=${BC250_MEMORY_HIGH_BYTES:-7549747200} +BC250_MEMORY_MAX_BYTES=${BC250_MEMORY_MAX_BYTES:-8912896000} +BC250_MEMORY_SWAP_MAX_BYTES=${BC250_MEMORY_SWAP_MAX_BYTES:-2147483648} +BC250_SCOPE=relink-bc250-$BASHPID +BC250_LAUNCH_PID=0 +BC250_RESTORE_VT= + +[[ $BC250_RUNTIME_MAX_SEC =~ ^[0-9]+$ ]] || { + echo "error: BC250_RUNTIME_MAX_SEC must be an integer number of seconds" >&2 + exit 2 +} +(( BC250_RUNTIME_MAX_SEC >= 60 )) || { + echo "error: BC250_RUNTIME_MAX_SEC must be at least 60 seconds" >&2 + exit 2 +} +for BC250_LIMIT_NAME in \ + BC250_MEMORY_HIGH_BYTES BC250_MEMORY_MAX_BYTES BC250_MEMORY_SWAP_MAX_BYTES; do + if [[ ! ${!BC250_LIMIT_NAME} =~ ^[0-9]+$ ]]; then + echo "error: $BC250_LIMIT_NAME must be an integer number of bytes" >&2 + exit 2 + fi +done +if (( BC250_MEMORY_HIGH_BYTES > BC250_MEMORY_MAX_BYTES )); then + echo "error: BC250_MEMORY_HIGH_BYTES must not exceed BC250_MEMORY_MAX_BYTES" >&2 + exit 2 +fi +if [[ ! -S $XDG_RUNTIME_DIR/$WAYLAND_DISPLAY ]]; then + echo "error: Wayland socket is unavailable: $XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" >&2 + exit 1 +fi + +# ReLink or a display hotplug can leave the kernel on a blank virtual terminal. +# Prefer the seat's Wayland VT and return to it on every normal/signal exit. +if [[ -r /sys/class/tty/tty0/active ]]; then + BC250_ACTIVE_TTY=$(/dev/null 2>&1; then + while read -r BC250_SESSION_ID _ _ BC250_SESSION_SEAT _; do + [[ $BC250_SESSION_SEAT == seat0 ]] || continue + BC250_SESSION_TYPE=$(loginctl show-session "$BC250_SESSION_ID" -p Type --value 2>/dev/null || true) + BC250_SESSION_VT=$(loginctl show-session "$BC250_SESSION_ID" -p VTNr --value 2>/dev/null || true) + if [[ $BC250_SESSION_TYPE == wayland && $BC250_SESSION_VT =~ ^[1-9][0-9]*$ ]]; then + BC250_RESTORE_VT=$BC250_SESSION_VT + break + fi + done < <(loginctl list-sessions --no-legend 2>/dev/null || true) +fi +if [[ -n $BC250_RESTORE_VT ]]; then + sudo -n chvt "$BC250_RESTORE_VT" +fi + +cleanup_launcher() { + local status=$? + trap - EXIT TERM INT HUP + if systemctl --user is-active --quiet "$BC250_SCOPE.scope" 2>/dev/null; then + systemctl --user stop "$BC250_SCOPE.scope" 2>/dev/null || true + fi + if (( BC250_LAUNCH_PID > 0 )) && kill -0 "$BC250_LAUNCH_PID" 2>/dev/null; then + kill -TERM "$BC250_LAUNCH_PID" 2>/dev/null || true + wait "$BC250_LAUNCH_PID" 2>/dev/null || true + fi + BC250_LAUNCH_PID=0 + if [[ -n $BC250_RESTORE_VT ]]; then + sudo -n chvt "$BC250_RESTORE_VT" 2>/dev/null || true + fi + return "$status" +} + +trap cleanup_launcher EXIT +trap 'exit 143' TERM +trap 'exit 130' INT +trap 'exit 129' HUP + +mkdir -p "$BC250_LOG_DIR" + +echo "build: $BC250_BUILD_DIR" +echo "executable: $BC250_GAME_EXE" +echo "profile: $BC250_PROFILE_ENV" +echo "ICD: $VK_DRIVER_FILES" +echo "replay: $RX_VK_PARALLEL thread(s; 0/1 is serial)" +echo "resolution: output $HOS_OUTPUT_RES, scale $HOS_RES_SCALE, dynamic $HOS_DYNRES" +echo "MangoHud: $MANGOHUD" +echo "guest memory: ${HOS_MEMORY_MB} MiB" +echo "metrics: $BC250_METRICS_LOG" +echo "game log: $BC250_GAME_LOG" +echo "time limit: ${BC250_RUNTIME_MAX_SEC}s" +echo "memory: high $BC250_MEMORY_HIGH_BYTES, max $BC250_MEMORY_MAX_BYTES, swap $BC250_MEMORY_SWAP_MAX_BYTES" + +cd -- "$BC250_BUILD_DIR" +"$BC250_SCRIPT_DIR/with-bazzite-performance.sh" \ + systemd-run --user --scope --collect --quiet --unit="$BC250_SCOPE" \ + --property=MemoryHigh="$BC250_MEMORY_HIGH_BYTES" \ + --property=MemoryMax="$BC250_MEMORY_MAX_BYTES" \ + --property=MemorySwapMax="$BC250_MEMORY_SWAP_MAX_BYTES" \ + --property=CPUWeight=10000 \ + --property=IOWeight=1000 \ + --property=RuntimeMaxSec="$BC250_RUNTIME_MAX_SEC" \ + -- "$BC250_SCRIPT_DIR/guard-bc250-relink.sh" \ + "$BC250_METRICS_LOG" "$BC250_GAME_LOG" "$BC250_GAME_EXE" & +BC250_LAUNCH_PID=$! +set +e +wait "$BC250_LAUNCH_PID" +BC250_LAUNCH_STATUS=$? +set -e +BC250_LAUNCH_PID=0 + +cleanup_launcher +trap - EXIT +exit "$BC250_LAUNCH_STATUS" diff --git a/scripts/bc250/with-bazzite-performance.sh b/scripts/bc250/with-bazzite-performance.sh new file mode 100755 index 0000000..80c2ad8 --- /dev/null +++ b/scripts/bc250/with-bazzite-performance.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# Hold Bazzite's tuned-ppd performance profile for exactly one command. +set -euo pipefail + +if (( $# == 0 )); then + echo "usage: $0 COMMAND [ARG ...]" >&2 + exit 2 +fi + +if [[ ${BC250_PERFORMANCE_HOLD:-1} == 0 ]]; then + exec "$@" +fi + +for required in busctl sudo /usr/bin/python3; do + if ! command -v "$required" >/dev/null 2>&1; then + echo "error: performance-profile helper not found: $required" >&2 + exit 1 + fi +done + +if ! /usr/bin/python3 -c 'import dbus' >/dev/null 2>&1; then + echo "error: Python D-Bus bindings are required (Fedora: python3-dbus)" >&2 + exit 1 +fi +if ! busctl introspect --system net.hadess.PowerProfiles \ + /net/hadess/PowerProfiles net.hadess.PowerProfiles \ + | grep -q '[.]HoldProfile'; then + echo "error: tuned-ppd does not expose HoldProfile on the system bus" >&2 + exit 1 +fi + +# tuned-ppd associates a hold with its D-Bus caller and releases it if that +# caller disappears. Keeping the helper's stdin pipe open therefore gives us +# both an orderly ReleaseProfile and a crash-safe release on parent death. +BC250_HOLDER_CODE=$(printf '%s\n' \ + 'import dbus' \ + 'import sys' \ + 'bus = dbus.SystemBus()' \ + 'obj = bus.get_object("net.hadess.PowerProfiles", "/net/hadess/PowerProfiles")' \ + 'iface = dbus.Interface(obj, "net.hadess.PowerProfiles")' \ + 'cookie = iface.HoldProfile("performance", "Project ReLink", "org.biosed.OcarinaAsahiPort")' \ + 'print(int(cookie), flush=True)' \ + 'try:' \ + ' sys.stdin.buffer.read()' \ + 'finally:' \ + ' try:' \ + ' iface.ReleaseProfile(cookie)' \ + ' except dbus.DBusException:' \ + ' pass') + +BC250_HOLDER_PID=0 +BC250_HOLDER_IN_FD=-1 +BC250_HOLDER_OUT_FD=-1 +BC250_CHILD_PID=0 +BC250_HOLDER_CLEANED=0 +BC250_RELEASE_FAILED=0 + +release_profile() { + (( BC250_HOLDER_CLEANED == 0 )) || return 0 + BC250_HOLDER_CLEANED=1 + + if (( BC250_HOLDER_IN_FD >= 0 )); then + exec {BC250_HOLDER_IN_FD}>&- + BC250_HOLDER_IN_FD=-1 + fi + if (( BC250_HOLDER_OUT_FD >= 0 )); then + exec {BC250_HOLDER_OUT_FD}>&- + BC250_HOLDER_OUT_FD=-1 + fi + if (( BC250_HOLDER_PID > 0 )); then + wait "$BC250_HOLDER_PID" 2>/dev/null || true + BC250_HOLDER_PID=0 + fi + + for _ in {1..30}; do + BC250_ACTIVE_PROFILE=$(busctl get-property --system net.hadess.PowerProfiles \ + /net/hadess/PowerProfiles net.hadess.PowerProfiles ActiveProfile 2>/dev/null || true) + if [[ $BC250_ACTIVE_PROFILE != 's "performance"' ]]; then + return 0 + fi + sleep 0.1 + done + echo "error: performance profile remained active after releasing the hold" >&2 + BC250_RELEASE_FAILED=1 +} + +forward_signal() { + local status=$1 + if (( BC250_CHILD_PID > 0 )) && kill -0 "$BC250_CHILD_PID" 2>/dev/null; then + kill -TERM "$BC250_CHILD_PID" 2>/dev/null || true + wait "$BC250_CHILD_PID" 2>/dev/null || true + fi + BC250_CHILD_PID=0 + exit "$status" +} + +on_exit() { + local status=$? + trap - EXIT + release_profile + exit "$status" +} + +trap on_exit EXIT +trap 'forward_signal 143' TERM +trap 'forward_signal 130' INT +trap 'forward_signal 129' HUP + +coproc BC250_PPD_HOLDER { + sudo -n /usr/bin/python3 -c "$BC250_HOLDER_CODE" +} +BC250_HOLDER_PID=$BC250_PPD_HOLDER_PID +BC250_HOLDER_OUT_FD=${BC250_PPD_HOLDER[0]} +BC250_HOLDER_IN_FD=${BC250_PPD_HOLDER[1]} + +if ! IFS= read -r BC250_HOLD_COOKIE <&"$BC250_HOLDER_OUT_FD"; then + echo "error: could not acquire Bazzite's performance profile" >&2 + echo "hint: run 'sudo -v' once, then retry" >&2 + exit 1 +fi +if [[ ! $BC250_HOLD_COOKIE =~ ^[0-9]+$ ]]; then + echo "error: tuned-ppd returned an invalid hold cookie" >&2 + exit 1 +fi + +# The D-Bus method returns before TuneD has necessarily finished applying the +# mapped profile. Do not launch until both the profile and all CPU policies are +# visibly in performance mode. +BC250_PROFILE_READY=0 +for _ in {1..50}; do + BC250_ACTIVE_PROFILE=$(busctl get-property --system net.hadess.PowerProfiles \ + /net/hadess/PowerProfiles net.hadess.PowerProfiles ActiveProfile 2>/dev/null || true) + BC250_GOVERNORS_OK=1 + BC250_POLICY_COUNT=0 + shopt -s nullglob + for BC250_GOVERNOR_PATH in /sys/devices/system/cpu/cpufreq/policy*/scaling_governor; do + (( BC250_POLICY_COUNT += 1 )) + if [[ $(<"$BC250_GOVERNOR_PATH") != performance ]]; then + BC250_GOVERNORS_OK=0 + break + fi + done + shopt -u nullglob + if [[ $BC250_ACTIVE_PROFILE == 's "performance"' ]] \ + && (( BC250_POLICY_COUNT > 0 )) && (( BC250_GOVERNORS_OK == 1 )); then + BC250_PROFILE_READY=1 + break + fi + sleep 0.1 +done + +if (( BC250_PROFILE_READY == 0 )); then + echo "error: Bazzite's performance profile did not select every CPU policy" >&2 + exit 1 +fi + +echo "power profile: performance (${BC250_POLICY_COUNT} CPU policies, hold ${BC250_HOLD_COOKIE})" + +"$@" & +BC250_CHILD_PID=$! +set +e +wait "$BC250_CHILD_PID" +BC250_CHILD_STATUS=$? +set -e +BC250_CHILD_PID=0 + +release_profile +trap - EXIT +(( BC250_RELEASE_FAILED == 0 )) || exit 1 +exit "$BC250_CHILD_STATUS"