Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions gpu/third_party/suyu/shader_recompiler/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ add_library(shader_recompiler STATIC
ir_opt/lower_fp16_to_fp32.cpp
ir_opt/lower_fp64_to_fp32.cpp
ir_opt/lower_int64_to_int32.cpp
ir_opt/native_cbuf_pointer_policy.h
ir_opt/passes.h
ir_opt/position_pass.cpp
ir_opt/rescaling_pass.cpp
Expand Down
5 changes: 4 additions & 1 deletion gpu/vk/VULKAN.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ Why Vulkan, after WebGPU:
- **Guest memory in place.** `VK_EXT_external_memory_host` imports the game's pools as buffers. GPU writes land in
guest memory and CPU writes are seen by the GPU, with no mirrored buffer cache, write-back syncs or upload
streams (the WebGPU renderer's biggest costs). Hot read-only ranges can later be mirrored to device memory, as
totk-pc-recomp does.
totk-pc-recomp does. On Linux, a memfd plus `/dev/udmabuf` avoids re-pinning every host page at each AMD submit.
`RX_VK_SPARSE_GUEST=1` is an opt-in low-memory form of that path: when the device guarantees strict zero reads
from non-resident sparse-buffer blocks, only populated device-sized blocks are imported and untouched blocks stay
unbound. It falls back to the ordinary complete import when any required capability or import is unavailable.
- **SPIR-V straight from the IR.** No WGSL, no naga, and none of the WebGPU workarounds the WGSL back end carries
(texture swizzle and border emulation, d16 quantization, storage format casts, vertex pulling).
- **No validation or usage tracking on every call**, and bindless descriptors like NVN's own pools.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "shader_recompiler/frontend/ir/value.h"
#include "shader_recompiler/host_translate_info.h"
#include "shader_recompiler/ir_opt/passes.h"
#include "shader_recompiler/ir_opt/native_cbuf_pointer_policy.h"

namespace Shader::Optimization {

Expand Down Expand Up @@ -1309,7 +1310,8 @@ bool MixesLoadedPointer(const IR::Value& value) {
}

thread_local bool g_follow_stage = false; // (relink: the program being translated is compute, where the page table is)
thread_local bool g_graphics_stage = false; // (sharkiix: the program being translated is a geometry stage: vertex, tessellation, geometry, task, mesh)
thread_local bool g_graphics_stage = false; // (relink: any graphics stage, including fragment)
thread_local bool g_geometry_stage = false; // (sharkiix: vertex, tessellation, geometry, task or mesh; not fragment)

/// Collects the storage buffer used by a global memory instruction and the instruction itself
void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info, Environment& env,
Expand Down Expand Up @@ -1492,10 +1494,19 @@ void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info,
{ // (sharkiix, from BernabeCR93 in upstream issue 667) A graphics stage's address tracked to another constant buffer than
// c0 while part of it comes from memory: the far-foliage and mesh programs (95db514a35353e28, d905f52b7517c681,
// 51183fe750d5d0c8...) took float constants for a buffer address ("storage buffer NOT bound (reads zeros)", c5+0x60),
// read zeros and drew the plants without colour (white/sky-blue "mirror" ferns, glowing flowers). Left to the run-time
// search. RX_VK_NO_CBUF_LOAD_RULE=1: off
// read zeros and drew the plants without colour (white/sky-blue "mirror" ferns, glowing flowers). A native argument
// table can also be mistaken directly for a pointer: da7a7ab28b8e73d9's fragment stage took c5+0's
// 0x4180000000000200 (16.0f and a 0x200 scalar). The environment validates candidate words against mapped GPU
// pools. The memory-origin rule remains geometry-only; native scalar rejection also covers fragment stages.
// Both cases stay on the run-time address search. RX_VK_NO_CBUF_LOAD_RULE=1: off
static const bool cbuf_load_rule{std::getenv("RX_VK_NO_CBUF_LOAD_RULE") == nullptr};
if (cbuf_load_rule && storage_buffer->index != 0 && g_graphics_stage && Origins(inst.Arg(0)).load) {
const bool native_program{env.IsNativeProgram()};
const bool validate_pointer{native_program && g_graphics_stage && storage_buffer->index != 0};
const bool mapped_pointer{!validate_pointer ||
env.IsCbufStorageBufferPointer(storage_buffer->index, storage_buffer->offset)};
if (cbuf_load_rule && native_cbuf_pointer_policy::leave_global(
native_program, g_graphics_stage, g_geometry_stage, storage_buffer->index,
Origins(inst.Arg(0)).load, mapped_pointer)) {
leave_global();
return;
}
Expand Down Expand Up @@ -1660,8 +1671,10 @@ void GlobalMemoryToStorageBufferPass(Environment& env, IR::Program& program,
// (relink)
static const bool no_mesh{std::getenv("RX_VK_FOLLOW_MESH") == nullptr};
g_follow_stage = program.stage == Stage::Compute || (!no_mesh && (program.stage == Stage::Task || program.stage == Stage::Mesh));
// (geometry stages only: with the fragment stage too, the sky drew black rectangles and the lighting went too bright at the castle)
g_graphics_stage = program.stage != Stage::Compute && program.stage != Stage::Fragment;
g_graphics_stage = program.stage != Stage::Compute;
// The older memory-origin workaround remains geometry-only: enabling it for fragment shaders made the sky draw
// black rectangles and lifted the castle lighting. Native unmapped-candidate rejection is independent and safe there.
g_geometry_stage = g_graphics_stage && program.stage != Stage::Fragment;
g_origins.clear(); // (EvilOOT: Origins' walk, kept per program)
const bool pairing{!GmLegacyFor(program.stage)};
std::optional<TrackingConditions> conditions;
Expand Down
27 changes: 27 additions & 0 deletions gpu/vk/eden/shader_recompiler/ir_opt/native_cbuf_pointer_policy.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// SPDX-License-Identifier: MIT
#pragma once

#include <cstdint>

namespace native_cbuf_pointer_policy {

/// A geometry-stage global-memory access tracked to a non-driver constant bank must stay
/// on the run-time address search when the candidate is not a mapped GPU pointer. Native
/// argument tables mix pointers with handles, sizes, strides, and floating-point values.
///
/// The memory-origin case predates the native validity check: rebasing an address that is
/// partly loaded through memory onto one constant-buffer candidate loses the loaded path.
inline bool leave_global(bool native_program, bool graphics_stage, bool geometry_stage,
uint32_t cbuf_index, bool address_loads_memory,
bool mapped_pointer) noexcept {
if (cbuf_index == 0) {
return false;
}
// The older memory-origin workaround is deliberately geometry-only: applying it to
// fragment shaders changed valid lighting paths. Rejecting an unmapped native scalar is
// a narrower invariant and applies to every graphics stage, including fragment shaders.
return (geometry_stage && address_loads_memory) ||
(graphics_stage && native_program && !mapped_pointer);
}

} // namespace native_cbuf_pointer_policy
197 changes: 197 additions & 0 deletions gpu/vk/sparse_guest_ranges.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
#pragma once

#include <algorithm>
#include <cstdint>
#include <utility>
#include <vector>

namespace sparse_guest {

struct Run {
uint64_t offset = 0, size = 0;
};

/// Append a sorted run, merging overlap and adjacency. False rejects wrapped
/// or out-of-order input instead of producing an unsafe backing range.
inline bool append_merged(std::vector<Run>& runs, Run r) {
if (!r.size) return true;
if (r.offset > UINT64_MAX - r.size) return false;
if (runs.empty()) {
runs.push_back(r);
return true;
}
Run& last = runs.back();
if (last.offset > UINT64_MAX - last.size || r.offset < last.offset) return false;
const uint64_t last_hi = last.offset + last.size;
if (r.offset > last_hi) {
runs.push_back(r);
return true;
}
last.size = std::max(last_hi, r.offset + r.size) - last.offset;
return true;
}

/// Number of bytes actually backed by a packed sparse allocation. This is
/// deliberately not the sparse resource's virtual span: holes have no dma-buf
/// pages and must not be credited as freed under memory pressure.
inline uint64_t bytes(const std::vector<Run>& runs) {
uint64_t result = 0;
for (const Run& r : runs) {
if (r.size > UINT64_MAX - result) return UINT64_MAX;
result += r.size;
}
return result;
}

inline bool bound(const std::vector<uint64_t>& words, uint64_t block) {
return block / 64 < words.size() && (words[block / 64] >> (block & 63) & 1);
}

inline void set_bound(std::vector<uint64_t>& words, uint64_t first, uint64_t count) {
for (uint64_t block = first; block < first + count; block++)
words[block / 64] |= 1ull << (block & 63);
}

inline void clear_bound(std::vector<uint64_t>& words, uint64_t first, uint64_t count) {
for (uint64_t block = first; block < first + count; block++)
if (block / 64 < words.size()) words[block / 64] &= ~(1ull << (block & 63));
}

/// Whether any byte backed by these runs touches [lo, hi). `resource_lo` is the guest address represented by offset 0.
inline bool touches(const std::vector<Run>& runs, uint64_t resource_lo, uint64_t lo, uint64_t hi) {
if (hi <= lo) return false;
for (const Run& r : runs) {
if (r.offset > UINT64_MAX - resource_lo) return true;
const uint64_t a = resource_lo + r.offset;
const uint64_t b = r.size > UINT64_MAX - a ? UINT64_MAX : a + r.size;
if (a < hi && b > lo) return true;
}
return false;
}

/// Whether every byte backed by these runs is outside the sorted live ranges. A sparse allocation may pack several
/// disjoint runs into one VkDeviceMemory, so it can be freed only when none of them is live.
template <class Ranges>
inline bool all_outside(const std::vector<Run>& runs, uint64_t resource_lo, const Ranges& live) {
for (const Run& r : runs) {
if (r.offset > UINT64_MAX - resource_lo) return false;
const uint64_t a = resource_lo + r.offset;
const uint64_t b = r.size > UINT64_MAX - a ? UINT64_MAX : a + r.size;
for (const auto& span : live) {
if (span.lo >= b) break;
if (span.lo < b && span.hi > a) return false;
}
}
return true;
}

/// Whether [lo, hi) touches a block already bound to the sparse resource. The range may extend past the resource;
/// callers use this when deciding whether freed guest pages are still visible to the GPU.
inline bool overlaps_bound(const std::vector<uint64_t>& words, uint64_t resource_lo, uint64_t block_size,
uint64_t lo, uint64_t hi) {
if (!block_size || hi <= lo || hi <= resource_lo || words.empty()) return false;
lo = std::max(lo, resource_lo);
const uint64_t first = (lo - resource_lo) / block_size;
const uint64_t bytes = hi - resource_lo;
const uint64_t limit = words.size() > UINT64_MAX / 64 ? UINT64_MAX : words.size() * 64;
const uint64_t last = std::min<uint64_t>(limit, bytes / block_size + (bytes % block_size != 0));
for (uint64_t block = first; block < last; block++)
if (bound(words, block)) return true;
return false;
}

/// Mark every sparse block touched by [lo, hi). The caller supplies ranges already clamped to the resource.
inline void mark(std::vector<uint8_t>& wanted, uint64_t resource_lo, uint64_t block_size, uint64_t lo, uint64_t hi) {
if (!block_size || hi <= lo || lo < resource_lo) return;
const uint64_t first = (lo - resource_lo) / block_size;
const uint64_t bytes = hi - resource_lo;
const uint64_t last = std::min<uint64_t>(wanted.size(), bytes / block_size + (bytes % block_size != 0));
for (uint64_t block = first; block < last; block++) wanted[(size_t)block] = 1;
}

/// Merge consecutive wanted, not-yet-bound blocks into Vulkan sparse binding runs.
inline std::vector<Run> runs(const std::vector<uint8_t>& wanted, const std::vector<uint64_t>& bound_words,
uint64_t block_size) {
std::vector<Run> result;
for (uint64_t block = 0; block < wanted.size();) {
while (block < wanted.size() && (!wanted[(size_t)block] || bound(bound_words, block))) block++;
const uint64_t first = block;
while (block < wanted.size() && wanted[(size_t)block] && !bound(bound_words, block)) block++;
if (block > first) result.push_back({first * block_size, (block - first) * block_size});
}
return result;
}

/// Merge the not-yet-bound blocks touched by [lo, hi) into runs. Offsets are
/// relative to the sparse resource. This lets callers avoid querying their
/// backing store again for blocks whose Vulkan bindings already exist.
inline std::vector<Run> unbound_runs(const std::vector<uint64_t>& bound_words, uint64_t block_size,
uint64_t resource_size, uint64_t lo, uint64_t hi) {
std::vector<Run> result;
if (!block_size || hi <= lo || lo >= resource_size) return result;
hi = std::min(hi, resource_size);
const uint64_t first = lo / block_size;
const uint64_t last = (hi - 1) / block_size + 1;
for (uint64_t block = first; block < last;) {
while (block < last && bound(bound_words, block)) block++;
const uint64_t begin = block;
while (block < last && !bound(bound_words, block)) block++;
if (block > begin) result.push_back({begin * block_size, (block - begin) * block_size});
}
return result;
}

/// Continue an extent scan after the sparse block containing `data`. Once any
/// byte in a block is allocated, that whole block is wanted; visiting further
/// file extents inside it cannot change the resulting Vulkan binding.
inline uint64_t scan_advance(uint64_t resource_lo, uint64_t block_size, uint64_t data,
uint64_t hole, uint64_t limit) {
if (!block_size || data < resource_lo || hole < data) return limit;
const uint64_t block = (data - resource_lo) / block_size;
uint64_t block_hi = UINT64_MAX;
if (block < (UINT64_MAX - resource_lo) / block_size)
block_hi = resource_lo + (block + 1) * block_size;
return std::min(limit, std::max(hole, block_hi));
}

/// Split aligned runs into the packed allocations accepted by UDMABUF_CREATE_LIST. Both the packed byte total and
/// extent count are per ioctl; a single contiguous run may therefore become several allocations. An empty result for
/// non-empty input means the limits or the input alignment are invalid.
inline std::vector<std::vector<Run>> batches(const std::vector<Run>& input, uint64_t max_bytes,
uint64_t max_runs, uint64_t alignment) {
std::vector<std::vector<Run>> result;
if (input.empty()) return result;
if (!max_bytes || !max_runs || !alignment) return result;
const uint64_t byte_limit = max_bytes / alignment * alignment;
if (!byte_limit) return result;
std::vector<Run> batch;
uint64_t bytes = 0;
const auto finish = [&] {
if (!batch.empty()) result.push_back(std::move(batch));
batch.clear();
bytes = 0;
};
for (const Run& r : input) {
if (!r.size || r.offset % alignment || r.size % alignment || r.size > UINT64_MAX - r.offset) {
result.clear();
return result;
}
uint64_t offset = r.offset, left = r.size;
while (left) {
if (bytes == byte_limit || batch.size() >= max_runs) finish();
const uint64_t take = std::min(left, byte_limit - bytes);
if (!take || take % alignment) {
result.clear();
return result;
}
batch.push_back({offset, take});
bytes += take;
offset += take;
left -= take;
}
}
finish();
return result;
}

} // namespace sparse_guest
23 changes: 23 additions & 0 deletions gpu/vk/texture_memory_policy.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// Effective texture-memory allowance for policies that choose image formats and staging retention.
// nx2/test_texture_memory_policy.py
#pragma once
#include <algorithm>
#include <cstdint>

namespace texture_memory_policy {

constexpr uint64_t MiB = 1ull << 20;
constexpr uint64_t IntegratedDefault = 2560ull * MiB;

// A discrete heap is dedicated VRAM. An integrated heap is shared system RAM, so the driver's heap allowance is not
// what textures may consume beside the game. An explicit texture budget is a still tighter statement of that limit.
inline uint64_t effective_allowance(uint64_t driver_allowed, bool integrated, uint64_t configured_budget = 0) {
const uint64_t cap = configured_budget ? configured_budget : integrated ? IntegratedDefault : 0;
return cap ? (driver_allowed ? std::min(driver_allowed, cap) : cap) : driver_allowed;
}

inline uint64_t staging_keep(uint64_t effective_allowed) {
return effective_allowed && effective_allowed <= 6656ull * MiB ? 64ull * MiB : 256ull * MiB;
}

} // namespace texture_memory_policy
1 change: 1 addition & 0 deletions gpu/vk/vk_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ struct Device {
device_fault = false, // (VK_EXT_device_fault: what a lost device faulted on)
sparse_images = false, // (sparse resident 2D images on the queue: the terrain's huge arrays)
sparse_buffers = false, // (sparse-bound buffers on the queue: a pool imported as several dma-bufs)
sparse_zero_buffers = false, // (partly unbound buffers read zero: sparse guest-memory holes)
view_2d_of_3d = false, // (VK_EXT_image_2d_view_of_3d: a 3D texture a 2D sample instruction reads)
coop_matrix = false, // (VK_KHR_cooperative_matrix with its feature)
max_reconvergence = false, // (relink, EDEN_NX2_MAXRECONV=1: VK_KHR_shader_maximal_reconvergence with its feature)
Expand Down
2 changes: 2 additions & 0 deletions gpu/vk/vk_main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,8 @@ bool create_device() {
dev.sparse_images = (families[dev.queue_family].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) && f.features.sparseBinding &&
f.features.sparseResidencyImage2D && !(getenv("RX_VK_SPARSE") && !atoi(getenv("RX_VK_SPARSE")));
dev.sparse_buffers = (families[dev.queue_family].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) && f.features.sparseBinding;
dev.sparse_zero_buffers = dev.sparse_buffers && f.features.sparseResidencyBuffer &&
dev.props.sparseProperties.residencyNonResidentStrict;
VkDeviceCreateInfo ci{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};
ci.pNext = &f;
ci.queueCreateInfoCount = compute_family != UINT32_MAX ? 2 : 1;
Expand Down
Loading