From 47215ccbe0335166946f8cc32ec9470ef41aca66 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:39:00 +0200 Subject: [PATCH 01/40] Cache clean GPU memory stamps until the next physical write Track a process-wide write epoch under the SDK critical region so repeated validation of unchanged page snapshots skips redundant scans. Physical writes and device shutdown invalidate the shortcut; counter wrap disables it permanently. Validation: production watcher fixture, cache-removed negative control, four real SDK page-fault modes, and the 51-check portable GPU suite passed. The native Windows build passed; no frame-rate improvement is claimed. --- rexlego/src/gpu_native/memory_watch.cpp | 10 ++++ rexlego/src/gpu_native/memory_watch.h | 3 ++ tools/gpu-aot/test_memory_watch.py | 70 +++++++++++++++++++++---- 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/rexlego/src/gpu_native/memory_watch.cpp b/rexlego/src/gpu_native/memory_watch.cpp index 3c9f8ac..431e976 100644 --- a/rexlego/src/gpu_native/memory_watch.cpp +++ b/rexlego/src/gpu_native/memory_watch.cpp @@ -22,10 +22,16 @@ constexpr uint32_t kPageSize = 4096, kPhysicalSize = 0x20000000; // Protected by the SDK global critical region. Fault callbacks never take // renderer/resource locks and never call the driver. std::array g_page_versions{}; +uint64_t g_write_epoch = 1; rex::thread::global_critical_region g_watch_region; rex::memory::Memory* g_memory = nullptr; void* g_callback = nullptr; +void AdvanceWriteEpoch() { + // Zero permanently disables the shortcut if this counter ever wraps. + if (g_write_epoch) ++g_write_epoch; +} + bool SelectedMemoryWatch() { // Vertex animation textures retain every-use content checks. static const bool enabled = [] { @@ -49,6 +55,7 @@ bool SelectedBufferWatch() { void Invalidate(uint32_t address, uint32_t length) { if (!length || address >= kPhysicalSize) return; + AdvanceWriteEpoch(); const uint64_t end = std::min(uint64_t(address) + length, uint64_t(kPhysicalSize)); for (uint32_t page = address / kPageSize; uint64_t(page) * kPageSize < end; ++page) ++g_page_versions[page]; @@ -121,8 +128,10 @@ CpuMemoryStamp WatchCpuMemory(std::span spans) { bool CpuMemoryUnchanged(const CpuMemoryStamp& stamp) { if (stamp.pages.empty()) return false; auto lock = g_watch_region.Acquire(); + if (g_write_epoch && stamp.checked_write_epoch == g_write_epoch) return true; for (const auto [page, version] : stamp.pages) if (g_page_versions[page] != version) return false; + stamp.checked_write_epoch = g_write_epoch; return true; } @@ -137,6 +146,7 @@ void ShutdownCpuMemoryWatch() { g_callback = nullptr; g_memory = nullptr; // Invalidate stamps even if a new device is created with the same Memory. + AdvanceWriteEpoch(); for (auto& version : g_page_versions) ++version; } } // namespace legodimensions::gpu_native diff --git a/rexlego/src/gpu_native/memory_watch.h b/rexlego/src/gpu_native/memory_watch.h index a2c608b..4281ab8 100644 --- a/rexlego/src/gpu_native/memory_watch.h +++ b/rexlego/src/gpu_native/memory_watch.h @@ -10,6 +10,9 @@ namespace legodimensions::gpu_native { struct CpuMemorySpan { uint32_t address = 0, length = 0; }; struct CpuMemoryStamp { std::vector> pages; + // Page snapshots are immutable after publication. Validation and this cache + // are serialized with physical write callbacks by the SDK critical region. + mutable uint64_t checked_write_epoch = 0; }; bool CpuMemoryWatchEnabled(); bool CpuBufferMemoryWatchEnabled(); diff --git a/tools/gpu-aot/test_memory_watch.py b/tools/gpu-aot/test_memory_watch.py index 82289b1..4a29f4d 100644 --- a/tools/gpu-aot/test_memory_watch.py +++ b/tools/gpu-aot/test_memory_watch.py @@ -5,7 +5,7 @@ from pathlib import Path import subprocess -p=argparse.ArgumentParser(description=__doc__); p.add_argument('output',type=Path); a=p.parse_args() +p=argparse.ArgumentParser(description=__doc__); p.add_argument('output',type=Path); p.add_argument('--verify-regression',action='store_true'); p.add_argument('--compiler',default='clang++'); a=p.parse_args() root=Path(__file__).resolve().parents[2]; a.output.mkdir(parents=True,exist_ok=True) stub=a.output/'include/rex/system'; stub.mkdir(parents=True,exist_ok=True) (stub.parent/'cvar.h').write_text(r''' @@ -73,6 +73,10 @@ #include #include #include +#ifdef WATCH_FIXTURE_COUNTER +extern uint64_t fixture_page_checks; +namespace legodimensions::gpu_native { void FixtureSetEpoch(uint64_t); } +#endif using namespace legodimensions::gpu_native; CpuMemoryStamp Stamp(uint32_t at,uint32_t length) { CpuMemorySpan span{at,length}; return WatchCpuMemory({&span,1}); @@ -109,25 +113,71 @@ ShutdownCpuMemoryWatch();assert(!CpuMemoryUnchanged(after)); auto recreated=Stamp(0xA0009000,4096);assert(CpuMemoryUnchanged(recreated)); memory.Write(0xE0008001);assert(!CpuMemoryUnchanged(recreated)); + #ifdef WATCH_FIXTURE_COUNTER + auto large=Stamp(0xA0010000,128*4096); + fixture_page_checks=0;assert(CpuMemoryUnchanged(large));assert(fixture_page_checks==128); + for(unsigned i=0;i<10000;++i) assert(CpuMemoryUnchanged(large)); + assert(fixture_page_checks==128); // No page walk without an intervening write. + std::thread reader([&]{for(unsigned i=0;i<1000;++i) assert(CpuMemoryUnchanged(large));}); + reader.join();assert(fixture_page_checks==128); + InvalidateCpuPhysicalMemory(0x1000,4); // Unrelated write: validate once again. + assert(CpuMemoryUnchanged(large));assert(fixture_page_checks==256); + assert(CpuMemoryUnchanged(large));assert(fixture_page_checks==256); + InvalidateCpuPhysicalMemory(0x10004,4);assert(!CpuMemoryUnchanged(large)); + large=Stamp(0xA0010000,128*4096);assert(CpuMemoryUnchanged(large)); + FixtureSetEpoch(UINT64_MAX);assert(CpuMemoryUnchanged(large)); + InvalidateCpuPhysicalMemory(0x1000,4); // Wrap disables cache permanently. + fixture_page_checks=0; + assert(CpuMemoryUnchanged(large));assert(CpuMemoryUnchanged(large)); + assert(fixture_page_checks==256); + InvalidateCpuPhysicalMemory(0x1000,4); + assert(CpuMemoryUnchanged(large));assert(fixture_page_checks==384); + ShutdownCpuMemoryWatch();assert(!CpuMemoryUnchanged(large)); + #endif ShutdownCpuMemoryWatch(); } ''') exe=a.output/'memory-watch-test.exe' -subprocess.run(['clang++','-std=c++20','-DNOMINMAX','-I'+str(a.output/'include'), - '-I'+str(root/'rexlego/src'),str(root/'rexlego/src/gpu_native/memory_watch.cpp'),str(h),'-o',str(exe)],check=True) +subprocess.run([a.compiler,'-std=c++20','-DNOMINMAX','-I'+str(a.output/'include'), + '-I'+str(root/'rexlego/src'),str(root/'rexlego/src/gpu_native/memory_watch.cpp'),str(h),'-o',str(exe)],check=True,timeout=45) env=dict(os.environ);env.pop('LEGO_NATIVE_NO_MEMORY_WATCH',None);env['LEGO_NATIVE_STATIC_TEXTURE_WATCH']='1' -subprocess.run([str(exe.resolve())],env=env,check=True) -env['LEGO_NATIVE_NO_MEMORY_WATCH']='1';subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True) +subprocess.run([str(exe.resolve())],env=env,check=True,timeout=10) +env['LEGO_NATIVE_NO_MEMORY_WATCH']='1';subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True,timeout=10) env.pop('LEGO_NATIVE_NO_MEMORY_WATCH');env['LEGO_NATIVE_STATIC_TEXTURE_WATCH']='0' -subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True) +subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True,timeout=10) env.pop('LEGO_NATIVE_STATIC_TEXTURE_WATCH') -subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True) +subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True,timeout=10) env['LEGO_NATIVE_BUFFER_WATCH']='1';env['LEGO_NATIVE_STATIC_TEXTURE_WATCH']='0' -subprocess.run([str(exe.resolve())],env=env,check=True) +subprocess.run([str(exe.resolve())],env=env,check=True,timeout=10) env['LEGO_NATIVE_NO_MEMORY_WATCH']='1' -subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True) +subprocess.run([str(exe.resolve()),'disabled'],env=env,check=True,timeout=10) +# Count page comparisons in the production loop; all decisions remain unchanged. +production=(root/'rexlego/src/gpu_native/memory_watch.cpp').read_text() +loop=' for (const auto [page, version] : stamp.pages)\n if (g_page_versions[page] != version) return false;' +assert production.count(loop)==1 +instrumented=production.replace(loop, ' for (const auto [page, version] : stamp.pages) {\n ++fixture_page_checks;\n if (g_page_versions[page] != version) return false;\n }') +instrumented='#include \nuint64_t fixture_page_checks=0;\n'+instrumented +instrumented+='\nnamespace legodimensions::gpu_native { void FixtureSetEpoch(uint64_t epoch) { auto lock=g_watch_region.Acquire();g_write_epoch=epoch; } }\n' +counted=a.output/'watch-counted.cpp';counted.write_text(instrumented) +count_exe=a.output/'memory-watch-counted.exe' +command=[a.compiler,'-std=c++20','-DNOMINMAX','-DWATCH_FIXTURE_COUNTER','-I'+str(a.output/'include'),'-I'+str(root/'rexlego/src'),str(counted),str(h),'-o',str(count_exe)] +subprocess.run(command,check=True,timeout=45) +env['LEGO_NATIVE_STATIC_TEXTURE_WATCH']='1';env.pop('LEGO_NATIVE_NO_MEMORY_WATCH',None) +subprocess.run([str(count_exe.resolve())],env=env,check=True,timeout=10) +if a.verify_regression: + shortcut=' if (g_write_epoch && stamp.checked_write_epoch == g_write_epoch) return true;' + assert instrumented.count(shortcut)==1 + counted.write_text(instrumented.replace(shortcut,'')) + negative_exe=a.output/'memory-watch-uncached-control.exe' + negative_command=command[:-1]+[str(negative_exe)] + subprocess.run(negative_command,check=True,timeout=45) + import resource + def no_core(): resource.setrlimit(resource.RLIMIT_CORE,(0,0)) + bad=subprocess.run([str(negative_exe.resolve())],env=env,capture_output=True,text=True,timeout=10,preexec_fn=no_core) + assert bad.returncode!=0 and 'fixture_page_checks==128' in bad.stderr,bad.stderr + counted.write_text(instrumented) (a.output/'verification.json').write_text(json.dumps({'production_source':True,'sdk_model':True,'checks':[ 'A/C/E aliases and +4KB','same-frame writes','neighbor watch retained','base+mips','physical host write', 'write between arm and reuse','unknown/uncommitted/overflow fallback','device recreation','disabled fallback'], - 'passed':True},indent=2)) + 'clean_validation_page_walks':128,'repeated_clean_checks':11000,'unrelated_write_revalidates':True,'epoch_wrap_disables_cache':True,'negative_control_passed':a.verify_regression,'passed':True},indent=2)) print('Passed production watcher with SDK callback model: aliases, same-frame writes, separate mips, races, neighbor protection, physical copies and fallback.') From a3652074c2de8547de4bebce7e81800945c708e7 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:41:58 +0200 Subject: [PATCH 02/40] Recover missing native shaders from validated additive DXIL packs Bound shader parsing and placement lookup by readable guest allocations, capture missing shaders by full container identity, and distinguish failed nonnull binds from intentional null pixel shaders. Add opt-in offline pack export and transactional runtime publication tied to exact renderer/compiler provenance. Preserve existing archive pointers and successful bindings while allowing missing resources to recover without a restart. Validation: production archive, guarded parsing, capture deduplication and lookup fixtures passed in the 51-check portable suite; the native Windows build passed. Actual exported packs were accepted by the production parser and generation 1 loaded in a running game. Campaign-wide shader coverage remains incomplete. --- rexlego/src/gpu_native/draw.cpp | 16 +- rexlego/src/gpu_native/shader_archive.cpp | 316 ++++++++++++++++-- rexlego/src/gpu_native/shader_archive.h | 21 +- rexlego/src/gpu_native/shader_container.h | 32 ++ rexlego/src/gpu_native/shaders.cpp | 275 +++++++++++---- rexlego/src/gpu_native/shaders.h | 4 +- tools/gpu-aot/PRECOMPILED-PACKS.md | 54 +++ tools/gpu-aot/export_precompiled_pack.py | 266 +++++++++++++++ .../gpu-aot/test_placement_capture_lookup.py | 2 +- tools/gpu-aot/test_precompiled_pack.py | 314 +++++++++++++++++ tools/gpu-aot/test_shader_bounds.py | 174 ++++++++++ tools/gpu-aot/test_shader_capture_dedup.py | 246 ++++++++++++++ tools/gpu-aot/test_shader_lookup_hashes.py | 25 +- 13 files changed, 1622 insertions(+), 123 deletions(-) create mode 100644 rexlego/src/gpu_native/shader_container.h create mode 100644 tools/gpu-aot/PRECOMPILED-PACKS.md create mode 100644 tools/gpu-aot/export_precompiled_pack.py create mode 100644 tools/gpu-aot/test_precompiled_pack.py create mode 100644 tools/gpu-aot/test_shader_bounds.py create mode 100644 tools/gpu-aot/test_shader_capture_dedup.py diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp index 446370b..ebf2840 100644 --- a/rexlego/src/gpu_native/draw.cpp +++ b/rexlego/src/gpu_native/draw.cpp @@ -673,8 +673,20 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, commands->setFramebuffer(framebuffer); PipelineKey key; - key.vertex_shader = BoundShaderAddress(ShaderStage::kVertex); - key.pixel_shader = BoundShaderAddress(ShaderStage::kPixel); + bool vertex_binding_failed = false; + key.vertex_shader = BoundShaderAddress(ShaderStage::kVertex, &vertex_binding_failed); + bool pixel_binding_failed = false; + key.pixel_shader = BoundShaderAddress(ShaderStage::kPixel, &pixel_binding_failed); + // Failed nonnull binds must not reuse a cached PSO or turn a requested + // pixel shader into a depth-only pass. Check before shader resolution. + if (vertex_binding_failed) { + LongProbeEvent("draw_missing_vertex_shader", true, "guest_vs=", key.vertex_shader); + return false; + } + if (pixel_binding_failed) { + LongProbeEvent("draw_missing_pixel_shader", true, "guest_ps=", key.pixel_shader); + return false; + } key.vertex_shader_hash = BoundShaderHash(ShaderStage::kVertex); key.pixel_shader_hash = BoundShaderHash(ShaderStage::kPixel); key.declaration_hash = declaration.content_hash; diff --git a/rexlego/src/gpu_native/shader_archive.cpp b/rexlego/src/gpu_native/shader_archive.cpp index 1e5a9e5..46c9793 100644 --- a/rexlego/src/gpu_native/shader_archive.cpp +++ b/rexlego/src/gpu_native/shader_archive.cpp @@ -7,8 +7,17 @@ */ #include "gpu_native/shader_archive.h" +#include "gpu_native/shader_container.h" #include +#include +#include +#include +#include +#include +#include +#include +#include "native_gpu_build_info.h" #include #include #include @@ -26,34 +35,11 @@ namespace legodimensions::gpu_native { -namespace { - -uint32_t LoadBigEndianU32(const uint8_t* bytes) { - uint32_t value = 0; - for (size_t i = 0; i != sizeof(value); ++i) { - value = (value << 8) | bytes[i]; - } - return value; -} - -} // namespace - -size_t ShaderContainerByteLength(const void* shader_container) { - const auto* bytes = static_cast(shader_container); - const uint32_t virtual_size = LoadBigEndianU32(bytes + sizeof(uint32_t)); - const uint32_t physical_size = LoadBigEndianU32(bytes + 2 * sizeof(uint32_t)); - constexpr uint32_t kMaximumSectionSize = 4 * 1024 * 1024; - if (virtual_size > kMaximumSectionSize || physical_size > kMaximumSectionSize) { - return size_t{kMaximumSectionSize} * 2 + sizeof(uint32_t); - } - // Archive containers repeat physical_size at the section boundary, while - // shaders built by the game's runtime compiler omit that marker. - const uint32_t boundary = LoadBigEndianU32(bytes + virtual_size); - const size_t padding = boundary == physical_size ? sizeof(uint32_t) : 0; - return size_t{virtual_size} + padding + size_t{physical_size}; +size_t ShaderContainerByteLength(const void* shader_container, size_t readable_size) { + return BoundedShaderContainerByteLength(shader_container, readable_size); } -const ShaderCacheEntry* FindShader(uint64_t hash) { +static const ShaderCacheEntry* FindBuiltinShader(uint64_t hash) { const auto find_in = [hash](ShaderCacheEntry* begin, size_t count) { ShaderCacheEntry* const end = begin + count; ShaderCacheEntry* const result = std::lower_bound( @@ -68,6 +54,229 @@ const ShaderCacheEntry* FindShader(uint64_t hash) { return find_in(g_runtimeShaderCacheEntries, g_runtimeShaderCacheEntryCount); } + +namespace { +constexpr size_t kPackHeaderBytes = 160; +constexpr size_t kMaximumPackBytes = 64 * 1024 * 1024; +constexpr size_t kMaximumPublishedBytes = 256 * 1024 * 1024; +constexpr std::string_view kPackCommonHeader = + "11fb0c3e07bd0d8cf4930a73035f98ab623f9877c3d167e82b0c0fccf56f1a9a"; +constexpr std::string_view kPackCompilerRevision = + "339af41df2c23dbe3256c1c377716b81a0e0fe6b"; +struct PrecompiledBank { + std::vector shaders; + std::vector stages; + std::vector texture_masks; + std::vector microcodes; + std::vector variants; + std::vector dxil; +}; +std::mutex g_pack_mutex; +std::vector> g_packs; +std::unordered_map> g_pack_shaders; +size_t g_published_pack_bytes = 0; +std::atomic g_pack_generation{0}; + +uint32_t PackU32(const uint8_t* p) { + return uint32_t(p[0]) | uint32_t(p[1]) << 8 | + uint32_t(p[2]) << 16 | uint32_t(p[3]) << 24; +} +uint64_t PackU64(const uint8_t* p) { + return PackU32(p) | uint64_t(PackU32(p + 4)) << 32; +} +bool ValidPackDxil(std::span bytes, uint32_t stage) { + if (bytes.size() < 36 || std::memcmp(bytes.data(), "DXBC", 4) || + PackU32(bytes.data() + 24) != bytes.size()) return false; + const uint32_t chunks = PackU32(bytes.data() + 28); + if (!chunks || chunks > 64 || chunks > (bytes.size() - 32) / 4) return false; + bool dxil = false; + std::vector> ranges; + for (uint32_t i = 0; i < chunks; ++i) { + const size_t offset = PackU32(bytes.data() + 32 + i * 4); + if (offset < 32 + size_t(chunks) * 4 || offset > bytes.size() - 8) return false; + const size_t size = PackU32(bytes.data() + offset + 4); + if (size > bytes.size() - offset - 8) return false; + ranges.emplace_back(offset, offset + 8 + size); + if (!std::memcmp(bytes.data() + offset, "DXIL", 4)) { + // DXIL program header: ShaderKind/version, word count, DXIL magic, + // DXIL version, bitcode offset and bitcode size. + if (size < 24 || std::memcmp(bytes.data() + offset + 16, "DXIL", 4)) return false; + const uint32_t program = PackU32(bytes.data() + offset + 8); + if ((program >> 16) != (stage ? 0u : 1u) || + ((program >> 4) & 0xFu) != 6u || + uint64_t(PackU32(bytes.data() + offset + 12)) * 4 != size) return false; + const uint32_t bitcode_offset = PackU32(bytes.data() + offset + 24); + const uint32_t bitcode_size = PackU32(bytes.data() + offset + 28); + if (bitcode_offset < 16 || bitcode_offset > size - 8 || + bitcode_size < 4 || bitcode_size > size - 8 - bitcode_offset || + std::memcmp(bytes.data() + offset + 16 + bitcode_offset, "BC\xC0\xDE", 4)) return false; + dxil = true; + } + } + std::sort(ranges.begin(), ranges.end()); + for (size_t i = 1; i < ranges.size(); ++i) + if (ranges[i].first < ranges[i - 1].second) return false; + return dxil; +} +} // namespace + +const ShaderCacheEntry* FindShader(uint64_t hash) { + if (const auto* builtin = FindBuiltinShader(hash)) return builtin; + if (!PrecompiledShaderPackGeneration()) return nullptr; + std::lock_guard lock(g_pack_mutex); + const auto it = g_pack_shaders.find(hash); + return it == g_pack_shaders.end() ? nullptr : + &it->second.first->shaders[it->second.second]; +} + +uint64_t PrecompiledShaderPackGeneration() { + return g_pack_generation.load(std::memory_order_acquire); +} + +bool LoadPrecompiledShaderPack(std::span bytes, std::string& error) { + const auto fail = [&](const char* message) { error = message; return false; }; + error.clear(); + if (bytes.size() < kPackHeaderBytes || bytes.size() > kMaximumPackBytes) + return fail("pack size outside bounds"); + if (std::memcmp(bytes.data(), "LEGODX1\0", 8) || PackU32(bytes.data() + 8) != 1 || + PackU32(bytes.data() + 12) != 624) + return fail("pack format or constants ABI mismatch"); + const uint32_t ns = PackU32(bytes.data() + 16); + const uint32_t nm = PackU32(bytes.data() + 20); + const uint32_t nv = PackU32(bytes.data() + 24); + const uint32_t nb = PackU32(bytes.data() + 28); + if (!ns || ns > 256 || !nm || nm > 2048 || !nv || nv > 1024) + return fail("pack table count outside bounds"); + if (std::string_view(reinterpret_cast(bytes.data() + 32), 16) != + kNativeGpuBuildFingerprint || + std::string_view(reinterpret_cast(bytes.data() + 48), 64) != + kPackCommonHeader || + std::string_view(reinterpret_cast(bytes.data() + 112), 40) != + kPackCompilerRevision) + return fail("pack build or compiler provenance mismatch"); + const uint64_t tables_end = kPackHeaderBytes + uint64_t(ns) * 24 + + uint64_t(nm) * 40 + uint64_t(nv) * 24; + if (tables_end + nb != bytes.size() || !nb) + return fail("pack table/payload size mismatch"); + if (XXH3_64bits(bytes.data() + kPackHeaderBytes, + bytes.size() - kPackHeaderBytes) != PackU64(bytes.data() + 152)) + return fail("pack checksum mismatch"); + auto bank = std::make_unique(); + size_t at = kPackHeaderBytes; + std::unordered_map shader_indexes; + for (uint32_t i = 0; i < ns; ++i, at += 24) { + const auto* p = bytes.data() + at; + const uint64_t hash = PackU64(p); + const uint32_t stage = PackU32(p + 8), mask = PackU32(p + 12); + const uint32_t texture_mask = PackU32(p + 16); + if (!hash || stage > 1 || (mask & ~(stage ? 2u : 1u)) || + PackU32(p + 20) || (i && hash <= bank->shaders.back().hash)) + return fail("invalid or unsorted shader metadata"); + shader_indexes.emplace(hash, i); + bank->shaders.push_back({hash, 0, 0, 0, 0, mask}); + bank->stages.push_back(stage); + bank->texture_masks.push_back(texture_mask); + } + std::vector masks(ns, 0), micro_counts(ns, 0); + using MicroKey = std::tuple; + std::set micro_keys; + for (uint32_t i = 0; i < nm; ++i, at += 40) { + const auto* p = bytes.data() + at; + const uint64_t prefix = PackU64(p), physical = PackU64(p + 8), hash = PackU64(p + 16); + const uint32_t size = PackU32(p + 24), stage = PackU32(p + 28), texture = PackU32(p + 32); + const auto it = shader_indexes.find(hash); + if (it == shader_indexes.end() || stage != bank->stages[it->second] || + size < 8 || size > 4 * 1024 * 1024 || PackU32(p + 36) || + !micro_keys.emplace(stage, size, physical).second) + return fail("invalid or ambiguous microcode metadata"); + masks[it->second] |= texture; + ++micro_counts[it->second]; + bank->microcodes.push_back({prefix, physical, hash, size, stage, texture}); + } + for (size_t i = 0; i < ns; ++i) + if (!micro_counts[i] || masks[i] != bank->texture_masks[i]) + return fail("missing microcode or texture metadata mismatch"); + std::vector> specs(ns); + size_t next_payload_offset = 0; + for (uint32_t i = 0; i < nv; ++i, at += 24) { + const auto* p = bytes.data() + at; + const uint64_t hash = PackU64(p); + const uint32_t spec = PackU32(p + 8), offset = PackU32(p + 12), size = PackU32(p + 16); + const auto it = shader_indexes.find(hash); + if (it == shader_indexes.end() || (spec & ~bank->shaders[it->second].specConstantsMask) || + PackU32(p + 20) || offset != next_payload_offset || !size || + uint64_t(offset) + size > nb || + (i && std::pair{hash, spec} <= std::pair{bank->variants.back().hash, + bank->variants.back().specConstants}) || + !ValidPackDxil(bytes.subspan(size_t(tables_end) + offset, size), + bank->stages[it->second])) + return fail("invalid or unsorted DXIL variant"); + specs[it->second].push_back(spec); + bank->variants.push_back({hash, spec, offset, size}); + next_payload_offset += size; + } + if (next_payload_offset != nb) return fail("unreferenced DXIL payload bytes"); + for (size_t i = 0; i < ns; ++i) { + const uint32_t mask = bank->shaders[i].specConstantsMask; + if (specs[i] != (mask ? std::vector{0, mask} : std::vector{0})) + return fail("incomplete specialization variants"); + } + // Publish only after complete validation. Existing pointers never move and + // conflicting built-in or previously published identities are rejected. + std::lock_guard lock(g_pack_mutex); + if (g_packs.size() >= 64 || bytes.size() > kMaximumPublishedBytes - g_published_pack_bytes) + return fail("process pack lifetime budget exceeded"); + for (const auto& shader : bank->shaders) + if (FindBuiltinShader(shader.hash) || g_pack_shaders.contains(shader.hash)) + return fail("pack attempts to replace an existing shader"); + const auto conflicts = [&](const ShaderMicrocodeEntry& m) { + return micro_keys.contains(MicroKey{m.stage, m.microcodeSize, m.microcodeHash}); + }; + for (size_t i = 0; i < g_shaderMicrocodeEntryCount; ++i) + if (conflicts(g_shaderMicrocodeEntries[i])) return fail("built-in microcode identity conflict"); + for (const auto& existing : g_packs) + for (const auto& m : existing->microcodes) + if (conflicts(m)) return fail("published microcode identity conflict"); + bank->dxil.assign(bytes.begin() + size_t(tables_end), bytes.end()); + auto* published = bank.get(); + // Build the replacement index before changing visibility: unordered_map + // insertion can allocate even after reserve. Any failure leaves publication + // and its generation unchanged. + auto next_index = g_pack_shaders; + next_index.reserve(next_index.size() + ns); + for (size_t i = 0; i < published->shaders.size(); ++i) + next_index.emplace(published->shaders[i].hash, std::pair{published, i}); + g_packs.reserve(g_packs.size() + 1); + g_packs.push_back(std::move(bank)); + g_pack_shaders.swap(next_index); + g_published_pack_bytes += bytes.size(); + g_pack_generation.fetch_add(1, std::memory_order_release); + return true; +} + +uint32_t FindShaderPhysicalSize(uint64_t hash, uint32_t stage) { + uint32_t size = 0; + const auto visit = [&](const ShaderMicrocodeEntry& e) { + if (e.containerHash == hash && e.stage == stage) size = std::max(size, e.microcodeSize); + }; + for (size_t i = 0; i < g_shaderMicrocodeEntryCount; ++i) visit(g_shaderMicrocodeEntries[i]); + std::lock_guard lock(g_pack_mutex); + for (const auto& bank : g_packs) for (const auto& e : bank->microcodes) visit(e); + return size; +} + +std::unordered_set FindPixelShaderContainers( + const std::unordered_set& physical_hashes) { + std::unordered_set result; + const auto visit = [&](const ShaderMicrocodeEntry& e) { + if (e.stage == 1 && physical_hashes.contains(e.microcodeHash)) result.insert(e.containerHash); + }; + for (size_t i = 0; i < g_shaderMicrocodeEntryCount; ++i) visit(g_shaderMicrocodeEntries[i]); + std::lock_guard lock(g_pack_mutex); + for (const auto& bank : g_packs) for (const auto& e : bank->microcodes) visit(e); + return result; +} + uint32_t FindShaderTextureMask(uint64_t hash) { static const auto masks = [] { std::unordered_map result; @@ -78,12 +287,36 @@ uint32_t FindShaderTextureMask(uint64_t hash) { return result; }(); const auto it = masks.find(hash); - return it == masks.end() ? ~uint32_t{0} : it->second; + if (it != masks.end()) return it->second; + if (FindBuiltinShader(hash)) return ~uint32_t{0}; + std::lock_guard lock(g_pack_mutex); + const auto overlay = g_pack_shaders.find(hash); + return overlay == g_pack_shaders.end() ? ~uint32_t{0} : + overlay->second.first->texture_masks[overlay->second.second]; +} + +static const ShaderCacheEntry* FindPrecompiledByMicrocode( + const void* microcode, uint32_t stage, size_t readable_size, uint64_t prefix, + std::unordered_map& hashes_by_size) { + if (!PrecompiledShaderPackGeneration()) return nullptr; + { + std::lock_guard lock(g_pack_mutex); + for (const auto& bank : g_packs) for (const auto& entry : bank->microcodes) { + if (entry.stage != stage || entry.prefixHash != prefix || entry.microcodeSize > readable_size) continue; + auto [hash, inserted] = hashes_by_size.try_emplace(entry.microcodeSize, 0); + if (inserted) hash->second = XXH3_64bits(microcode, entry.microcodeSize); + if (hash->second == entry.microcodeHash) { + const auto shader = g_pack_shaders.find(entry.containerHash); + return &shader->second.first->shaders[shader->second.second]; + } + } + } + return nullptr; } const ShaderCacheEntry* FindShaderByMicrocode(const void* microcode, - uint32_t stage) { - if (!microcode) { + uint32_t stage, size_t readable_size) { + if (!microcode || readable_size < 8) { return nullptr; } using Index = std::unordered_multimap; @@ -108,9 +341,11 @@ const ShaderCacheEntry* FindShaderByMicrocode(const void* microcode, // candidate on every bind. Reuse hashes only within THIS lookup, keyed by // the exact byte length; never cache across guest writes or bind calls. std::unordered_map hashes_by_size; + if (const auto* overlay = FindPrecompiledByMicrocode( + microcode, stage, readable_size, prefix, hashes_by_size)) return overlay; for (auto it = first; it != last; ++it) { const ShaderMicrocodeEntry* entry = it->second; - if (entry->stage != stage) continue; + if (entry->stage != stage || entry->microcodeSize > readable_size) continue; auto [hash, inserted] = hashes_by_size.try_emplace(entry->microcodeSize, 0); if (inserted) hash->second = XXH3_64bits(microcode, entry->microcodeSize); if (hash->second == entry->microcodeHash) { @@ -120,16 +355,25 @@ const ShaderCacheEntry* FindShaderByMicrocode(const void* microcode, return nullptr; } -uint64_t HashShaderContainer(const void* shader_container) { - if (!shader_container) { - return 0; - } - return XXH3_64bits(shader_container, ShaderContainerByteLength(shader_container)); +uint64_t HashShaderContainer(const void* shader_container, size_t readable_size) { + const size_t byte_length = ShaderContainerByteLength(shader_container, readable_size); + return byte_length ? XXH3_64bits(shader_container, byte_length) : 0; } ShaderBytecode FindDxil(uint64_t hash, uint32_t spec_constants) { - const ShaderCacheEntry* const shader = FindShader(hash); + const ShaderCacheEntry* const shader = FindBuiltinShader(hash); if (!shader) { + std::lock_guard lock(g_pack_mutex); + const auto it = g_pack_shaders.find(hash); + if (it != g_pack_shaders.end()) { + const auto& bank = *it->second.first; + const uint32_t masked = spec_constants & bank.shaders[it->second.second].specConstantsMask; + const auto key = std::pair{hash, masked}; + const auto variant = std::lower_bound(bank.variants.begin(), bank.variants.end(), key, + [](const LinkedShaderCacheEntry& e, const auto& k) { return std::pair{e.hash, e.specConstants} < k; }); + if (variant == bank.variants.end() || std::pair{variant->hash, variant->specConstants} != key) return {}; + return {bank.dxil.data() + variant->dxilOffset, variant->dxilSize}; + } REXLOG_WARN("Native GPU shader cache miss: hash=0x{:016X}", hash); return {}; } diff --git a/rexlego/src/gpu_native/shader_archive.h b/rexlego/src/gpu_native/shader_archive.h index fdb54a8..af95943 100644 --- a/rexlego/src/gpu_native/shader_archive.h +++ b/rexlego/src/gpu_native/shader_archive.h @@ -10,6 +10,9 @@ #include #include +#include +#include +#include // XenosRecomp emits these names in the global namespace. struct ShaderCacheEntry { @@ -65,21 +68,33 @@ struct ShaderBytecode { // Archive containers repeat physical_size between their virtual and physical // programs. Runtime-created containers omit that marker; both layouts occur in // LEGO Dimensions and are detected from the container contents. -size_t ShaderContainerByteLength(const void* shader_container); +// readable_size is the validated source span; malformed/truncated inputs return 0. +size_t ShaderContainerByteLength(const void* shader_container, size_t readable_size); const ShaderCacheEntry* FindShader(uint64_t hash); uint32_t FindShaderTextureMask(uint64_t hash); // Matches an XDK placement shader, which retains only its physical microcode // pointer, back to the original container compiled into the AOT archive. +// Prefix and candidate hashes are restricted to readable_size source bytes. const ShaderCacheEntry* FindShaderByMicrocode(const void* microcode, - uint32_t stage); + uint32_t stage, size_t readable_size); // Hashes the exact Xbox container span used by the AOT generator. -uint64_t HashShaderContainer(const void* shader_container); +uint64_t HashShaderContainer(const void* shader_container, size_t readable_size); // Returns complete DXIL ready for D3D12. Specialization-library entries are // resolved exclusively through the build-time linked archive. ShaderBytecode FindDxil(uint64_t hash, uint32_t spec_constants); +// Development-only additive packs. Published entries and bytecode live until +// process exit; existing shaders are never replaced. The caller serializes +// publication and binding recovery with HostDevice's recording lock. +bool LoadPrecompiledShaderPack(std::span bytes, + std::string& error); +uint64_t PrecompiledShaderPackGeneration(); +uint32_t FindShaderPhysicalSize(uint64_t hash, uint32_t stage); +std::unordered_set FindPixelShaderContainers( + const std::unordered_set& physical_hashes); + } // namespace legodimensions::gpu_native diff --git a/rexlego/src/gpu_native/shader_container.h b/rexlego/src/gpu_native/shader_container.h new file mode 100644 index 0000000..9e3c279 --- /dev/null +++ b/rexlego/src/gpu_native/shader_container.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +namespace legodimensions::gpu_native { + +// The source may end at a guest allocation boundary. Never inspect a section +// marker until its four bytes are within the caller's readable span. +inline size_t BoundedShaderContainerByteLength(const void* container, + size_t readable_size) { + constexpr size_t kHeaderSize = 0x24; + constexpr size_t kMaximumSectionSize = 4 * 1024 * 1024; + if (!container || readable_size < kHeaderSize) return 0; + const auto* bytes = static_cast(container); + const auto read_word = [&](size_t offset) { + uint32_t word = 0; + for (size_t i = 0; i < 4; ++i) word = (word << 8) | bytes[offset + i]; + return word; + }; + const size_t virtual_size = read_word(4); + const size_t physical_size = read_word(8); + if (virtual_size < kHeaderSize || virtual_size > kMaximumSectionSize || + physical_size > kMaximumSectionSize || virtual_size > readable_size || + physical_size > readable_size - virtual_size) return 0; + const size_t remaining = readable_size - virtual_size; + const size_t marker_size = remaining >= 4 && read_word(virtual_size) == physical_size ? 4 : 0; + if (marker_size > remaining || physical_size > remaining - marker_size) return 0; + return virtual_size + marker_size + physical_size; +} + +} // namespace legodimensions::gpu_native diff --git a/rexlego/src/gpu_native/shaders.cpp b/rexlego/src/gpu_native/shaders.cpp index 97aec0a..53f8985 100644 --- a/rexlego/src/gpu_native/shaders.cpp +++ b/rexlego/src/gpu_native/shaders.cpp @@ -52,6 +52,9 @@ struct ShaderResource { std::mutex g_registry_mutex; std::unordered_map> g_registry; std::array, 2> g_bound_shaders; +// A failed nonnull bind is distinct from an intentional null pixel shader. +// Keep the guest request even when no AOT resource could be adopted. +std::array g_bound_shader_addresses{}; std::atomic g_shader_warning_count{0}; size_t StageIndex(ShaderStage stage) { @@ -72,7 +75,7 @@ void DumpMissingShader(const ShaderContainer* container, size_t byte_length, ShaderStage stage, u64 hash, const char* category = ""); bool ReadableGuestRange(u32 address, u32 size) { - if (!size || uint64_t(address) + size > 0x100000000ull) return false; + if (!address || !size || uint64_t(address) + size > 0x100000000ull) return false; auto* memory = REX_KERNEL_MEMORY(); for (uint64_t at = address; at < uint64_t(address) + size;) { auto* heap = memory->LookupHeap(static_cast(at)); @@ -85,6 +88,106 @@ bool ReadableGuestRange(u32 address, u32 size) { return true; } +// Query page-aligned addresses: QueryRegionInfo counts whole pages, even when +// its input is not aligned. Bound every result by the original allocation. +size_t ReadableGuestSpan(u32 address) { + if (!address) return 0; + auto* memory = REX_KERNEL_MEMORY(); + auto* heap = memory->LookupHeap(address); + if (!heap) return 0; + const u32 page_size = heap->page_size(); + if (!page_size || (page_size & (page_size - 1))) return 0; + const u32 page_mask = page_size - 1; + const u32 heap_base = heap->heap_base(); + if (address < heap_base) return 0; + const auto page_start = [&](u32 at) { return heap_base + ((at - heap_base) & ~page_mask); }; + rex::memory::HeapAllocationInfo initial{}; + if (!heap->QueryRegionInfo(page_start(address), &initial) || + !(initial.state & rex::memory::kMemoryAllocationCommit) || + !(initial.protect & rex::memory::kMemoryProtectRead)) return 0; + const uint64_t allocation_end = uint64_t(initial.allocation_base) + initial.allocation_size; + // Creation already rejects containers larger than 4 MiB. The extra word + // allows checking the archive marker at the end of that bounded window. + const uint64_t limit = std::min({allocation_end, uint64_t{0x100000000ull}, + uint64_t(address) + 4 * 1024 * 1024 + 4}); + uint64_t at = address; + while (at < limit) { + const u32 page = page_start(u32(at)); + rex::memory::HeapAllocationInfo info{}; + if (!heap->QueryRegionInfo(page, &info) || + info.allocation_base != initial.allocation_base || + !(info.state & rex::memory::kMemoryAllocationCommit) || + !(info.protect & rex::memory::kMemoryProtectRead)) break; + const uint64_t end = std::min(limit, uint64_t(page) + info.region_size); + if (end <= at) break; + at = end; + } + return at > address ? size_t(at - address) : 0; +} + +void CaptureMissingPlacementShader(u32 guest_address, u32 container_address, + const uint8_t* microcode, size_t readable_size, ShaderStage stage) { + static const bool enabled = [] { + char* value = nullptr; + size_t length = 0; + _dupenv_s(&value, &length, "LEGO_DUMP_MISSING_SHADERS"); + const bool result = value && length > 1; + std::free(value); + return result; + }(); + if (!enabled) return; + static std::mutex capture_mutex; + static std::array, 2> captured_containers; + { + std::lock_guard lock(capture_mutex); + if (captured_containers[0].size() + captured_containers[1].size() >= 256) return; + } + // XGSet*ShaderHeader copies the virtual section immediately after the SDK + // object. Validate both sections before reading, hashing or reserving a slot. + const u64 header_offset = u64(guest_address) + + (stage == ShaderStage::kVertex ? kVertexShaderGuestSize : kPixelShaderGuestSize); + if (header_offset > UINT32_MAX) return; + const u32 header_address = static_cast(header_offset); + if (!ReadableGuestRange(header_address, sizeof(ShaderContainer))) return; + const auto* virtual_header = + REX_KERNEL_MEMORY()->TranslateVirtual(header_address); + const u32 flags = virtual_header->flags; + const u32 virtual_size = virtual_header->virtual_size; + const u32 physical_size = virtual_header->physical_size; + if ((flags & 0xFFFFFFFEu) != 0x102A1100u || + (flags & 1u) != (stage == ShaderStage::kVertex ? 1u : 0u) || + virtual_size < sizeof(ShaderContainer) || virtual_size > 65536 || + physical_size < 8 || physical_size > 65536 || physical_size > readable_size || + !ReadableGuestRange(header_address, virtual_size) || + !ReadableGuestRange(container_address, physical_size)) return; + // Preserve the exact virtual/physical bytes and explicit archive marker. + std::vector capture(virtual_size + 4 + physical_size); + std::memcpy(capture.data(), virtual_header, virtual_size); + std::memcpy(capture.data() + virtual_size, + reinterpret_cast(virtual_header) + 8, 4); + std::memcpy(capture.data() + virtual_size + 4, microcode, physical_size); + const u64 hash = XXH3_64bits(capture.data(), capture.size()); + { + std::lock_guard lock(capture_mutex); + // Placement addresses can be reused for different code. Deduplicate the + // complete container and stage, preserving the process-wide capture cap. + if (captured_containers[0].size() + captured_containers[1].size() >= 256 || + !captured_containers[StageIndex(stage)].insert(hash).second) return; + } + // Record the exact identity even when its capture file already exists from + // an earlier run. The validated range and capture budget bound this work. + LongProbeEvent("missing_placement_shader_identity", true, + "stage=", static_cast(stage), "guest=", guest_address, + "physical=", container_address, "virtual_bytes=", virtual_size, + "physical_bytes=", physical_size, "container_hash=", hash, + "physical_hash=", XXH3_64bits(capture.data() + virtual_size + 4, physical_size)); + DumpMissingShader(reinterpret_cast(microcode), + std::min({2048, 4096 - (container_address & 4095), readable_size}), + stage, XXH3_64bits(microcode, 8), "placement-raw"); + DumpMissingShader(reinterpret_cast(capture.data()), + capture.size(), stage, hash, "placement-containers"); +} + std::shared_ptr AdoptShader(u32 guest_address, ShaderStage expected_stage) { if (const auto existing = FindResource(guest_address)) { @@ -93,6 +196,9 @@ std::shared_ptr AdoptShader(u32 guest_address, if (!guest_address) { return nullptr; } + const u32 guest_size = expected_stage == ShaderStage::kVertex ? + kVertexShaderGuestSize : kPixelShaderGuestSize; + if (!ReadableGuestRange(guest_address, guest_size)) return nullptr; auto* memory = REX_KERNEL_MEMORY(); const auto* bytes = memory->TranslateVirtual(guest_address); const auto* header = reinterpret_cast(bytes); @@ -128,60 +234,13 @@ std::shared_ptr AdoptShader(u32 guest_address, // AOT entry before creating any host object. const auto* microcode = memory->TranslateVirtual(container_address); + const size_t readable_size = ReadableGuestSpan(container_address); + if (readable_size < 8) return nullptr; const ShaderCacheEntry* cache_entry = FindShaderByMicrocode( - microcode, expected_stage == ShaderStage::kPixel ? 1u : 0u); + microcode, expected_stage == ShaderStage::kPixel ? 1u : 0u, readable_size); if (!cache_entry) { - // Diagnostic only, opt-in. Bound distinct objects rather than bind calls: - // repeated draws must not exhaust the capture budget on the first pair. - static const bool capture_enabled = [] { - char* value = nullptr; - size_t length = 0; - _dupenv_s(&value, &length, "LEGO_DUMP_MISSING_SHADERS"); - const bool enabled = value && length > 1; - std::free(value); - return enabled; - }(); - static std::mutex capture_mutex; - static std::unordered_set captured_objects; - bool capture_object = false; - if (capture_enabled) { - std::lock_guard lock(capture_mutex); - capture_object = captured_objects.size() < 256 && - captured_objects.insert(guest_address).second; - } - if (capture_object) { - DumpMissingShader(reinterpret_cast(microcode), - std::min(2048, 4096 - (container_address & 4095)), - expected_stage, XXH3_64bits(microcode, 8), "placement-raw"); - // XGSet*ShaderHeader copies the virtual section immediately after the - // SDK object (TU23 83FB7618 / 83FB74F0). Validate both ranges before - // reconstructing a container from the separately allocated sections. - const u32 header_address = guest_address + - (expected_stage == ShaderStage::kVertex ? kVertexShaderGuestSize - : kPixelShaderGuestSize); - const auto* virtual_header = - memory->TranslateVirtual(header_address); - if (ReadableGuestRange(header_address, sizeof(ShaderContainer))) { - const u32 virtual_size = virtual_header->virtual_size; - const u32 physical_size = virtual_header->physical_size; - if ((u32(virtual_header->flags) & 0xFFFFFFFEu) == 0x102A1100u && - virtual_size >= sizeof(ShaderContainer) && virtual_size <= 65536 && - physical_size >= 8 && physical_size <= 65536 && - ReadableGuestRange(header_address, virtual_size) && - ReadableGuestRange(container_address, physical_size)) { - // Preserve the exact two sections, with an explicit archive marker. - // This is a capture, not a substitute shader or a guessed binding. - std::vector capture(virtual_size + 4 + physical_size); - std::memcpy(capture.data(), virtual_header, virtual_size); - std::memcpy(capture.data() + virtual_size, - reinterpret_cast(virtual_header) + 8, 4); - std::memcpy(capture.data() + virtual_size + 4, microcode, physical_size); - DumpMissingShader(reinterpret_cast(capture.data()), - capture.size(), expected_stage, - XXH3_64bits(capture.data(), capture.size()), "placement-containers"); - } - } - } + CaptureMissingPlacementShader(guest_address, container_address, microcode, + readable_size, expected_stage); if (g_shader_warning_count.fetch_add(1, std::memory_order_relaxed) < 16) { const auto* words = reinterpret_cast(microcode); REXLOG_WARN("Native GPU: placement {} shader 0x{:08X} backing " @@ -190,7 +249,8 @@ std::shared_ptr AdoptShader(u32 guest_address, expected_stage == ShaderStage::kVertex ? "vertex" : "pixel", guest_address, container_address, XXH3_64bits(microcode, 8), static_cast(words[0]), static_cast(words[1]), - static_cast(words[2]), static_cast(words[3])); + readable_size >= 12 ? static_cast(words[2]) : 0u, + readable_size >= 16 ? static_cast(words[3]) : 0u); } return nullptr; } @@ -202,12 +262,8 @@ std::shared_ptr AdoptShader(u32 guest_address, resource->cache_entry = cache_entry; resource->owns_guest_memory = false; resource->physical_code_address = container_address; - for (size_t i = 0; i < g_shaderMicrocodeEntryCount; ++i) { - const auto& entry = g_shaderMicrocodeEntries[i]; - if (entry.containerHash == resource->hash && - entry.stage == (expected_stage == ShaderStage::kPixel ? 1u : 0u)) - resource->physical_code_size = std::max(resource->physical_code_size, entry.microcodeSize); - } + resource->physical_code_size = FindShaderPhysicalSize(resource->hash, + expected_stage == ShaderStage::kPixel ? 1u : 0u); { std::lock_guard lock(g_registry_mutex); const auto [it, inserted] = g_registry.emplace(guest_address, resource); @@ -259,11 +315,81 @@ void DumpMissingShader(const ShaderContainer* container, size_t byte_length, } } + +void PollPrecompiledShaders() { + static const auto paths = [] { + std::pair result; + if (const char* value = std::getenv("LEGO_NATIVE_SHADER_PACK")) result.first = value; + if (const char* value = std::getenv("LEGO_NATIVE_SHADER_PACK_TRIGGER")) result.second = value; + return result; + }(); + if (paths.first.empty() || paths.second.empty()) return; + auto recording = HostDevice::LockRecording(); + static uint32_t last_frame = ~uint32_t{0}; + const uint32_t frame = g_probe_frame.load(std::memory_order_relaxed); + if (frame == last_frame) return; + last_frame = frame; + std::error_code filesystem_error; + if (!std::filesystem::exists(paths.second, filesystem_error)) return; + if (!std::filesystem::remove(paths.second, filesystem_error) || filesystem_error) { + REXLOG_WARN("Native GPU: precompiled pack trigger could not be consumed"); + return; + } + try { + std::ifstream input(paths.first, std::ios::binary | std::ios::ate); + const auto size = input.tellg(); + if (!input || size < 160 || size > 64 * 1024 * 1024) { + REXLOG_WARN("Native GPU: precompiled pack file outside size bounds"); + return; + } + std::vector bytes(static_cast(size)); + input.seekg(0); + if (!input.read(reinterpret_cast(bytes.data()), size) || input.peek() != EOF) { + REXLOG_WARN("Native GPU: precompiled pack file changed or could not be read"); + return; + } + std::string error; + if (!LoadPrecompiledShaderPack(bytes, error)) { + REXLOG_WARN("Native GPU: rejected precompiled pack: {}", error); + return; + } + std::array retry{}; + { + std::lock_guard lock(g_registry_mutex); + // Create hooks may have allocated a guest resource before its hash was + // available. Repair these resources as well as failed placement binds. + for (auto& [address, resource] : g_registry) { + if (!resource->cache_entry) { + resource->cache_entry = FindShader(resource->hash); + if (resource->cache_entry) + resource->texture_mask = FindShaderTextureMask(resource->hash); + } + } + for (size_t i = 0; i < retry.size(); ++i) + if (g_bound_shader_addresses[i] && + !g_bound_shaders[i]) + retry[i] = g_bound_shader_addresses[i]; + } + // Release registry and archive locks before adoption. Successful bindings + // remain intact; rebinding is needed only for previously missing resources. + for (size_t i = 0; i < retry.size(); ++i) + if (retry[i]) BindShader(i ? ShaderStage::kPixel : ShaderStage::kVertex, retry[i]); + LongProbeEvent("precompiled_shader_pack_loaded", false, "generation=", + PrecompiledShaderPackGeneration(), "bytes=", bytes.size()); + REXLOG_INFO("Native GPU: loaded additive precompiled pack, generation={}", + PrecompiledShaderPackGeneration()); + } catch (const std::exception& error) { + REXLOG_WARN("Native GPU: precompiled pack failed: {}", error.what()); + } +} + } // namespace bool ShouldSkipBoundPixelShader() { + PollPrecompiledShaders(); // Called under the recording lock; rebuild only when the user changes it. static std::string previous; + static uint64_t previous_generation = 0; static std::unordered_set containers; auto list = rex::cvar::GetFlagByName("skip_pixel_shaders"); // Native mode may not link the Xenos command processor that registers its @@ -272,15 +398,13 @@ bool ShouldSkipBoundPixelShader() { if (!list.empty()) list += ','; list += cheats::kDofPixelShaderHash; } - if (list != previous) { + const uint64_t generation = PrecompiledShaderPackGeneration(); + if (list != previous || generation != previous_generation) { + previous_generation = generation; previous = list; containers.clear(); const auto hashes = ParsePixelShaderFilter(list); - for (size_t i = 0; i < g_shaderMicrocodeEntryCount; ++i) { - const auto& entry = g_shaderMicrocodeEntries[i]; - if (entry.stage == 1 && hashes.contains(entry.microcodeHash)) - containers.insert(entry.containerHash); - } + containers = FindPixelShaderContainers(hashes); REXLOG_INFO("Native GPU: pixel shader filter matches {} AOT containers", containers.size()); } return containers.contains(BoundShaderHash(ShaderStage::kPixel)); @@ -292,7 +416,8 @@ u32 CreateShaderResource(mapped_u32 function, ShaderStage stage) { } const auto* container = reinterpret_cast(function.host_address()); - const size_t byte_length = ShaderContainerByteLength(container); + const size_t readable_size = ReadableGuestSpan(function.guest_address()); + const size_t byte_length = ShaderContainerByteLength(container, readable_size); // Corrupt sizes must never turn a guest pointer into an unbounded host read. if (byte_length < sizeof(ShaderContainer) || byte_length > 4 * 1024 * 1024) { REXLOG_ERROR("Native GPU: rejected {} shader container of {} bytes", @@ -300,7 +425,7 @@ u32 CreateShaderResource(mapped_u32 function, ShaderStage stage) { return 0; } - const u64 hash = HashShaderContainer(container); + const u64 hash = HashShaderContainer(container, readable_size); const ShaderCacheEntry* cache_entry = FindShader(hash); if (!cache_entry) { REXLOG_WARN("Native GPU: shader not present in AOT cache, hash=0x{:016X}", hash); @@ -343,6 +468,7 @@ u32 CreateShaderResource(mapped_u32 function, ShaderStage stage) { } bool BindShader(ShaderStage stage, u32 guest_address) { + auto recording = HostDevice::LockRecording(); std::shared_ptr resource; if (guest_address) { resource = AdoptShader(guest_address, stage); @@ -351,6 +477,7 @@ bool BindShader(ShaderStage stage, u32 guest_address) { "stage=", static_cast(stage)); { std::lock_guard lock(g_registry_mutex); + g_bound_shader_addresses[StageIndex(stage)] = guest_address; g_bound_shaders[StageIndex(stage)].reset(); } if (g_shader_warning_count.fetch_add(1, std::memory_order_relaxed) < 32) { @@ -362,6 +489,7 @@ bool BindShader(ShaderStage stage, u32 guest_address) { } } std::lock_guard lock(g_registry_mutex); + g_bound_shader_addresses[StageIndex(stage)] = guest_address; g_bound_shaders[StageIndex(stage)] = std::move(resource); return true; } @@ -401,10 +529,13 @@ plume::RenderShader* ResolveBoundShader(ShaderStage stage, u32 spec_constants) { return it->second.get(); } -u32 BoundShaderAddress(ShaderStage stage) { +u32 BoundShaderAddress(ShaderStage stage, bool* binding_failed) { + PollPrecompiledShaders(); std::lock_guard lock(g_registry_mutex); - const auto& resource = g_bound_shaders[StageIndex(stage)]; - return resource ? resource->guest_address : 0; + const size_t index = StageIndex(stage); + const u32 address = g_bound_shader_addresses[index]; + if (binding_failed) *binding_failed = address && !g_bound_shaders[index]; + return address; } bool IsNativeShader(u32 guest_address) { @@ -506,10 +637,12 @@ u32 ReleaseNativeShader(u32 guest_address) { } void ResetShaderResources() { + auto recording = HostDevice::LockRecording(); std::vector guest_addresses; { std::lock_guard lock(g_registry_mutex); g_bound_shaders = {}; + g_bound_shader_addresses = {}; guest_addresses.reserve(g_registry.size()); for (const auto& [guest_address, resource] : g_registry) { if (resource->owns_guest_memory) { diff --git a/rexlego/src/gpu_native/shaders.h b/rexlego/src/gpu_native/shaders.h index 1ae2e84..03e70a8 100644 --- a/rexlego/src/gpu_native/shaders.h +++ b/rexlego/src/gpu_native/shaders.h @@ -27,7 +27,9 @@ bool BindShader(ShaderStage stage, u32 guest_address); // Materializes the correct prelinked specialization at pipeline creation time. // A null result marks the containing render pass as unsupported by native GPU. plume::RenderShader* ResolveBoundShader(ShaderStage stage, u32 spec_constants); -u32 BoundShaderAddress(ShaderStage stage); +// Returns the latest guest request, including an unavailable nonnull binding. +// Optional failure status is sampled under the same lock as the address. +u32 BoundShaderAddress(ShaderStage stage, bool* binding_failed = nullptr); u32 BoundShaderTextureMask(ShaderStage stage); u64 BoundShaderHash(ShaderStage stage); // Same microcode skip list used by the existing Graphics/DoF setting. diff --git a/tools/gpu-aot/PRECOMPILED-PACKS.md b/tools/gpu-aot/PRECOMPILED-PACKS.md new file mode 100644 index 0000000..eb28b94 --- /dev/null +++ b/tools/gpu-aot/PRECOMPILED-PACKS.md @@ -0,0 +1,54 @@ +# Additive precompiled shader packs + +The native renderer can load missing shaders compiled offline without restarting +the game. This development feature is disabled by default. It never compiles +shaders at runtime or replaces shaders already present in the executable. + +Set both `LEGO_NATIVE_SHADER_PACK` and `LEGO_NATIVE_SHADER_PACK_TRIGGER` before +launching. Each must name a path readable by the running Windows process. Under +Wine, use the corresponding Windows drive paths. The first names the pack file; +the second names a trigger file that the renderer consumes once. + +Build a strict candidate with `prepare_native_compiler.py`, then export its +additions against the candidate actually compiled into the running executable: + +```sh +python3 tools/gpu-aot/export_precompiled_pack.py \ + --candidate /path/to/new-candidate \ + --baseline /path/to/running-build-candidate \ + --build-info /path/to/build/generated/native_gpu_build_info.h \ + --output /path/to/additions.pack +``` + +For subsequent packs, repeat `--previous-pack /path/to/accepted.pack` for every +pack accepted by this process. Keep those files immutable. Check the export +report, publish the completed pack by an atomic rename, and only then create the +trigger. A rejected pack consumes its trigger; inspect the log before retrying. +Look for `precompiled_shader_pack_loaded` and its generation in the trace, then +verify that the affected draws recover. Acceptance alone does not prove coverage. + +The exporter checks candidate provenance and output hashes. It does not install +the pack, write a trigger, or change the active shader bank. A renderer rebuild +changes the required fingerprint, so packs must be exported again for that build. + +The runtime checks the exact build fingerprint, constants ABI, common shader +header and compiler revision, checksum, complete specialization variants, DXIL +stage and chunk bounds, and physical microcode identities. Publication is atomic; +failed validation leaves the archive unchanged. Published bytecode and lookup +pointers remain valid until process exit, including across resource resets. +Missing bindings can recover; intentionally null pixel bindings remain null. + +Each pack is limited to 64 MiB and 256 shaders. A process accepts at most 64 packs +with a combined input size of 256 MiB. Restart with a rebuilt bank when those +limits are reached. + +Run the production archive and binding fixture without game assets: + +```sh +python3 tools/gpu-aot/test_precompiled_pack.py /tmp/lego-pack-check \ + --compiler clang++ --verify-regression +``` + +The fixture covers malformed and truncated packs, publication rollback, +specialization lookup, stable pointers, binding recovery, and a negative control +that removes the provenance rejection. It does not validate an entire campaign. diff --git a/tools/gpu-aot/export_precompiled_pack.py b/tools/gpu-aot/export_precompiled_pack.py new file mode 100644 index 0000000..bd590e2 --- /dev/null +++ b/tools/gpu-aot/export_precompiled_pack.py @@ -0,0 +1,266 @@ +"""Export an additive development pack from verified offline AOT outputs. + +This compiles the generated C++ tables with a bounded host compiler. It never +compiles guest shaders, edits a bank, installs a pack or touches the game. +""" +import argparse +import ctypes +import ctypes.util +import hashlib +import json +import os +from pathlib import Path +import re +import struct +import subprocess +import tempfile + +COMMON = '11fb0c3e07bd0d8cf4930a73035f98ab623f9877c3d167e82b0c0fccf56f1a9a' +REVISION = '339af41df2c23dbe3256c1c377716b81a0e0fe6b' + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def identity(path): + data = path.read_bytes() + return dict(bytes=len(data), sha256=hashlib.sha256(data).hexdigest()) + + +def load_hash64(root, compiler, scratch): + # Use the pinned XXHash implementation, including on machines without a + # system XXHash library. This small host library contains no game assets. + source = scratch / 'pack-hash.cpp' + source.write_text('#define XXH_INLINE_ALL\n#include \n' + '#include \n' + '#ifdef _WIN32\n#define API __declspec(dllexport)\n' + '#else\n#define API\n#endif\n' + 'extern "C" API uint64_t PackHash(const void* p,size_t n)' + '{return XXH3_64bits(p,n); }\n') + library_path = scratch / ('pack-hash.dll' if os.name == 'nt' else 'pack-hash.so') + subprocess.run([compiler, '-std=c++20', '-shared', + *([] if os.name == 'nt' else ['-fPIC']), + '-I' + str(root / 'rexglue-sdk/thirdparty/xxHash'), + str(source), '-o', str(library_path)], check=True, timeout=45) + library = ctypes.CDLL(str(library_path)) + library.PackHash.argtypes = (ctypes.c_void_p, ctypes.c_size_t) + library.PackHash.restype = ctypes.c_uint64 + return lambda data: library.PackHash(data, len(data)) + + +def verify_provenance(path): + result = json.loads(path.read_text()) + require(result['constants_bytes'] == 624, 'Constants ABI must be624') + require(result['tools']['header']['sha256'] == COMMON, 'Unrecognized common header') + require(result['settings']['XENOS_RECOMP_DXIL_ONLY'] == '1' and + result['settings']['XENOS_RECOMP_LEGO_NATIVE_SCALE'] == '1', + 'Candidate must be DXIL_ONLY and NATIVE_SCALE') + for name in ('runtime-cache.cpp', 'runtime-linked-dxil-cache.cpp', 'lego-microcode-index.cpp'): + expected = result['outputs'][name] + require(identity(Path(expected['path'])) == + {key: expected[key] for key in ('bytes', 'sha256')}, + f'Changed candidate output: {name}') + for tool in ('header', 'compiler', 'dxc', 'linker', 'prelink'): + expected = result['tools'][tool] + require(identity(Path(expected['path'])) == + {key: expected[key] for key in ('bytes', 'sha256')}, + f'Changed candidate tool: {tool}') + return result + + +BRIDGE = r''' +#include "gpu_native/shader_archive.h" +#include "gpu_native/linked_shader_cache.h" +#include +#define MINIZ_HEADER_FILE_ONLY +#include +#include +#include +#include +#include +extern ShaderMicrocodeEntry baselineMicrocodes[]; +extern const size_t baselineMicrocodeCount; +void u32(FILE* f,uint32_t v){for(unsigned i=0;i<4;++i)std::fputc(v>>(8*i),f);} +void u64(FILE* f,uint64_t v){u32(f,v);u32(f,v>>32);} +int main(int argc,char** argv){ + if(argc!=2 || g_dxilCacheDecompressedSize>64*1024*1024 || + g_linkedDxilCacheDecompressedSize>64*1024*1024)return 2; + std::vector raw(g_dxilCacheDecompressedSize),linked(g_linkedDxilCacheDecompressedSize); + if(ZSTD_decompress(raw.data(),raw.size(),g_compressedDxilCache,g_dxilCacheCompressedSize)!=raw.size())return 3; + mz_ulong n=linked.size(); + if(mz_uncompress(linked.data(),&n,g_compressedLinkedDxilCache,g_linkedDxilCacheCompressedSize)!=MZ_OK || n!=linked.size())return 4; + FILE* f=std::fopen(argv[1],"wb");if(!f)return 5; + u32(f,g_shaderCacheEntryCount);u32(f,g_shaderMicrocodeEntryCount); + u32(f,g_linkedShaderCacheEntryCount);u32(f,baselineMicrocodeCount); + u32(f,raw.size());u32(f,linked.size()); + for(size_t i=0;i= 160 and data[:8] == b'LEGODX1\0' and + data[32:48] == fingerprint.encode() and + data[48:112] == COMMON.encode() and data[112:152] == REVISION.encode() and + hash64(data[160:]) == struct.unpack_from(' 0 and voff + vsize <= len(source), 'Generated DXIL outside decoded blob') + code = source[voff:voff+vsize] + require(code[:4] == b'DXBC' and len(code) >= 36 and + struct.unpack_from('hash!=expected) { std::cerr << path << " section " << start << " failed exact lookup\n"; std::abort(); diff --git a/tools/gpu-aot/test_precompiled_pack.py b/tools/gpu-aot/test_precompiled_pack.py new file mode 100644 index 0000000..d8eb4c3 --- /dev/null +++ b/tools/gpu-aot/test_precompiled_pack.py @@ -0,0 +1,314 @@ +"""Exercise the complete production additive archive and binding recovery.""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import struct +import subprocess +import ctypes +import ctypes.util + + +def function(text, signature): + start = text.index(signature) + end = text.index('{', start) + 1 + depth = 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--compiler', default='clang++') + parser.add_argument('--verify-regression', action='store_true') + parser.add_argument('--real-pack', type=Path) + args = parser.parse_args() + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=True) + root = Path(__file__).resolve().parents[2] + spec = importlib.util.spec_from_file_location('exporter', root / 'tools/gpu-aot/export_precompiled_pack.py') + exporter = importlib.util.module_from_spec(spec) + spec.loader.exec_module(exporter) + if os.name == 'posix': + import resource + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + hash64 = exporter.load_hash64(root, args.compiler, args.output) + fingerprint = '0123456789abcdef' + common = exporter.COMMON + revision = exporter.REVISION + def dxil(stage): + # Synthetic DXIL envelopes exercise validation and ownership, not GPU + # execution. No test container is published as a real game shader. + chunk = struct.pack('<6I', (0 if stage else 1) << 16 | 0x60, 7, + 0x4c495844, 0x100, 16, 4) + b'BC\xc0\xde' + size = 36 + 8 + len(chunk) + return b'DXBC' + bytes(16) + struct.pack(' FindResource(')] + bodies = '\n'.join(function(shaders, signature) for signature in ( + 'bool BindShader(', 'void PollPrecompiledShaders(', 'u32 BoundShaderAddress(', + 'u64 BoundShaderHash(', 'void ResetShaderResources(', + 'bool ShouldSkipBoundPixelShader(')) + code = r''' +#include "gpu_native/shader_archive.h" +#include "gpu_native/linked_shader_cache.h" +#include "gpu_native/pixel_shader_filter.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define REXLOG_WARN(...) ((void)0) +#define REXLOG_INFO(...) ((void)0) +ShaderCacheEntry g_shaderCacheEntries[]={{1,0,0,0,0,0}}; +const size_t g_shaderCacheEntryCount=1; +ShaderCacheEntry g_runtimeShaderCacheEntries[]={{2,0,0,0,0,0}}; +const size_t g_runtimeShaderCacheEntryCount=1; +const uint8_t g_compressedDxilCache[]={0},g_runtimeCompressedDxilCache[]={0}; +const size_t g_dxilCacheCompressedSize=0,g_dxilCacheDecompressedSize=0, + g_runtimeDxilCacheCompressedSize=0,g_runtimeDxilCacheDecompressedSize=0; +LinkedShaderCacheEntry g_linkedShaderCacheEntries[]={{1,0,0,0}},g_runtimeLinkedShaderCacheEntries[]={{2,0,0,0}}; +const size_t g_linkedShaderCacheEntryCount=0,g_runtimeLinkedShaderCacheEntryCount=0; +const uint8_t g_compressedLinkedDxilCache[]={0},g_runtimeCompressedLinkedDxilCache[]={0}; +const size_t g_linkedDxilCacheCompressedSize=0,g_linkedDxilCacheDecompressedSize=0, + g_runtimeLinkedDxilCacheCompressedSize=0,g_runtimeLinkedDxilCacheDecompressedSize=0; +ShaderMicrocodeEntry g_shaderMicrocodeEntries[]={{0,0,1,32,0,0}}; +const size_t g_shaderMicrocodeEntryCount=1; +using namespace legodimensions::gpu_native; +using u32=uint32_t;using u64=uint64_t; +enum class ShaderStage:u32{kVertex=6,kPixel=7}; +namespace plume {struct RenderShader{};} +struct HostDevice {static inline std::recursive_mutex recording; + static auto LockRecording(){return std::unique_lock(recording);}}; +struct Memory{void SystemHeapFree(u32){}} memory; +#define REX_KERNEL_MEMORY() (&memory) +std::atomic g_probe_frame{0}; +templatevoid LongProbeEvent(const char*,bool,A&&...){} +''' + registry + r''' +u32 adopts=0; +std::shared_ptr AdoptShader(u32 address,ShaderStage stage){ + ++adopts;assert(address==77&&stage==ShaderStage::kVertex); + auto* shader=FindShader(9);if(!shader)return nullptr; + auto r=std::make_shared();r->guest_address=address;r->stage=stage; + r->cache_entry=shader;r->hash=9;r->owns_guest_memory=false; + r->texture_mask=FindShaderTextureMask(9);g_registry[address]=r;return r; +} +bool BindShader(ShaderStage,u32); +u32 BoundShaderAddress(ShaderStage,bool* = nullptr); +u64 BoundShaderHash(ShaderStage); +std::string skip_list; +namespace rex::cvar {std::string GetFlagByName(const char* name){return std::string(name)=="skip_pixel_shaders"?skip_list:"true";}} +namespace cheats {constexpr const char* kDofPixelShaderHash="";} +''' + bodies + r''' +std::vector read(const std::filesystem::path& p){ + std::ifstream f(p,std::ios::binary);return {std::istreambuf_iterator(f),{}}; +} +int main(int argc,char** argv){ + assert(argc>=3);std::filesystem::path dir=argv[1];std::string error; + auto valid=read(dir/"valid.pack"); + if(std::string(argv[2])=="real"){ + auto real=read(argv[3]);std::memcpy(real.data()+32,"0123456789abcdef",16); + assert(LoadPrecompiledShaderPack(real,error)); + uint32_t n=real[16]|uint32_t(real[17])<<8;assert(n); + for(uint32_t i=0;ispecConstantsMask)assert(FindDxil(h,e->specConstantsMask)); + }return 0; + } + if(std::string(argv[2])=="recovery"){ + assert(!BindShader(ShaderStage::kVertex,77));assert(BindShader(ShaderStage::kPixel,0)); + auto owned=std::make_shared();owned->hash=10;owned->stage=ShaderStage::kPixel; + owned->guest_address=88;g_registry[88]=owned; + bool failed=false;assert(BoundShaderAddress(ShaderStage::kVertex,&failed)==77); + if(std::getenv("LEGO_NATIVE_SHADER_PACK")){ + assert(!failed&&BoundShaderHash(ShaderStage::kVertex)==9&&adopts==2); + assert(owned->cache_entry==FindShader(10)&&owned->texture_mask==5); + assert(BoundShaderAddress(ShaderStage::kPixel)==0); + assert(!std::filesystem::exists(dir/"trigger")); + assert(PrecompiledShaderPackGeneration()==1); + auto* stable=g_bound_shaders[0].get(); + std::ofstream(dir/"trigger").put('x');BoundShaderAddress(ShaderStage::kVertex); + assert(std::filesystem::exists(dir/"trigger")); // no second poll this frame + ++g_probe_frame;BoundShaderAddress(ShaderStage::kVertex); + assert(PrecompiledShaderPackGeneration()==1&&g_bound_shaders[0].get()==stable); + assert(adopts==2); // rejected replacement cannot re-adopt + ResetShaderResources();assert(BoundShaderAddress(ShaderStage::kVertex)==0); + assert(FindShader(9)); // reset cannot invalidate retained pack pointers + }else{assert(failed&&PrecompiledShaderPackGeneration()==0&&adopts==1);} + return 0; + } + for(int i=3;i(pixel_filter)); + skip_list=filter_text;auto filtered=std::make_shared();filtered->hash=10; + g_bound_shaders[1]=filtered;assert(!ShouldSkipBoundPixelShader()); + assert(LoadPrecompiledShaderPack(valid,error)&&error.empty()); + assert(ShouldSkipBoundPixelShader()); // unchanged list must refresh on generation + assert(PrecompiledShaderPackGeneration()==1); + auto* entry=FindShader(9);auto dxil=FindDxil(9,0);assert(entry&&dxil&&FindDxil(9,1)); + assert(FindDxil(9,0x100).data==dxil.data);assert(!FindDxil(200,0)); + assert(FindShaderTextureMask(9)==0x30000&&FindShaderPhysicalSize(9,0)==16); + assert(FindShaderPhysicalSize(9,1)==0); + uint64_t words[]={9,14};auto* bytes=reinterpret_cast(words); + assert(FindShaderByMicrocode(bytes,0,16)==entry); + assert(!FindShaderByMicrocode(bytes,1,16)&&!FindShaderByMicrocode(bytes,0,15)); + // Read the exact pixel physical hash from the validated pack table. + uint64_t pixel=0;for(unsigned i=0;i<8;++i)pixel|=uint64_t(valid[256+i])<<(8*i); + assert(FindPixelShaderContainers({pixel}).contains(10)); + assert(!LoadPrecompiledShaderPack(valid,error));assert(PrecompiledShaderPackGeneration()==1); + assert(!LoadPrecompiledShaderPack(read(dir/"conflict.pack"),error)); + std::atomic done{false};std::thread reader([&]{while(!done){ + assert(FindShader(9)==entry&&FindDxil(9,0).data==dxil.data&&FindShaderTextureMask(9)==0x30000);}}); + assert(LoadPrecompiledShaderPack(read(dir/"next.pack"),error));done=true;reader.join(); + assert(PrecompiledShaderPackGeneration()==2&&FindShader(9)==entry&&FindDxil(9,0).data==dxil.data); + assert(FindShader(20)->specConstantsMask==0&&FindDxil(20,0)&&FindDxil(20,99).data==FindDxil(20,0).data); + for(unsigned i=0;i huge(64*1024*1024+1);assert(!LoadPrecompiledShaderPack(huge,error)); + assert(error=="pack size outside bounds"); +} +''' + cpp = args.output / 'fixture.cpp' + cpp.write_text(code) + flags = ['-std=c++20', '-O0', '-I' + str(stubs), '-I' + str(root / 'rexlego/src'), + '-I' + str(root / 'thirdparty/miniz'), '-I' + str(root / 'thirdparty/zstd'), + '-I' + str(root / 'rexglue-sdk/thirdparty/xxHash')] + decoder = args.output / 'decoder.o' + subprocess.run([args.compiler, '-x', 'c', '-c', str(root / 'thirdparty/zstd/zstddeclib.c'), + '-o', str(decoder)], check=True, timeout=45) + xxhash = args.output / 'xxhash.o' + subprocess.run([args.compiler, '-x', 'c', '-c', str(root / 'rexglue-sdk/thirdparty/xxHash/xxhash.c'), + '-o', str(xxhash)], check=True, timeout=45) + def build(archive, suffix): + exe = args.output / ('fixture' + suffix) + subprocess.run([args.compiler, *flags, str(cpp), str(archive), + str(root / 'thirdparty/miniz/miniz.cpp'), str(decoder), str(xxhash), + '-pthread', '-o', str(exe)], check=True, timeout=45) + return exe + archive = root / 'rexlego/src/gpu_native/shader_archive.cpp' + exe = build(archive, '') + subprocess.run([str(exe), str(args.output), 'archive', *map(str, invalid)], check=True, timeout=10) + if args.real_pack: + subprocess.run([str(exe), str(args.output), 'real', str(args.real_pack.resolve())], check=True, timeout=10) + env = dict(os.environ) + env.pop('LEGO_NATIVE_SHADER_PACK', None) + env.pop('LEGO_NATIVE_SHADER_PACK_TRIGGER', None) + subprocess.run([str(exe), str(args.output), 'recovery'], env=env, check=True, timeout=10) + (args.output / 'trigger').write_text('load') + env.update(LEGO_NATIVE_SHADER_PACK=str(args.output / 'valid.pack'), + LEGO_NATIVE_SHADER_PACK_TRIGGER=str(args.output / 'trigger')) + subprocess.run([str(exe), str(args.output), 'recovery'], env=env, check=True, timeout=10) + negative = None + if args.verify_regression: + bad = args.output / 'bad-archive.cpp' + bad.write_text(archive.read_text().replace('return fail("pack build or compiler provenance mismatch");', ';')) + result = subprocess.run([str(build(bad, '-negative')), str(args.output), 'archive', str(invalid[0])], + capture_output=True, timeout=10) + assert result.returncode != 0 + negative = result.returncode + report = dict(status='passed', production_archive_sha256=hashlib.sha256(archive.read_bytes()).hexdigest(), + production_recovery_sha256=hashlib.sha256(bodies.encode()).hexdigest(), + malformed_cases=len(invalid), truncated_cases=len(valid), + lifecycle_and_concurrent_reads=True, recovery_and_disabled=True, + negative_control_exit=negative, real_pack_validated=bool(args.real_pack), + pack_count_lifetime_budget=64, oversized_file_rejected=True, + actual_skip_filter_generation=True, vertex_texture_slots_16_17=True, texture_bit31=True, + limits='Synthetic DXIL envelopes and mocked guest adoption/host resources; no GPU execution or campaign claim.') + (args.output / 'verification.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/tools/gpu-aot/test_shader_bounds.py b/tools/gpu-aot/test_shader_bounds.py new file mode 100644 index 0000000..5deb7b6 --- /dev/null +++ b/tools/gpu-aot/test_shader_bounds.py @@ -0,0 +1,174 @@ +"""Exercise production shader parsing/lookup next to inaccessible guard pages.""" +import argparse +from pathlib import Path +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('output', type=Path) +p.add_argument('--compiler', default='clang++') +a = p.parse_args() +a.output.mkdir(parents=True, exist_ok=True) +root = Path(__file__).resolve().parents[2] +archive = (root / 'rexlego/src/gpu_native/shader_archive.cpp').read_text() +shaders = (root / 'rexlego/src/gpu_native/shaders.cpp').read_text() + + +def function(source, signature): + start = source.index(signature) + opening = source.index('{', start) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +code = r''' +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#include +#endif +#include "gpu_native/shader_archive.h" +#include "gpu_native/shader_container.h" +using u32=uint32_t; +using namespace legodimensions::gpu_native; + +struct GuardedBytes { + uint8_t* allocation; + uint8_t* bytes; + size_t page, size; + explicit GuardedBytes(size_t n):size(n) { +#ifdef _WIN32 + SYSTEM_INFO info{};GetSystemInfo(&info);page=info.dwPageSize; + allocation=static_cast(VirtualAlloc(nullptr,page*2,MEM_RESERVE|MEM_COMMIT,PAGE_READWRITE)); + assert(allocation);DWORD old=0;assert(VirtualProtect(allocation+page,page,PAGE_NOACCESS,&old)); +#else + page=size_t(sysconf(_SC_PAGESIZE)); + auto* mapped=mmap(nullptr,page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + assert(mapped!=MAP_FAILED);allocation=static_cast(mapped); + assert(mprotect(allocation+page,page,PROT_NONE)==0); +#endif + assert(n<=page);bytes=allocation+page-n;std::memset(bytes,0,n); + } + ~GuardedBytes(){ +#ifdef _WIN32 + VirtualFree(allocation,0,MEM_RELEASE); +#else + munmap(allocation,page*2); +#endif + } + void word(size_t offset,uint32_t value){ + assert(offset+4<=size); + for(size_t i=0;i<4;++i) bytes[offset+i]=uint8_t(value>>(24-8*i)); + } +}; +uint64_t ZeroHash(size_t size) { + uint64_t hash=14695981039346656037ull; + for(size_t i=0;i(pointer); + uint64_t hash=14695981039346656037ull; + for(size_t i=0;i&){return nullptr;} + +''' + function(archive, 'size_t ShaderContainerByteLength(') + '\n' + function( + archive, 'const ShaderCacheEntry* FindShaderByMicrocode(') + '\n' + function( + archive, 'uint64_t HashShaderContainer(') + r''' +} +namespace rex::memory { +constexpr uint32_t kMemoryAllocationCommit=1,kMemoryProtectRead=1; +struct HeapAllocationInfo { + uint32_t allocation_base=0,allocation_size=0,region_size=0,state=0,protect=0; +}; +} +struct FakeHeap { + uint32_t size=4096; + uint32_t page_size()const{return size;} + uint32_t heap_base()const{return size>0x10000?size:0x10000;} + bool QueryRegionInfo(uint32_t page,rex::memory::HeapAllocationInfo* info) { + assert(page%size==0); + const uint32_t base=size>0x10000?size:0x10000; + if(page=base+size*2)return false; + *info={base,size*2,size,1,page==base?1u:0u};return true; + } +}; +struct FakeMemory { + FakeHeap heap; + FakeHeap* LookupHeap(uint32_t address){ + const uint32_t base=heap.size>0x10000?heap.size:0x10000; + return address>=base&&address AdoptShader(') + assert 'CaptureMissingPlacementShader(guest_address, container_address, microcode,' in adopt + assert adopt.index('if (!cache_entry)') < adopt.index('CaptureMissingPlacementShader(') + d3d = (root / 'rexlego/src/gpu_native/d3d.h').read_text() + container = function(d3d, 'struct ShaderContainer') + ';' + fixture = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define XXH_INLINE_ALL +#include +using u32=uint32_t;using u64=uint64_t; +struct be_u32 { + uint8_t bytes[4]{}; + operator u32()const{return (u32(bytes[0])<<24)|(u32(bytes[1])<<16)|(u32(bytes[2])<<8)|bytes[3];} + be_u32& operator=(u32 value){for(int i=0;i<4;++i)bytes[i]=uint8_t(value>>(24-i*8));return *this;} +}; +''' + container + r''' +static_assert(sizeof(ShaderContainer)==36); +enum class ShaderStage{ kVertex=6,kPixel=7 }; +constexpr u32 kVertexShaderGuestSize=872,kPixelShaderGuestSize=40; +''' + function(source, 'size_t StageIndex(') + r''' +int _dupenv_s(char** value,size_t* length,const char* name){ + const char* env=std::getenv(name);*value=nullptr;*length=0; + if(env){*length=std::strlen(env)+1;*value=static_cast(std::malloc(*length));std::memcpy(*value,env,*length);} + return 0; +} +constexpr u32 guest=0x1000,physical=0x10000; +std::vector memory_bytes(0x20000); +std::atomic range_calls=0,translations=0,hash_calls=0; +u32 header_begin=guest+kPixelShaderGuestSize,header_readable=64,physical_readable=32; +bool force_hash=false; +bool mutate_after_container_hash=false; +struct Memory { + templateT TranslateVirtual(u32 at){++translations;assert(at(memory_bytes.data()+at);} +} fixture_memory; +#define REX_KERNEL_MEMORY() (&fixture_memory) +bool ReadableGuestRange(u32 at,u32 size){ + ++range_calls; + const auto contained=[&](u32 start,u32 length){return at>=start&&u64(at)+size<=u64(start)+length;}; + return size&&(contained(header_begin,header_readable)||contained(physical,physical_readable)); +} +u64 RealHash(const void* data,size_t length){return XXH3_64bits(data,length);} +u64 CountedHash(const void* data,size_t length){ + ++hash_calls;assert(length<=65536*2+4);const u64 result=force_hash?7:RealHash(data,length); + if(mutate_after_container_hash&&length==100){memory_bytes[physical+20]^=0x55;mutate_after_container_hash=false;} + return result; +} +struct Captured {ShaderStage stage;std::string category;u64 hash;std::vector bytes;}; +struct Identity {u32 stage,guest,physical,virtual_bytes,physical_bytes;u64 container_hash,physical_hash;}; +std::mutex output_mutex; +std::vector captures; +std::vector identities; +bool existing_files=false; +u32 dump_calls=0; +void LongProbeEvent(const char* event,bool anomaly,const char*,u32 stage, + const char*,u32 guest,const char*,u32 physical,const char*,u32 virtual_bytes, + const char*,u32 physical_bytes,const char*,u64 container_hash,const char*,u64 physical_hash){ + assert(std::string(event)=="missing_placement_shader_identity"&&anomaly); + std::lock_guard lock(output_mutex); + identities.push_back({stage,guest,physical,virtual_bytes,physical_bytes,container_hash,physical_hash}); +} +void DumpMissingShader(const ShaderContainer* bytes,size_t length,ShaderStage stage,u64 hash,const char* category){ + std::lock_guard lock(output_mutex); + ++dump_calls;if(existing_files)return; + const auto* begin=reinterpret_cast(bytes); + captures.push_back({stage,category,hash,{begin,begin+length}}); +} +#undef XXH3_64bits +#define XXH3_64bits CountedHash +''' + body + r''' +#undef XXH3_64bits +ShaderContainer& header(){return *reinterpret_cast(memory_bytes.data()+header_begin);} +void valid(ShaderStage stage=ShaderStage::kPixel,u32 address=guest){ + header_begin=address+(stage==ShaderStage::kVertex?kVertexShaderGuestSize:kPixelShaderGuestSize); + header_readable=64;physical_readable=32; + std::fill(memory_bytes.begin()+header_begin,memory_bytes.begin()+header_begin+64,0xAB); + header().flags=stage==ShaderStage::kVertex?0x102A1101:0x102A1100; + header().virtual_size=64;header().physical_size=32; + for(u32 i=0;i<32;++i)memory_bytes[physical+i]=uint8_t(i+1); +} +void capture(ShaderStage stage=ShaderStage::kPixel,u32 address=guest,size_t readable=32){ + CaptureMissingPlacementShader(address,physical,memory_bytes.data()+physical,readable,stage); +} +size_t complete_count(){size_t count=0;for(auto& entry:captures)count+=entry.category=="placement-containers";return count;} +void exact_identity(){ + const auto& out=identities.back(); + assert(out.stage==u32(ShaderStage::kPixel)&&out.guest==guest&&out.physical==physical); + assert(out.virtual_bytes==64&&out.physical_bytes==32); + std::vector expected(memory_bytes.begin()+header_begin,memory_bytes.begin()+header_begin+64); + expected.insert(expected.end(),memory_bytes.begin()+header_begin+8,memory_bytes.begin()+header_begin+12); + expected.insert(expected.end(),memory_bytes.begin()+physical,memory_bytes.begin()+physical+32); + assert(out.container_hash==RealHash(expected.data(),expected.size())); + assert(out.physical_hash==RealHash(memory_bytes.data()+physical,32)); +} +void exact_last(){ + assert(identities.size()==complete_count());exact_identity(); + const auto& out=captures.back();assert(out.category=="placement-containers"); + assert(out.bytes.size()==100); + assert(std::memcmp(out.bytes.data(),memory_bytes.data()+header_begin,64)==0); + assert(std::memcmp(out.bytes.data()+64,memory_bytes.data()+header_begin+8,4)==0); + assert(std::memcmp(out.bytes.data()+68,memory_bytes.data()+physical,32)==0); + assert(out.hash==RealHash(out.bytes.data(),out.bytes.size())); + assert(captures[captures.size()-2].category=="placement-raw"); + assert(captures[captures.size()-2].bytes.size()==32); +} +int main(int argc,char** argv){ + assert(argc==2);const std::string mode=argv[1];valid(); + if(mode=="identity"){ + capture();assert(complete_count()==1);exact_last();capture();assert(complete_count()==1); + memory_bytes[physical+20]^=0x55;capture(); + assert((complete_count()==2)&&"same guest address with changed valid bytes must capture again");exact_last(); + memory_bytes[header_begin+48]^=0x33;capture();assert(complete_count()==3);exact_last(); + const auto copy=std::vector(memory_bytes.begin()+header_begin,memory_bytes.begin()+header_begin+64); + header_begin=0x2000+kPixelShaderGuestSize; + std::memcpy(memory_bytes.data()+header_begin,copy.data(),copy.size()); + capture(ShaderStage::kPixel,0x2000);assert(complete_count()==3); + }else if(mode=="malformed"){ + header_readable=35;capture();assert(hash_calls==0);valid(); + header().flags=0x102A1110;capture();valid(); + header().flags=0x102A1101;capture();valid(); + header().virtual_size=35;capture();valid(); + header().virtual_size=65537;capture();valid(); + header().virtual_size=65;capture();valid(); + header().physical_size=7;capture();valid(); + header().physical_size=65537;capture();valid(); + capture(ShaderStage::kPixel,guest,31); + physical_readable=31;capture();valid(); + const auto reads=range_calls.load();capture(ShaderStage::kVertex,0xFFFFFF00u); + assert(range_calls==reads);assert(hash_calls==0&&captures.empty()&&identities.empty()); + capture();assert(complete_count()==1);exact_last(); + }else if(mode=="budget"){ + for(u32 i=0;i<256;++i){ + const auto stage=i%2?ShaderStage::kVertex:ShaderStage::kPixel;valid(stage); + std::memcpy(memory_bytes.data()+physical,&i,sizeof(i));capture(stage); + assert(complete_count()==i+1); + } + assert(captures.size()==512&&identities.size()==256);const auto reads=range_calls.load(),hashes=hash_calls.load(); + valid();memory_bytes[physical+20]^=0xCC;capture(); + assert(complete_count()==256&&identities.size()==256&&range_calls==reads&&hash_calls==hashes); + }else if(mode=="stage"){ + force_hash=true;capture();valid(ShaderStage::kVertex);capture(ShaderStage::kVertex); + assert(complete_count()==2);capture(ShaderStage::kVertex);assert(complete_count()==2); + assert(captures[1].stage==ShaderStage::kPixel&&captures[3].stage==ShaderStage::kVertex); + assert(identities.size()==2&&identities[0].stage==u32(ShaderStage::kPixel)&&identities[1].stage==u32(ShaderStage::kVertex)); + }else if(mode=="concurrent"){ + std::vector workers;for(int i=0;i<8;++i)workers.emplace_back([]{for(int j=0;j<20;++j)capture();}); + for(auto& worker:workers)worker.join();assert(complete_count()==1&&captures.size()==2);exact_last(); + }else if(mode=="disabled"){ + capture();assert(captures.empty()&&identities.empty()&&range_calls==0&&translations==0&&hash_calls==0); + }else if(mode=="existing-files"){ + existing_files=true;capture();assert(captures.empty()&&dump_calls==2&&identities.size()==1);exact_identity(); + capture();assert(dump_calls==2&&identities.size()==1); + }else if(mode=="mutating-source"){ + mutate_after_container_hash=true;capture();assert(complete_count()==1&&identities.size()==1); + const auto& full=captures.back();const auto& identity=identities.back(); + assert(identity.container_hash==RealHash(full.bytes.data(),full.bytes.size())); + assert(identity.physical_hash==RealHash(full.bytes.data()+68,32)); + assert(identity.physical_hash!=RealHash(memory_bytes.data()+physical,32)); + }else assert(false); +} +''' + args.output.mkdir(parents=True, exist_ok=True) + executable_suffix = '.exe' if os.name == 'nt' else '' + + def build(name, text): + cpp = args.output / (name + '.cpp') + executable = args.output / (name + executable_suffix) + cpp.write_text(text) + subprocess.run([args.compiler, '-std=c++20', '-pthread', '-UNDEBUG', '-I', + str(root / 'rexglue-sdk/thirdparty/xxHash'), str(cpp), '-o', + str(executable)], check=True, timeout=45) + return executable.resolve() + + executable = build('capture', fixture) + modes = ['identity', 'malformed', 'budget', 'stage', 'concurrent', 'disabled', + 'existing-files', 'mutating-source'] + for mode in modes: + env = dict(os.environ) + env.pop('LEGO_DUMP_MISSING_SHADERS', None) + if mode != 'disabled': + env['LEGO_DUMP_MISSING_SHADERS'] = '1' + subprocess.run([str(executable), mode], env=env, check=True, timeout=10) + regression = 'not requested' + if args.verify_regression: + old = fixture.replace('captured_containers[StageIndex(stage)].insert(hash)', + 'captured_containers[StageIndex(stage)].insert(guest_address)') + assert old != fixture + executable = build('address-dedup', old) + env = dict(os.environ, LEGO_DUMP_MISSING_SHADERS='1') + result = subprocess.run([str(executable), 'identity'], env=env, capture_output=True, + text=True, timeout=10) + assert result.returncode != 0 and 'changed valid bytes must capture again' in result.stderr + regression = 'address-based dedup rejects the second valid container at the same address' + report = { + 'result': 'PASS', 'regression_proof': regression, + 'production_body_sha256': hashlib.sha256(body.encode()).hexdigest(), + 'cases': modes, + 'boundaries': 'capture and stage selection bodies plus container layout are production; ' + 'xxHash is the pinned implementation; guest memory/range queries and dump I/O ' + 'are simulated. Tests verify exact identity events before existing-file ' + 'dump suppression and emitted bytes, not filesystem persistence.', + } + (args.output / 'verification.json').write_text(json.dumps(report, indent=2) + '\n') + print('PASS: placement capture content/stage identity events, exact bytes, validation and 256-container cap') + + +if __name__ == '__main__': + main() diff --git a/tools/gpu-aot/test_shader_lookup_hashes.py b/tools/gpu-aot/test_shader_lookup_hashes.py index 0654c6b..30eed9c 100644 --- a/tools/gpu-aot/test_shader_lookup_hashes.py +++ b/tools/gpu-aot/test_shader_lookup_hashes.py @@ -2,7 +2,8 @@ import argparse from pathlib import Path import subprocess -p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args() +p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path) +p.add_argument('--compiler',default='clang++');a=p.parse_args() a.output.mkdir(parents=True,exist_ok=True) root=Path(__file__).resolve().parents[2] src=(root/'rexlego/src/gpu_native/shader_archive.cpp').read_text() @@ -26,6 +27,9 @@ uint64_t CountHash(const void* p,size_t n){++calls;return XXH3_64bits(p,n);} ShaderCacheEntry hit{9,0,0,0,0,0}; const ShaderCacheEntry* FindShader(uint64_t id){return id==9?&hit:nullptr;} +const ShaderCacheEntry* FindPrecompiledByMicrocode(const void*,uint32_t,size_t,uint64_t, + std::unordered_map&){return nullptr;} + ''' + body + r''' } int main(){ @@ -36,20 +40,23 @@ entries.push_back({prefix,0,100+i,128u<<(i%3),i%2,0}); entries.push_back({prefix,XXH3_64bits(bytes.data(),512),9,512,0,0}); g_shaderMicrocodeEntries=entries.data();g_shaderMicrocodeEntryCount=entries.size(); - assert(FindShaderByMicrocode(bytes.data(),0)==&hit); + assert(FindShaderByMicrocode(bytes.data(),0,bytes.size())==&hit); assert(calls<=4); // Prefix plus at most three lengths, not 1501 hashes. calls=0; - assert(FindShaderByMicrocode(bytes.data(),1)==nullptr);assert(calls==4); + assert(FindShaderByMicrocode(bytes.data(),1,bytes.size())==nullptr);assert(calls==4); bytes[20]^=1;calls=0; - assert(FindShaderByMicrocode(bytes.data(),0)==nullptr);assert(calls==4); + assert(FindShaderByMicrocode(bytes.data(),0,bytes.size())==nullptr);assert(calls==4); bytes[20]^=1;calls=0; - assert(FindShaderByMicrocode(bytes.data(),0)==&hit);assert(calls<=4); - assert(!FindShaderByMicrocode(nullptr,0)); + assert(FindShaderByMicrocode(bytes.data(),0,bytes.size())==&hit);assert(calls<=4); + calls=0; + assert(!FindShaderByMicrocode(bytes.data(),0,7));assert(calls==0); + assert(!FindShaderByMicrocode(bytes.data(),0,127));assert(calls==1); + assert(!FindShaderByMicrocode(nullptr,0,bytes.size())); } ''' code=code.replace('g_shaderMicrocodeEntries','fixtureEntries').replace('g_shaderMicrocodeEntryCount','fixtureCount') cpp=a.output/'lookup.cpp';cpp.write_text(code);exe=a.output/'lookup.exe' -subprocess.run(['clang++','-std=c++20','-I'+str(root/'rexlego/src'), - '-I'+str(root/'rexglue-sdk/thirdparty/xxHash'),str(cpp),'-o',str(exe)],check=True) -subprocess.run([str(exe.resolve())],check=True) +subprocess.run([a.compiler,'-std=c++20','-I'+str(root/'rexlego/src'), + '-I'+str(root/'rexglue-sdk/thirdparty/xxHash'),str(cpp),'-o',str(exe)],check=True,timeout=45) +subprocess.run([str(exe.resolve())],check=True,timeout=10) print('PASS: actual lookup, 3001 colliding prefixes, stage/missing/success and same-address rewrites; <=4 hashes per lookup') From ef176fc90d20aacfbaba08b008c267f8fad18fca Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:43:46 +0200 Subject: [PATCH 03/40] Upload resident guest depth textures before native sampling Convert borrowed single-level UNORM24 textures into immutable R32 sampling mirrors and refresh the bound descriptor after first upload. Preserve host resolve authority and reject failed or unreadable CPU uploads instead of sampling stale storage. Bound 2D and cube block copies, track framebuffer owners for fenced invalidation, cache unchanged authority diagnostics, and allow exact pixel-shader upload capture filtering. Validation: depth upload and binding fixtures passed from the staged commit snapshot; authority views and event sequences were preserved. The complete native build and 51-check portable suite passed. Live first-use atlas upload was verified; campaign-wide rendering remains unverified. --- rexlego/src/gpu_native/draw.cpp | 16 +- rexlego/src/gpu_native/texture_depth_upload.h | 33 ++ rexlego/src/gpu_native/textures.cpp | 261 +++++++++++-- rexlego/src/gpu_native/textures.h | 5 +- tools/gpu-aot/test_framebuffer_owners.py | 171 +++++++++ tools/gpu-aot/test_texture_2d_upload.cpp | 81 ++++ tools/gpu-aot/test_texture_authority_probe.py | 233 ++++++++++++ .../gpu-aot/test_texture_binding_failures.py | 184 +++++++++ tools/gpu-aot/test_texture_depth_upload.py | 353 ++++++++++++++++++ .../gpu-aot/test_texture_source_residency.py | 82 ++++ tools/gpu-aot/test_texture_upload_capture.py | 55 ++- 11 files changed, 1440 insertions(+), 34 deletions(-) create mode 100644 rexlego/src/gpu_native/texture_depth_upload.h create mode 100644 tools/gpu-aot/test_framebuffer_owners.py create mode 100644 tools/gpu-aot/test_texture_2d_upload.cpp create mode 100644 tools/gpu-aot/test_texture_authority_probe.py create mode 100644 tools/gpu-aot/test_texture_binding_failures.py create mode 100644 tools/gpu-aot/test_texture_depth_upload.py create mode 100644 tools/gpu-aot/test_texture_source_residency.py diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp index ebf2840..a9b199d 100644 --- a/rexlego/src/gpu_native/draw.cpp +++ b/rexlego/src/gpu_native/draw.cpp @@ -288,7 +288,7 @@ bool BindConstants( } for (u32 i = 0; i < kNativeTextureSlots; ++i) { if (!(texture_mask & (1u << i))) continue; - const auto texture = ResolveTextureResource(bindings.textures[i]); + auto texture = ResolveTextureResource(bindings.textures[i]); if (!texture.texture || texture.descriptor_index == ~u32{0}) { if (LongProbeEnabled() && bindings.textures[i] && LongProbeOnce(0xBAD1000000000000ull | bindings.textures[i])) @@ -298,8 +298,18 @@ bool BindConstants( } // Vertex textures contain animation data. Until every writer participates // in invalidation, verify their contents at every use, even with watches. - UploadTextureResource(bindings.textures[i], commands, - (BoundShaderTextureMask(ShaderStage::kVertex) & (1u << i)) != 0); + if (!UploadTextureResource(bindings.textures[i], commands, + (BoundShaderTextureMask(ShaderStage::kVertex) & (1u << i)) != 0)) { + // Failed residency/conversion must not sample an old or uninitialized + // allocation. DispatchDraw propagates this failure to active queries. + LongProbeEvent("bound_texture_upload_failed", true, "slot=", i, + "guest=", bindings.textures[i]); + return false; + } + // A first CPU depth upload can publish an R32 sampling mirror. Bind the + // descriptor selected after successful upload, rather than its old DSV. + if (texture.depth && !texture.surface) + texture = ResolveTextureResource(bindings.textures[i]); TextureFetchWords fetch; for (u32 word = 0; word < fetch.size(); ++word) fetch[word] = device->fetch_constants[i].dword[word]; if (LongProbeEnabled()) { diff --git a/rexlego/src/gpu_native/texture_depth_upload.h b/rexlego/src/gpu_native/texture_depth_upload.h new file mode 100644 index 0000000..3527969 --- /dev/null +++ b/rexlego/src/gpu_native/texture_depth_upload.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +#include "depth_alias.h" +#include "texture_volume_upload.h" + +namespace legodimensions::gpu_native { + +// The SDK DepthUnorm loader first applies fetch endianness, discards the +// low eight stencil bits, then normalizes the remaining UNORM24 depth. +// SourceOffset uses the SDK guest layout, including tiled row padding. +template +bool CopyTextureDepth24(std::span source, + std::span destination, + uint32_t width, uint32_t height, uint32_t row_pitch, + uint32_t endian_xor, SourceOffset source_offset) { + if (!CopyTextureVolumeBlocks(source, destination, width, height, 1, 4, + row_pitch, endian_xor, source_offset)) return false; + for (uint32_t y = 0; y < height; ++y) { + for (uint32_t x = 0; x < width; ++x) { + auto* pixel = destination.data() + uint64_t(y) * row_pitch + uint64_t(x) * 4; + uint32_t packed; + std::memcpy(&packed, pixel, sizeof(packed)); + const float depth = UnpackDepth24(packed >> 8); + std::memcpy(pixel, &depth, sizeof(depth)); + } + } + return true; +} + +} // namespace legodimensions::gpu_native diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp index c4f6c70..a339faf 100644 --- a/rexlego/src/gpu_native/textures.cpp +++ b/rexlego/src/gpu_native/textures.cpp @@ -39,6 +39,7 @@ #include "gpu_native/long_probe.h" #include "gpu_native/memory_watch.h" #include "gpu_native/depth_alias.h" +#include "gpu_native/texture_depth_upload.h" namespace legodimensions::gpu_native { namespace { @@ -102,6 +103,10 @@ struct TextureResource { bool upload_source_key_valid = false; CpuMemoryStamp cpu_stamp; bool resolved_on_host = false; + // Diagnostic-only cache, serialized by the caller's LockRecording. Keep the + // global probe deduplication for authority transitions and resource reuse. + u64 last_authority_probe_signature = 0; + bool authority_probe_signature_valid = false; u32 edram_base = ~0u; u32 guest_msaa = 0; u64 write_generation = 0; @@ -114,6 +119,10 @@ std::unique_ptr CreateHostTexture( std::mutex g_textures_mutex; std::unordered_map> g_textures; +// Only these resources can contain cached framebuffer dependencies. Serialized +// with framebuffer creation/invalidation/reset by HostDevice::LockRecording; +// weak pointers never extend texture or framebuffer lifetimes. +std::unordered_map> g_framebuffer_owners; std::atomic g_texture_lifecycle_logs{0}; TextureUploadTiming g_upload_timing; // Serialized by LockRecording. u64 g_depth_alias_generation = 0; // Serialized by LockRecording. @@ -392,12 +401,22 @@ bool UpdateResolvedSampling(TextureResource& resource) { } void InvalidateFramebufferReferences(u32 guest_address) { + auto recording = HostDevice::LockRecording(); std::vector> resources; { std::lock_guard lock(g_textures_mutex); - resources.reserve(g_textures.size()); - for (const auto& [address, resource] : g_textures) { - resources.push_back(resource); + resources.reserve(g_framebuffer_owners.size()); + for (auto it = g_framebuffer_owners.begin(); it != g_framebuffer_owners.end();) { + const auto resource = it->second.lock(); + const auto current = g_textures.find(it->first); + // A released/replaced owner retains its own framebuffers through fenced + // retirement. Only current owners need their attachment keys invalidated. + if (!resource || current == g_textures.end() || current->second != resource) { + it = g_framebuffer_owners.erase(it); + } else { + resources.push_back(resource); + ++it; + } } } for (const auto& resource : resources) { @@ -412,6 +431,8 @@ void InvalidateFramebufferReferences(u32 guest_address) { it = resource->framebuffers.erase(it); } else ++it; } + if (resource->framebuffers.empty()) + g_framebuffer_owners.erase(resource->guest_address); } for (auto& framebuffer : retired) HostDevice::RetireResource(std::move(framebuffer)); } @@ -475,6 +496,8 @@ struct TextureCaptureState { std::string stage; std::unordered_set completed_stage_names; bool all_levels = false, armed = false; + bool pixel_shader_filter_enabled = false, pixel_shader_filter_valid = true; + u64 pixel_shader_filter = 0; u32 checked_frame = ~u32{0}; u64 written_bytes = 0; std::unordered_set dumped; @@ -493,6 +516,26 @@ TextureCaptureState& TextureCapture() { result.trigger = value ? value : ""; std::free(value); result.all_levels = std::getenv("LEGO_DUMP_TEXTURE_UPLOADS_ALL_LEVELS") != nullptr; + value = nullptr; + _dupenv_s(&value, &length, "LEGO_DUMP_TEXTURE_UPLOADS_PIXEL_SHADER"); + if (value) { + result.pixel_shader_filter_enabled = true; + result.pixel_shader_filter_valid = std::strlen(value) == 16; + if (result.pixel_shader_filter_valid) { + for (const char* digit = value; *digit; ++digit) { + u32 nibble; + if (*digit >= '0' && *digit <= '9') nibble = *digit - '0'; + else if (*digit >= 'a' && *digit <= 'f') nibble = *digit - 'a' + 10; + else if (*digit >= 'A' && *digit <= 'F') nibble = *digit - 'A' + 10; + else { result.pixel_shader_filter_valid = false; break; } + result.pixel_shader_filter = (result.pixel_shader_filter << 4) | nibble; + } + } + if (!result.pixel_shader_filter_valid) { + REXLOG_WARN("Native GPU: texture upload capture disabled: LEGO_DUMP_TEXTURE_UPLOADS_PIXEL_SHADER must be exactly 16 hexadecimal digits"); + } + } + std::free(value); return result; }(); return state; @@ -503,6 +546,9 @@ TextureCaptureState& TextureCapture() { bool TextureUploadCapturePending(const TextureResource& resource) { auto& state = TextureCapture(); if (state.root.empty() || state.written_bytes >= 128ull * 1024 * 1024) return false; + if (!state.pixel_shader_filter_valid || + (state.pixel_shader_filter_enabled && + BoundShaderHash(ShaderStage::kPixel) != state.pixel_shader_filter)) return false; if (!state.trigger.empty() && state.checked_frame != g_probe_frame.load()) { state.checked_frame = g_probe_frame.load(); std::error_code error; @@ -689,6 +735,146 @@ std::unique_ptr CreateHostTexture( return texture; } +bool EnsureDepthSamplingMirror(TextureResource& resource) { + if (resource.sampled_texture && resource.sampled_view && + resource.sampled_descriptor_index != ~0u) return true; + plume::RenderTextureDesc desc; + desc.dimension = plume::RenderTextureDimension::TEXTURE_2D; + desc.width = resource.width; desc.height = resource.height; + desc.depth = 1; desc.mipLevels = 1; desc.arraySize = 1; + desc.multisampling.sampleCount = plume::RenderSampleCount::COUNT_1; + desc.format = plume::RenderFormat::R32_FLOAT; + desc.flags |= plume::RenderTextureFlag::RENDER_TARGET; + desc.committed = true; + auto texture = HostDevice::Device()->createTexture(desc); + if (!texture || !static_cast(texture.get())->d3d) return false; + plume::RenderTextureViewDesc view_desc; + view_desc.dimension = plume::RenderTextureViewDimension::TEXTURE_2D; + view_desc.format = plume::RenderFormat::R32_FLOAT; + view_desc.mipLevels = 1; + constexpr plume::RenderSwizzle swizzles[] = { + plume::RenderSwizzle::R, plume::RenderSwizzle::G, + plume::RenderSwizzle::B, plume::RenderSwizzle::A, + plume::RenderSwizzle::ZERO, plume::RenderSwizzle::ONE}; + const u32 swizzle = NativeTextureSwizzle(resource.guest_format, + resource.guest_fetch.swizzle); + view_desc.componentMapping = plume::RenderComponentMapping( + swizzles[swizzle & 7], swizzles[(swizzle >> 3) & 7], + swizzles[(swizzle >> 6) & 7], swizzles[(swizzle >> 9) & 7]); + auto view = texture->createTextureView(view_desc); + if (!view) return false; + const u32 descriptor = HostDevice::RegisterTexture(texture.get(), view.get()); + if (descriptor == ~0u) return false; + // Publish a complete immutable descriptor only after all allocation steps + // succeed. Host resolves reuse this same mirror and remain authoritative. + resource.sampled_texture = std::move(texture); + resource.sampled_view = std::move(view); + resource.sampled_descriptor_index = descriptor; + return true; +} + +bool UploadGuestDepthTexture(TextureResource& resource, + plume::RenderCommandList* commands, + bool require_content_hash) { + namespace tu = rex::graphics::texture_util; + using rex::graphics::xenos::DataDimension; + std::lock_guard lock(resource.mutex); + const auto& fetch = resource.guest_fetch; + // Only the actual UNORM24 2D atlas path is supported here. FLOAT24 has a + // different encoding; cubes, volumes and mip chains need separate coverage. + if (!commands || resource.surface || resource.owns_guest_memory || + resource.resolved_on_host || !resource.texture || resource.guest_format != 22 || + !IsDepthFormat(resource.format) || resource.levels != 1 || + resource.d3d_type != u32(D3DResourceType::kTexture) || + fetch.dimension != DataDimension::k2DOrStacked || fetch.stacked || + !fetch.base_address || fetch.num_format || fetch.exp_adjust || + (u32(fetch.sign_x) | u32(fetch.sign_y) | u32(fetch.sign_z) | u32(fetch.sign_w))) + return false; + const bool audit = std::getenv("LEGO_NATIVE_AUDIT_TEXTURE_WATCH") != nullptr; + if (!require_content_hash && !audit && resource.guest_uploaded && + resource.sampled_valid && CpuMemoryUnchanged(resource.cpu_stamp)) { + if (NativeTextureTimingEnabled()) ++g_upload_timing.source_hits; + return true; + } + const auto layout = tu::GetGuestTextureLayout(fetch.dimension, fetch.pitch, + resource.width, resource.height, 1, fetch.tiled, fetch.format, + fetch.packed_mips, true, 0); + const u32 address = fetch.base_address << 12; + const u32 size = layout.base.level_data_extent_bytes; + if (!size || layout.base.array_slice_data_extent_bytes > size || + !ReadableUploadSpan(address, size)) return false; + const CpuMemorySpan spans[] = {{address, size}}; + auto stamp = CpuMemoryWatchEnabled() ? WatchCpuMemory(spans) : CpuMemoryStamp{}; + const auto* source = address < 0x20000000u + ? REX_KERNEL_MEMORY()->TranslatePhysical(address) + : REX_KERNEL_MEMORY()->TranslateVirtual(address); + const auto start = NativeTextureTimingEnabled() ? std::chrono::steady_clock::now() + : std::chrono::steady_clock::time_point{}; + const auto key = MakeTextureUploadSourceKey({source, size}, {}, + {reinterpret_cast(&fetch), sizeof(fetch)}); + if (NativeTextureTimingEnabled()) { + const double elapsed = std::chrono::duration( + std::chrono::steady_clock::now() - start).count(); + g_upload_timing.hash_ms += elapsed; + g_upload_timing.source_ms += elapsed; + g_upload_timing.hashed_bytes += size; + } + if (resource.guest_uploaded && resource.sampled_valid && + resource.upload_source_key_valid && resource.upload_source_key == key) { + resource.cpu_stamp = std::move(stamp); + if (NativeTextureTimingEnabled()) ++g_upload_timing.source_hits; + return true; + } + const u32 pitch = RowPitch(plume::RenderFormat::R32_FLOAT, resource.width); + const u64 bytes = u64(pitch) * resource.height; + if (!resource.width || !resource.height || !bytes || bytes > 128u * 1024 * 1024) + return false; + std::vector decoded(static_cast(bytes)); + u32 offset_x = 0, offset_y = 0, offset_z = 0; + if (fetch.packed_mips) + tu::GetPackedMipOffset(resource.width, resource.height, 1, fetch.format, + 0, offset_x, offset_y, offset_z); + const u32 endian_xor = u32(fetch.endianness) == 1 ? 1 + : u32(fetch.endianness) == 2 ? 3 : u32(fetch.endianness) == 3 ? 2 : 0; + if (!CopyTextureDepth24({source, layout.base.array_slice_data_extent_bytes}, + decoded, resource.width, resource.height, pitch, endian_xor, + [&](u32 x, u32 y, u32) -> int64_t { + if (fetch.tiled) + return tu::GetTiledOffset2D(x + offset_x, y + offset_y, + layout.base.row_pitch_bytes / 4, 2); + return u64(y + offset_y) * layout.base.row_pitch_bytes + + u64(x + offset_x) * 4; + })) return false; + if (!EnsureDepthSamplingMirror(resource)) return false; + auto upload = std::shared_ptr(HostDevice::Device()->createBuffer( + plume::RenderBufferDesc::UploadBuffer(static_cast(bytes))).release()); + if (!upload) return false; + auto* mapped = upload->map(); + if (!mapped) return false; + std::memcpy(mapped, decoded.data(), decoded.size()); + upload->unmap(); + commands->barriers(plume::RenderBarrierStage::COPY, + plume::RenderTextureBarrier(resource.sampled_texture.get(), plume::RenderTextureLayout::COPY_DEST)); + commands->copyTextureRegion( + plume::RenderTextureCopyLocation::Subresource(resource.sampled_texture.get(), 0, 0), + plume::RenderTextureCopyLocation::PlacedFootprint(upload.get(), plume::RenderFormat::R32_FLOAT, + resource.width, resource.height, 1, pitch / 4, 0)); + commands->barriers(plume::RenderBarrierStage::GRAPHICS, + plume::RenderTextureBarrier(resource.sampled_texture.get(), plume::RenderTextureLayout::SHADER_READ)); + HostDevice::RetireResource(std::move(upload)); + resource.guest_uploaded = true; + resource.sampled_valid = true; + resource.guest_content_hash = XXH3_64bits(decoded.data(), decoded.size()); + resource.upload_source_key = key; + resource.upload_source_key_valid = true; + resource.cpu_stamp = std::move(stamp); + if (NativeTextureTimingEnabled()) g_upload_timing.converted_bytes += bytes; + if (LongProbeEnabled()) LongProbeEvent("depth_texture_upload", false, + "guest=", resource.guest_address, "base=", address, "width=", resource.width, + "height=", resource.height, "bytes=", bytes); + return true; +} + u32 AllocateResource(const std::shared_ptr& resource, u32 usage, u32 multi_sample) { if (!resource->width || !resource->height || !resource->levels || @@ -1012,7 +1198,7 @@ TextureResourceView ResolveTextureResource(u32 guest_address) { if (!resource) { return {}; } - if (resource->resolved_on_host && resource->sampled_valid) + if ((resource->resolved_on_host || resource->guest_uploaded) && resource->sampled_valid) return {resource->sampled_texture.get(), resource->sampled_view.get(), IsDepthFormat(resource->format) ? plume::RenderFormat::R32_FLOAT : plume::RenderFormat::R32G32B32A32_FLOAT, @@ -1025,13 +1211,18 @@ TextureResourceView ResolveTextureResource(u32 guest_address) { (u64(resource->descriptor_index) << 3) ^ (u64(resource->resolved_descriptor_index) << 11) ^ (u64(resource->resolved_on_host) << 1) ^ u64(native_color); - if (LongProbeOnce(signature ^ 0xC010A07000000000ull)) - LongProbeEvent("texture_color_authority", false, "guest=", guest_address, - "host_resolved=", resource->resolved_on_host, "logical_rgba=", native_color, - "raw_descriptor=", resource->descriptor_index, - "resolved_descriptor=", resource->resolved_descriptor_index, - "guest_swizzle=", u32(resource->guest_fetch.swizzle), - "width=", resource->width, "height=", resource->height); + if (!resource->authority_probe_signature_valid || + resource->last_authority_probe_signature != signature) { + resource->last_authority_probe_signature = signature; + resource->authority_probe_signature_valid = true; + if (LongProbeOnce(signature ^ 0xC010A07000000000ull)) + LongProbeEvent("texture_color_authority", false, "guest=", guest_address, + "host_resolved=", resource->resolved_on_host, "logical_rgba=", native_color, + "raw_descriptor=", resource->descriptor_index, + "resolved_descriptor=", resource->resolved_descriptor_index, + "guest_swizzle=", u32(resource->guest_fetch.swizzle), + "width=", resource->width, "height=", resource->height); + } } return {resource->texture.get(), native_color ? resource->resolved_view.get() : resource->view.get(), @@ -1050,6 +1241,13 @@ bool UploadTextureResource(u32 guest_address, auto recording = HostDevice::LockRecording(); UploadTimer timer; const auto resource = FindTexture(guest_address); + // Render targets and native resolves already have authoritative host + // storage, including resolved depth sampling mirrors. No CPU upload is + // required; reporting success lets callers reject genuine upload failures. + if (resource && commands && resource->texture && + (resource->surface || resource->resolved_on_host)) return true; + if (resource && resource->guest_format == 22 && IsDepthFormat(resource->format)) + return UploadGuestDepthTexture(*resource, commands, require_content_hash); if (DepthAliasesEnabled() && resource && commands && !resource->surface && !resource->owns_guest_memory && !resource->resolved_on_host && resource->guest_format == 6 && resource->guest_fetch.num_format == 0 && resource->guest_fetch.exp_adjust == 0 && @@ -1131,9 +1329,10 @@ bool UploadTextureResource(u32 guest_address, if (LongProbeEnabled() && !source_key_valid && LongProbeOnce(0xBAD0000000000000ull | guest_address)) LongProbeEvent("unreadable_texture_source", true, "guest=", guest_address, "base=", u32(fetch.base_address<<12), "mip=", u32(fetch.mip_address<<12)); - // Never read an unvalidated volume extent. Volumes may have padding - // between Z slices, so validating just width*height*depth isn't enough. - if ((volume || alpha4 || capture_pending) && !source_key_valid) return false; + // Every format must have resident base/mip storage before decoding. The + // layout extents include tiled padding, cube faces and packed mip tails; + // a failed hash-span check must never fall through to unchecked CPU reads. + if (!source_key_valid) return false; if (source_key_valid) { const auto hash_start = timer.enabled ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -1248,17 +1447,23 @@ bool UploadTextureResource(u32 guest_address, destination_offset += LevelSize(*resource, level); continue; } - for (u32 y = 0; y < (height + block_width - 1) / block_width; ++y) { - for (u32 x = 0; x < (width + block_width - 1) / block_width; ++x) { - const u32 source_offset = fetch.tiled - ? tu::GetTiledOffset2D(x + offset_x, y + offset_y, - guest_level.row_pitch_bytes / block_bytes, rex::log2_floor(block_bytes)) - : (y + offset_y) * guest_level.row_pitch_bytes + (x + offset_x) * block_bytes; - for (u32 byte = 0; byte < block_bytes; ++byte) - mapped[destination_offset + y * pitch + x * block_bytes + byte] = - source[(source_offset + byte) ^ endian_xor]; - } - } + // The checked block copier also handles one 2D/cube slice. Validate each + // computed address, including endian XOR and packed-mip offsets, against + // the same guest extent whose residency was checked above. + const bool copied = CopyTextureVolumeBlocks( + {source, guest_level.array_slice_data_extent_bytes}, + {mapped + destination_offset, LevelSize(*resource, level)}, + (width + block_width - 1) / block_width, + (height + format_info->block_height - 1) / format_info->block_height, + 1, block_bytes, pitch, endian_xor, + [&](u32 x, u32 y, u32) -> int64_t { + if (fetch.tiled) + return tu::GetTiledOffset2D(x + offset_x, y + offset_y, + guest_level.row_pitch_bytes / block_bytes, rex::log2_floor(block_bytes)); + return u64(y + offset_y) * guest_level.row_pitch_bytes + + u64(x + offset_x) * block_bytes; + }); + if (!copied) return false; destination_offset += LevelSize(*resource, level); } DumpTextureUpload(*resource, mapped, @@ -1716,6 +1921,9 @@ plume::RenderFramebuffer* ResolveFramebuffer(u32 render_target, u32 depth_stenci plume::RenderFramebuffer* ResolveFramebuffer( const std::array& render_targets, u32 depth_stencil) { + // Registration and insertion must be atomic relative to invalidation; its + // owner snapshot must never miss a framebuffer being created concurrently. + auto recording = HostDevice::LockRecording(); std::array, 4> colors; u32 color_count = 0; for (u32 i = 0; i < colors.size(); ++i) { @@ -1763,6 +1971,7 @@ plume::RenderFramebuffer* ResolveFramebuffer( return nullptr; } auto* result = framebuffer.get(); + g_framebuffer_owners[owner->guest_address] = owner; owner->framebuffers.emplace(key, std::move(framebuffer)); return result; } @@ -1859,6 +2068,8 @@ u32 ReleaseNativeTexture(u32 guest_address) { } void ResetTextureResources() { + auto recording = HostDevice::LockRecording(); + g_framebuffer_owners.clear(); g_depth_resolves.clear(); g_edram_colors.clear(); g_depth_alias_generation = 0; diff --git a/rexlego/src/gpu_native/textures.h b/rexlego/src/gpu_native/textures.h index 4ef055b..e4840e6 100644 --- a/rexlego/src/gpu_native/textures.h +++ b/rexlego/src/gpu_native/textures.h @@ -43,8 +43,9 @@ TextureResourceView ResolveTextureResource(u32 guest_address); // Call at SetTexture and Resolve, where the header supplies the new binding. // A draw already in flight must retain its old texture and immutable SRV. void RefreshTextureHeader(u32 guest_address); -// Copies the complete CPU mirror into the host texture and leaves it in -// SHADER_READ. The CPU mirror remains authoritative for readback/correctness. +// Prepares texture contents for sampling. CPU-backed textures require a +// successful upload; host render targets and native resolves need no upload. +// A false result must not bind stale or uninitialized host storage. bool UploadTextureResource(u32 guest_address, plume::RenderCommandList* commands, bool require_content_hash = false); diff --git a/tools/gpu-aot/test_framebuffer_owners.py b/tools/gpu-aot/test_framebuffer_owners.py new file mode 100644 index 0000000..361608b --- /dev/null +++ b/tools/gpu-aot/test_framebuffer_owners.py @@ -0,0 +1,171 @@ +"""Exercise production framebuffer creation/invalidation/reset and owner lifetimes.""" +import argparse +from pathlib import Path +import subprocess + + +def function(text, signature): + start = text.index(signature) + brace = text.index('{', start) + depth = 1 + end = brace + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--compiler', default='clang++') + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + text = (root / 'rexlego/src/gpu_native/textures.cpp').read_text() + key = text[text.index('struct FramebufferKey {'):text.index('struct SurfaceStorage {')] + registry = text[text.index('std::mutex g_textures_mutex;'):text.index('std::atomic g_texture_lifecycle_logs')] + bodies = '\n'.join(function(text, signature) for signature in [ + 'void InvalidateFramebufferReferences(u32 guest_address)', + 'plume::RenderFramebuffer* ResolveFramebuffer(\n const std::array&', + 'void ResetTextureResources()']) + harness = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using u32 = uint32_t; +thread_local unsigned held_resource_locks = 0; +struct ResourceMutex { + std::mutex value; + void lock() { value.lock(); ++held_resource_locks; } + void unlock() { --held_resource_locks; value.unlock(); } +}; +namespace plume { +struct RenderTexture {}; +struct RenderFramebuffer { static inline unsigned destroyed = 0; ~RenderFramebuffer() { ++destroyed; } }; +struct RenderFramebufferDesc { const RenderTexture** colorAttachments{}; u32 colorAttachmentsCount{}; RenderTexture* depthAttachment{}; }; +} +''' + harness += key + r''' +struct TextureResource { + u32 guest_address{}, mirror_address{}; + bool owns_guest_memory = false; + bool depth_format = false; + ResourceMutex mutex; + std::unique_ptr texture = std::make_unique(); + int format = 0; + std::unordered_map, FramebufferKeyHash> framebuffers; +}; +''' + harness += registry + r''' +std::unordered_map> g_depth_resolves, g_edram_colors; +unsigned g_depth_alias_generation = 0; +struct Memory { unsigned frees = 0; void SystemHeapFree(u32) { ++frees; } } guest_memory; +#define REX_KERNEL_MEMORY() (&guest_memory) +#define REXLOG_ERROR(...) ((void)0) +bool IsDepthFormat(int format) { return format != 0; } +std::shared_ptr FindTexture(u32 address) { + std::lock_guard lock(g_textures_mutex); + auto it = g_textures.find(address); return it == g_textures.end() ? nullptr : it->second; +} +struct Device { + bool fail = false; + std::function before_create; + std::shared_ptr createFramebuffer(const plume::RenderFramebufferDesc&) { + if (before_create) before_create(); + return fail ? nullptr : std::make_shared(); + } +} device; +struct HostDevice { + static inline std::recursive_mutex recording; + static inline std::vector> retired; + static auto LockRecording() { return std::unique_lock(recording); } + static auto Device() { return &device; } + static void RetireResource(std::shared_ptr item) { + assert(held_resource_locks == 0); // retirement occurs after each owner's mutex is released + retired.push_back(std::move(item)); + } +}; +''' + harness += bodies + r''' +std::shared_ptr Add(u32 address, bool depth = false) { + auto r = std::make_shared(); r->guest_address = address; r->format = depth; + g_textures[address] = r; return r; +} +int main() { + auto a = Add(1), b = Add(2), c = Add(3), d = Add(4, true), e = Add(5, true); + // Thousands of sampled textures never become owners or incur invalidation locks. + for (u32 i = 100; i < 10100; ++i) Add(i); + const std::array ab{1, 2, 0, 0}, ac{1, 3, 0, 0}; + auto first = ResolveFramebuffer(ab, 4); assert(first); + assert(ResolveFramebuffer(ab, 4) == first); + assert(ResolveFramebuffer(ac, 4)); assert(ResolveFramebuffer(ab, 5)); + assert(g_framebuffer_owners.size() == 2); + InvalidateFramebufferReferences(2); // second color, cross-owner dependency + assert(HostDevice::retired.size() == 2); assert(d->framebuffers.size() == 1); + assert(e->framebuffers.empty()); assert(g_framebuffer_owners.size() == 1); + assert(plume::RenderFramebuffer::destroyed == 0); // fence retirement retains old objects + InvalidateFramebufferReferences(4); // depth dependency + assert(d->framebuffers.empty()); assert(HostDevice::retired.size() == 3); + assert(g_framebuffer_owners.empty()); + HostDevice::retired.clear(); assert(plume::RenderFramebuffer::destroyed == 3); + assert(ResolveFramebuffer(ac, 4)); // emptied owner becomes indexed again + // Same guest address can now name another resource generation. The retired + // owner keeps its framebuffer until that owner retires; no new owner false hit. + auto replacement = Add(4, true); + InvalidateFramebufferReferences(1); + assert(g_framebuffer_owners.empty()); assert(d->framebuffers.size() == 1); + assert(ResolveFramebuffer(ab, 4)); + InvalidateFramebufferReferences(1); assert(replacement->framebuffers.empty()); + // Expired weak pointers are pruned and do not keep texture lifetimes alive. + std::weak_ptr expired; + { auto dead = Add(20, true); expired = dead; g_framebuffer_owners[20] = dead; g_textures.erase(20); } + assert(expired.expired()); InvalidateFramebufferReferences(123); assert(!g_framebuffer_owners.contains(20)); + // A failed creation must not register a phantom owner; attachment gaps remain rejected. + device.fail = true; assert(!ResolveFramebuffer(ab, 5)); device.fail = false; + assert(!g_framebuffer_owners.contains(5)); + assert(!ResolveFramebuffer(std::array{1, 0, 3, 0}, 5)); + // Force invalidation to overlap the window before framebuffer creation/index + // registration. It must wait for the recording lock, then see the new entry. + std::promise reached, release; auto released = release.get_future().share(); + device.before_create = [&] { reached.set_value(); released.wait(); }; + auto creator = std::async(std::launch::async, [&] { assert(ResolveFramebuffer(ab, 5)); }); + reached.get_future().wait(); + std::promise invalidator_started; + auto invalidator = std::async(std::launch::async, [&] { invalidator_started.set_value(); InvalidateFramebufferReferences(2); }); + invalidator_started.get_future().wait(); + assert(invalidator.wait_for(std::chrono::milliseconds(75)) == std::future_status::timeout); + release.set_value(); creator.get(); invalidator.get(); device.before_create = {}; + assert(e->framebuffers.empty()); assert(!g_framebuffer_owners.contains(5)); + // Reset removes all weak generations and preserves the existing guest-free policy. + assert(ResolveFramebuffer(ab, 4)); replacement->owns_guest_memory = true; + replacement->mirror_address = 77; g_depth_resolves[9] = replacement; g_edram_colors[8] = a; + g_depth_alias_generation = 123; + ResetTextureResources(); assert(g_textures.empty() && g_framebuffer_owners.empty()); + assert(g_depth_resolves.empty() && g_edram_colors.empty() && g_depth_alias_generation == 0); + assert(guest_memory.frees == 2); + std::cout << "Production framebuffer-owner dependency/lifetime/concurrency regressions passed\n"; +} +''' + args.output.mkdir(parents=True, exist_ok=True) + source = args.output.resolve() / 'test.cpp' + binary = args.output.resolve() / 'test.exe' + source.write_text(harness) + subprocess.run([args.compiler, '-std=c++20', '-UNDEBUG', '-pthread', str(source), '-o', str(binary)], check=True, timeout=45) + subprocess.run([str(binary)], check=True, timeout=10) + + +if __name__ == '__main__': + main() diff --git a/tools/gpu-aot/test_texture_2d_upload.cpp b/tools/gpu-aot/test_texture_2d_upload.cpp new file mode 100644 index 0000000..b6c33a0 --- /dev/null +++ b/tools/gpu-aot/test_texture_2d_upload.cpp @@ -0,0 +1,81 @@ +#include "../../rexlego/src/gpu_native/texture_volume_upload.h" + +#include +#include +#include +#include +#include +#include +#include + +using legodimensions::gpu_native::CopyTextureVolumeBlocks; +using u32 = uint32_t; + +namespace { +void TestLinearSlice(u32 width_blocks, u32 height_blocks, u32 block_bytes) { + const u32 guest_pitch = width_blocks * block_bytes + 32; + constexpr u32 host_pitch = 256; + // Packed mip offsets and guest row padding must not reach the host image. + auto offset = [=](u32 x, u32 y, u32) -> int64_t { + return uint64_t(y + 1) * guest_pitch + uint64_t(x + 1) * block_bytes; + }; + for (u32 endian = 0; endian != 4; ++endian) { + std::vector source(size_t(offset(width_blocks - 1, height_blocks - 1, 0)) + block_bytes, 0xCD); + std::vector destination(host_pitch * height_blocks + 8, 0xFE); + for (u32 y = 0; y < height_blocks; ++y) + for (u32 x = 0; x < width_blocks; ++x) + for (u32 byte = 0; byte < block_bytes; ++byte) + source[(size_t(offset(x, y, 0)) + byte) ^ endian] = uint8_t(y * 37 + x * 11 + byte); + auto image = std::span(destination).first(host_pitch * height_blocks); + assert(CopyTextureVolumeBlocks(source, image, width_blocks, height_blocks, + 1, block_bytes, host_pitch, endian, offset)); + for (u32 y = 0; y < height_blocks; ++y) { + for (u32 x = 0; x < width_blocks; ++x) + for (u32 byte = 0; byte < block_bytes; ++byte) + assert(destination[y * host_pitch + x * block_bytes + byte] == uint8_t(y * 37 + x * 11 + byte)); + for (u32 byte = width_blocks * block_bytes; byte < host_pitch; ++byte) + assert(destination[y * host_pitch + byte] == 0xFE); + } + for (size_t at = image.size(); at < destination.size(); ++at) assert(destination[at] == 0xFE); + assert(!CopyTextureVolumeBlocks(std::span(source).first(source.size() - 1), + image, width_blocks, height_blocks, 1, block_bytes, host_pitch, endian, offset)); + assert(!CopyTextureVolumeBlocks(source, image.first(image.size() - 1), + width_blocks, height_blocks, 1, block_bytes, host_pitch, endian, offset)); + assert(!CopyTextureVolumeBlocks(source, image, width_blocks, height_blocks, + 1, block_bytes, width_blocks * block_bytes - 1, endian, offset)); + } +} + +void TestNonlinearSlice() { + // A cube face is independently bounded; a tile/packed-tail offset callback + // may visit blocks in a different order than destination rows. + std::vector cube(6 * 256, 0xCD), destination(16, 0xFE); + auto face = std::span(cube).subspan(3 * 256, 96); + auto offset = [](u32 x, u32 y, u32) -> int64_t { + return 64 + (x * 2 + y) * 4; + }; + for (u32 y = 0; y < 2; ++y) + for (u32 x = 0; x < 2; ++x) + for (u32 byte = 0; byte < 4; ++byte) + face[size_t(offset(x, y, 0)) + byte] = uint8_t((y * 2 + x) * 4 + byte); + assert(CopyTextureVolumeBlocks(face, destination, 2, 2, 1, 4, 8, 0, offset)); + for (u32 byte = 0; byte < 16; ++byte) assert(destination[byte] == byte); + assert(!CopyTextureVolumeBlocks(face.first(79), destination, 2, 2, 1, 4, 8, 0, offset)); + assert(!CopyTextureVolumeBlocks(face, destination, 2, 2, 1, 4, 8, 4, offset)); + assert(!CopyTextureVolumeBlocks(face, destination, 2, 2, 1, 4, 8, 0, + [](auto, auto, auto) { return int64_t{-1}; })); + assert(!CopyTextureVolumeBlocks(face, destination, 2, 2, 1, 4, 8, 0, + [](auto, auto, auto) { return std::numeric_limits::max(); })); + // Endian XOR can address bytes beyond an otherwise in-range block offset. + assert(!CopyTextureVolumeBlocks(face.first(5), destination, 1, 1, 1, 1, 1, 3, + [](auto, auto, auto) { return int64_t{4}; })); +} +} // namespace + +int main() { + TestLinearSlice(5, 3, 4); // Odd-sized uncompressed 2D image. + TestLinearSlice(2, 2, 8); // 7x5 texels rounded to compressed blocks. + TestLinearSlice(2, 2, 16); // Sixteen-byte compressed blocks. + TestNonlinearSlice(); + std::cout << "2D/cube bounded-copy regressions passed\n"; +} diff --git a/tools/gpu-aot/test_texture_authority_probe.py b/tools/gpu-aot/test_texture_authority_probe.py new file mode 100644 index 0000000..31006bd --- /dev/null +++ b/tools/gpu-aot/test_texture_authority_probe.py @@ -0,0 +1,233 @@ +"""Compare actual texture view/probe bodies before and after diagnostic caching.""" + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + + +def block(text, signature): + start = text.index(signature) + opening = text.index('{', start) + depth, end = 1, opening + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--compiler', default='clang++') + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + source = root / 'rexlego/src/gpu_native/textures.cpp' + probe_source = root / 'rexlego/src/gpu_native/long_probe.h' + view_source = root / 'rexlego/src/gpu_native/textures.h' + text, probes = source.read_text(), probe_source.read_text() + resolver = block(text, 'TextureResourceView ResolveTextureResource(') + # Reconstruct the original production body by removing only the diagnostic + # cache guard/assignments. View selection and the global dedup call stay real. + cached = block(resolver, ' if (!resource->authority_probe_signature_valid ||') + legacy_probe = cached[cached.index(' if (LongProbeOnce'):cached.rfind('}')] + legacy_probe = '\n'.join(line[2:] for line in legacy_probe.rstrip().splitlines()) + legacy = resolver.replace(cached, legacy_probe) + assert 'authority_probe_signature' not in legacy + declarations = '\n'.join(block(text, name) + ';' for name in + ('struct FramebufferKey {', 'struct FramebufferKeyHash {', + 'struct TextureResource {')) + declarations += '\n' + block(view_source.read_text(), 'struct TextureResourceView {') + ';' + once = block(probes, 'inline bool LongProbeOnce(') + event = block(probes, 'template\ninline void LongProbeEvent(') + harness = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using u32=uint32_t; using u64=uint64_t; +namespace plume { +enum class RenderFormat {UNKNOWN, R32_FLOAT, R32G32B32A32_FLOAT, COLOR, DEPTH}; +struct RenderTexture {}; struct RenderTextureView {}; struct RenderFramebuffer {}; +} +namespace rex::graphics::xenos { struct xe_gpu_texture_fetch_t {u32 swizzle=0xB48;}; } +struct TextureUploadSourceKey {}; struct CpuMemoryStamp {}; +''' + harness += declarations + r''' +bool IsDepthFormat(plume::RenderFormat format) {return format==plume::RenderFormat::DEPTH;} +using Event=std::tuple; +''' + for namespace, body in [('original', legacy), ('optimized', resolver)]: + harness += '\nnamespace ' + namespace + r''' { +bool enabled=false; +u64 lookups=0; +std::vector events; +std::unordered_map> resources; +bool LongProbeEnabled() {return enabled;} +void WriteLongProbeEvent(std::string_view kind,bool anomaly,std::string_view payload) { + events.emplace_back(kind,anomaly,payload); +} +''' + harness += once + '\n' + event + r''' +bool CountedProbeOnce(u64 key) {++lookups;return LongProbeOnce(key);} +auto AdoptTexture(u32 address) { + const auto found=resources.find(address); + return found==resources.end() ? nullptr : found->second; +} +''' + harness += body.replace('LongProbeOnce(', 'CountedProbeOnce(') + '\n}\n' + harness += r''' +constexpr u32 address=0xAC585174; +std::shared_ptr MakeResource(u32 guest) { + auto resource=std::make_shared(); + resource->guest_address=guest;resource->width=960;resource->height=3840; + resource->host_width=1920;resource->host_height=7680;resource->format=plume::RenderFormat::COLOR; + resource->descriptor_index=377;resource->d3d_type=3; + resource->texture=std::make_unique(); + resource->view=std::make_unique(); + resource->resolved_view=std::make_unique(); + resource->sampled_texture=std::make_unique(); + resource->sampled_view=std::make_unique(); + return resource; +} +void Create(u32 guest=address) { + original::resources[guest]=MakeResource(guest); + optimized::resources[guest]=MakeResource(guest); +} +template void ChangeBoth(Change change,u32 guest=address) { + change(*original::resources.at(guest));change(*optimized::resources.at(guest)); +} +int TextureRole(const TextureResourceView& v,const TextureResource* r) { + if (!v.texture)return 0; + if (v.texture==r->texture.get())return 1; + assert(v.texture==r->sampled_texture.get());return 2; +} +int ViewRole(const TextureResourceView& v,const TextureResource* r) { + if (!v.view)return 0; + if (v.view==r->view.get())return 1; + if (v.view==r->resolved_view.get())return 2; + assert(v.view==r->sampled_view.get());return 3; +} +void ResolveBoth(u32 guest=address) { + const auto before=original::ResolveTextureResource(guest); + const auto after=optimized::ResolveTextureResource(guest); + const auto a=original::AdoptTexture(guest),b=optimized::AdoptTexture(guest); + assert(TextureRole(before,a.get())==TextureRole(after,b.get())); + assert(ViewRole(before,a.get())==ViewRole(after,b.get())); + assert(std::tie(before.format,before.descriptor_index,before.width,before.height, + before.d3d_type,before.surface,before.depth)== + std::tie(after.format,after.descriptor_index,after.width,after.height, + after.d3d_type,after.surface,after.depth)); + assert(original::events==optimized::events); +} +bool LookupBound(u64 expected) { + if(optimized::lookups==expected)return true; + std::cerr<<"unchanged authority repeated global lookup: "< +#include +#include +#include +#include +#include +#include +using u32=uint32_t; using u64=uint64_t; using i32=int32_t; +constexpr u32 kNativeTextureSlots=3; +enum class ShaderStage {kVertex,kPixel}; +enum class D3DResourceType:u32 {kTexture=3,kVolumeTexture=4,kCubeTexture=5}; +using TextureFetchWords=std::array; +#define REXLOG_INFO(...) ((void)0) +bool LongProbeEnabled(){return false;} +bool LongProbeOnce(u64){return false;} +templatevoid LongProbeEvent(Args&&...){} +templateint LongProbeHex(const T&){return 0;} +u64 BoundShaderHash(ShaderStage){return 0;} +u32 vertex_mask=0; +u32 BoundShaderTextureMask(ShaderStage stage){return stage==ShaderStage::kVertex?vertex_mask:0;} +namespace plume {struct RenderCommandList{};} +struct HostDevice { + static int LockRecording(){return 0;} + static u32 RegisterSampler(const TextureFetchWords&){++sampler_calls;return 7;} + static inline u32 sampler_calls=0; +}; +struct D3DDevice {struct {std::arraydword{};}fetch_constants[kNativeTextureSlots];}; +struct DrawBindings {std::arraytextures{};}; +struct Shared { + std::arraysamplers{}; + std::arraytexture_2d{}; + std::arraytexture_3d{1,1,1}; + std::arraytexture_cube{2,2,2}; +}; +struct TextureResource { + void* texture=reinterpret_cast(1); + bool surface=false,resolved_on_host=false,cpu_upload_allowed=true; + u32 guest_format=0;int format=0; + u32 descriptor_index=3,d3d_type=u32(D3DResourceType::kTexture); +}; +struct TextureResourceView {void*texture=nullptr;u32 descriptor_index=~u32{0},d3d_type=3;bool surface=false,depth=false;}; +std::unordered_map>resources; +std::shared_ptrFindTexture(u32 address){ + auto it=resources.find(address);return it==resources.end()?nullptr:it->second; +} +TextureResourceView ResolveTextureResource(u32 address){ + auto resource=FindTexture(address);if(!resource)return {}; + return {resource->texture,resource->descriptor_index,resource->d3d_type}; +} +struct UploadTimer{}; +bool IsDepthFormat(int format){return format==22;} +bool UploadGuestDepthTexture(TextureResource&,plume::RenderCommandList*,bool){ + assert(false&&"Depth conversion is exercised by test_texture_depth_upload.py");return false; +} +u32 upload_calls=0,cpu_upload_calls=0; +std::vectorrequire_hashes; +bool UploadTextureResource(u32 guest_address,plume::RenderCommandList* commands,bool require_content_hash){ + ++upload_calls;require_hashes.push_back(require_content_hash); +''' + harness += upload_entry + r''' + // Only allocation/conversion machinery below the production entry guard is + // mocked. A residency or conversion failure is represented by false here. + ++cpu_upload_calls; + return resource&&commands&&resource->texture&&resource->cpu_upload_allowed; +} +bool BindTextureConstants(plume::RenderCommandList*commands,const D3DDevice*device, + const DrawBindings&bindings,u32 texture_mask,Shared&shared){ + constexpr bool trace_constants=false; +''' + harness += loop + r''' + return true; +} +plume::RenderCommandList command_list; +D3DDevice device; +DrawBindings bindings; +u32 texture_mask=7,query_failures=0,successful_draws=0; +Shared last_shared; +bool DispatchDraw(D3DDevice* value,u32,bool,u32,u32,i32){ + // BindConstants' caller rejects false before any draw submission. + const bool bound=BindTextureConstants(&command_list,value,bindings,texture_mask,last_shared); + if(bound)++successful_draws;return bound; +} +bool ConsumeCpuPoolCopyDraw(u32,u32,u32){return false;} +void FailActiveQueries(){++query_failures;} +''' + harness += hooks + r''' +void Reset(){ + resources.clear();bindings={};last_shared={};vertex_mask=0;texture_mask=7; + upload_calls=cpu_upload_calls=query_failures=successful_draws=HostDevice::sampler_calls=0; + require_hashes.clear(); +} +std::shared_ptrAdd(u32 address,u32 slot){ + auto value=std::make_shared();value->descriptor_index=address+2; + resources[address]=value;bindings.textures[slot]=address;return value; +} +int main(){ + for(bool indexed:{false,true}){ + Reset();auto value=Add(1,0);value->cpu_upload_allowed=false; + if(indexed)DrawIndexedVerticesHook(&device,0,0,0,3); + else DrawVerticesHook(&device,0,0,3); + assert(query_failures==1&&successful_draws==0&&upload_calls==1&&cpu_upload_calls==1); + assert(HostDevice::sampler_calls==0&&last_shared.texture_2d[0]==0); + } + { + Reset();auto first=Add(1,0);auto second=Add(2,1);second->cpu_upload_allowed=false; + DrawVerticesHook(&device,0,0,3); + assert(query_failures==1&&successful_draws==0&&upload_calls==2); + assert(HostDevice::sampler_calls==1&&last_shared.texture_2d[1]==0); + } + { + Reset();Add(1,0);auto volume=Add(2,1);volume->d3d_type=u32(D3DResourceType::kVolumeTexture); + auto cube=Add(3,2);cube->d3d_type=u32(D3DResourceType::kCubeTexture);vertex_mask=2; + DrawVerticesHook(&device,0,0,3); + assert(successful_draws==1&&query_failures==0&&cpu_upload_calls==3); + assert(last_shared.texture_2d[0]==3&&last_shared.texture_3d[1]==4&&last_shared.texture_cube[2]==5); + assert((require_hashes==std::vector{false,true,false})); + } + for(bool resolved:{false,true}){ + Reset();auto value=Add(1,0);value->cpu_upload_allowed=false; + value->surface=!resolved;value->resolved_on_host=resolved; + DrawVerticesHook(&device,0,0,3); + assert(successful_draws==1&&query_failures==0&&upload_calls==1&&cpu_upload_calls==0); + // Host authority cannot bypass validation of command/storage availability. + assert(!UploadTextureResource(1,nullptr,false)); + value->texture=nullptr;assert(!UploadTextureResource(1,&command_list,false)); + } + { + Reset();bindings.textures[0]=99;Add(2,1)->descriptor_index=~u32{0}; + Add(3,2)->texture=nullptr;DrawVerticesHook(&device,0,0,3); + assert(successful_draws==1&&query_failures==0&&upload_calls==0); + assert((last_shared.texture_2d==std::array{0,0,0})); + assert((last_shared.texture_3d==std::array{1,1,1})); + assert((last_shared.texture_cube==std::array{2,2,2})); + } + { + Reset();Add(1,0)->cpu_upload_allowed=false;texture_mask=0; + DrawVerticesHook(&device,0,0,3); + assert(successful_draws==1&&query_failures==0&&upload_calls==0); + } + std::cout<<"Production texture binding rejects failed uploads; GPU authority and null defaults preserved\n"; +} +''' + args.output.mkdir(parents=True, exist_ok=True) + source = args.output.resolve() / "texture-binding.cpp" + binary = args.output.resolve() / "texture-binding.exe" + source.write_text(harness) + subprocess.run([args.compiler, "-std=c++20", "-UNDEBUG", str(source), "-o", str(binary)], + check=True, timeout=45) + subprocess.run([str(binary)], check=True, timeout=10) + + +if __name__ == "__main__": + main() diff --git a/tools/gpu-aot/test_texture_depth_upload.py b/tools/gpu-aot/test_texture_depth_upload.py new file mode 100644 index 0000000..de466e4 --- /dev/null +++ b/tools/gpu-aot/test_texture_depth_upload.py @@ -0,0 +1,353 @@ +"""Execute actual native UNORM24 upload/resolve/binding bodies with bounded mocks.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + + +def function(text, signature): + start = text.index(signature) + brace = text.index('{', start) + depth, end = 1, brace + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--compiler', default='clang++') + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + source_path = root / 'rexlego/src/gpu_native/textures.cpp' + text = source_path.read_text() + sdk_path = root / 'rexglue-sdk/src/graphics/shaders/bytecode/d3d12_5_1/texture_load_depth_unorm_cs.h' + sdk = sdk_path.read_text().split('#endif', 1)[0] + # Pinned SDK's actual emitted DXBC disassembly: endian conversion precedes + # stencil removal, endpoint correction, unsigned conversion and scaling. + operations = ['ushr r2.xyzw, r2.xyzw, l(8, 8, 8, 8)', + 'ushr r3.xyzw, r2.xyzw, l(23, 23, 23, 23)', + 'iadd r2.xyzw, r2.xyzw, r3.xyzw', + 'utof r2.xyzw, r2.xyzw', 'mul r2.xyzw, r2.xyzw'] + cursor = 0 + for op in operations: + cursor = sdk.index(op, cursor) + len(op) + bodies = '\n'.join(function(text, sig) for sig in [ + 'bool EnsureDepthSamplingMirror(', 'bool UploadGuestDepthTexture(', + 'TextureResourceView ResolveTextureResource(']) + upload = function(text, 'bool UploadTextureResource(') + upload = upload[:upload.index(' if (DepthAliasesEnabled()')] + ' return false;\n}' + branch_begin = text.index(' // Xenos k_24_8 sampling') + branch_end = text.index(' // Preserve both host depth', branch_begin) + resolve_branch = text[branch_begin:branch_end] + draw_text = (root / 'rexlego/src/gpu_native/draw.cpp').read_text() + bind_begin = draw_text.index(' auto texture = ResolveTextureResource(bindings.textures[i]);') + bind_end = draw_text.index(' TextureFetchWords fetch;', bind_begin) + bind = draw_text[bind_begin:bind_end] + tiled = function((root / 'rexglue-sdk/src/graphics/pipeline/texture/util.cpp').read_text(), + 'int32_t GetTiledOffset2D(') + harness = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "texture_depth_upload.h" +#include "texture_swizzle.h" +using namespace legodimensions::gpu_native; +using u8=uint8_t; using u32=uint32_t; using u64=uint64_t; +namespace rex { +constexpr u32 align(u32 v,u32 a) { return (v+a-1)&~(a-1); } +namespace graphics { +namespace xenos { enum class DataDimension { k2DOrStacked=1, k3D=2, kCube=3 }; constexpr u32 kTextureTileWidthHeight=32; +struct Fetch { + u32 base_address=1, mip_address=0, pitch=3, tiled=1, format=22, packed_mips=0, swizzle=0xB48; + u32 stacked=0, num_format=0, sign_x=0, sign_y=0, sign_z=0, sign_w=0, endianness=2; + int exp_adjust=0; DataDimension dimension=DataDimension::k2DOrStacked; +}; } +namespace texture_util { +struct Layout { struct { u32 level_data_extent_bytes=0, array_slice_data_extent_bytes=0, row_pitch_bytes=0; } base; }; +inline bool oversized_slice=false; +Layout GetGuestTextureLayout(xenos::DataDimension,u32 pitch,u32 width,u32 height,u32,bool tiled,u32,bool,bool,u32) { + Layout l; l.base.row_pitch_bytes=(tiled ? rex::align(pitch<<5,32) : width)*4; + l.base.level_data_extent_bytes=l.base.row_pitch_bytes*(tiled ? rex::align(height,32) : height); + l.base.array_slice_data_extent_bytes=l.base.level_data_extent_bytes+(oversized_slice ? 4 : 0); return l; +} +void GetPackedMipOffset(u32,u32,u32,u32,u32,u32& x,u32& y,u32& z) { x=y=z=0; } +''' + harness += tiled + r''' +} } } +namespace plume { +enum class RenderFormat { UNKNOWN, R32_FLOAT, R32G32B32A32_FLOAT, D32_FLOAT_S8 }; +enum class RenderTextureDimension { TEXTURE_2D }; +enum class RenderTextureViewDimension { TEXTURE_2D }; +enum class RenderSampleCount { COUNT_1 }; +enum RenderTextureFlag { RENDER_TARGET=1 }; +enum class RenderSwizzle { R,G,B,A,ZERO,ONE }; +struct RenderComponentMapping { std::array lanes{}; + RenderComponentMapping()=default; + RenderComponentMapping(RenderSwizzle r,RenderSwizzle g,RenderSwizzle b,RenderSwizzle a):lanes{r,g,b,a}{} }; +struct RenderTextureDesc { RenderTextureDimension dimension{};u32 width{},height{},depth{},mipLevels{},arraySize{},flags{};bool committed{}; + struct { RenderSampleCount sampleCount{}; } multisampling; RenderFormat format{}; }; +struct RenderTextureViewDesc { RenderTextureViewDimension dimension{}; RenderFormat format{};u32 mipLevels{};RenderComponentMapping componentMapping; }; +struct RenderTextureView { RenderTextureViewDesc desc; }; +inline bool fail_view=false; +struct RenderTexture { std::vector contents;u32 width{},height{}; virtual ~RenderTexture()=default; + std::unique_ptr createTextureView(const RenderTextureViewDesc& desc) { + return fail_view ? nullptr : std::make_unique(RenderTextureView{desc}); } }; +struct D3D12Texture:RenderTexture { void* d3d=this; }; +inline bool fail_map=false; +struct RenderBuffer { static inline u32 destroyed=0;std::vector contents; explicit RenderBuffer(u32 size):contents(size){}; + ~RenderBuffer(){ ++destroyed; } u8* map(){return fail_map ? nullptr : contents.data();} void unmap(){} }; +struct RenderBufferDesc {u32 size;static RenderBufferDesc UploadBuffer(u32 size){return{size};}}; +enum class RenderBarrierStage { COPY,GRAPHICS }; +enum class RenderTextureLayout { COPY_DEST,SHADER_READ }; +struct RenderTextureBarrier { RenderTexture* texture; RenderTextureLayout layout; }; +struct RenderTextureCopyLocation {RenderTexture* texture=nullptr;RenderBuffer* buffer=nullptr;u32 width=0,height=0,pitch=0; + static RenderTextureCopyLocation Subresource(RenderTexture* t,u32,u32){return{t};} + static RenderTextureCopyLocation PlacedFootprint(RenderBuffer* b,RenderFormat format,u32 w,u32 h,u32 d,u32 p,u32 offset){ + assert(format==RenderFormat::R32_FLOAT && d==1 && offset==0);return{nullptr,b,w,h,p};} }; +struct RenderCommandList {u32 copies=0;std::vector barriers_seen; + void barriers(RenderBarrierStage,RenderTextureBarrier b){barriers_seen.push_back(b.layout);} + void copyTextureRegion(RenderTextureCopyLocation dst,RenderTextureCopyLocation src){ + assert(dst.texture && src.buffer);++copies;dst.texture->contents=src.buffer->contents; + assert(src.width==dst.texture->width && src.height==dst.texture->height); + assert(src.pitch%64==0 && src.pitch>=src.width); } }; +} +enum class D3DResourceType { kTexture=3,kVolumeTexture=4,kCubeTexture=5 }; +struct CpuMemoryStamp {u64 generation=0;}; struct CpuMemorySpan {u32 address,size;}; +struct TextureUploadSourceKey {u64 hash=0;bool operator==(const TextureUploadSourceKey&) const=default;}; +u64 XXH3_64bits(const void* ptr,size_t size) {auto* p=static_cast(ptr);u64 hash=14695981039346656037ull; + while(size--)hash=(hash^*p++)*1099511628211ull;return hash;} +TextureUploadSourceKey MakeTextureUploadSourceKey(std::span base,std::span,std::span metadata){ + return{XXH3_64bits(base.data(),base.size())^XXH3_64bits(metadata.data(),metadata.size())};} +struct TextureResource { + u32 guest_address=0x1234,width=67,height=35,levels=1,d3d_type=3,guest_format=22; + bool surface=false,owns_guest_memory=false,resolved_on_host=false,guest_uploaded=false,sampled_valid=false; + std::mutex mutex;plume::RenderFormat format=plume::RenderFormat::D32_FLOAT_S8; + rex::graphics::xenos::Fetch guest_fetch; + std::unique_ptr texture=std::make_unique(); + std::unique_ptr sampled_texture; + std::unique_ptr view=std::make_unique(),sampled_view,resolved_view; + u32 descriptor_index=4,sampled_descriptor_index=~0u,resolved_descriptor_index=~0u; + u64 guest_content_hash=0,depth_resolve_generation=0; + u64 last_authority_probe_signature=0;bool authority_probe_signature_valid=false; + TextureUploadSourceKey upload_source_key;bool upload_source_key_valid=false;CpuMemoryStamp cpu_stamp; + u32 HostWidth()const{return width;}u32 HostHeight()const{return height;} +}; +struct TextureResourceView {plume::RenderTexture* texture;plume::RenderTextureView* view;plume::RenderFormat format; + u32 descriptor_index,width,height,d3d_type;bool surface,depth;}; +struct {u64 source_hits=0,hashed_bytes=0,converted_bytes=0;double hash_ms=0,source_ms=0;} g_upload_timing; +bool NativeTextureTimingEnabled(){return true;} +struct UploadTimer{~UploadTimer(){}}; +bool IsDepthFormat(plume::RenderFormat f){return f==plume::RenderFormat::D32_FLOAT_S8;} +bool LongProbeEnabled(){return false;} bool LongProbeOnce(u64){return false;} +templatevoid LongProbeEvent(T&&...){} +u32 RowPitch(plume::RenderFormat f,u32 width){assert(f==plume::RenderFormat::R32_FLOAT);return rex::align(width*4,256);} +std::vector guest_bytes; +bool readable=true;u32 translations=0;u64 generation=1; +struct Memory { + templateT TranslatePhysical(u32 address){assert(readable && address==0x1000);++translations;return reinterpret_cast(guest_bytes.data());} + templateT TranslateVirtual(u32 address){assert(readable && address==0xE0001000);++translations;return reinterpret_cast(guest_bytes.data());} +} memory; +#define REX_KERNEL_MEMORY() (&memory) +#define REXLOG_INFO(...) ((void)0) +bool ReadableUploadSpan(u32 address,u32 size){return readable && (address==0x1000||address==0xE0001000) && size<=guest_bytes.size();} +bool CpuMemoryWatchEnabled(){return true;} +CpuMemoryStamp WatchCpuMemory(std::span spans){assert(spans.size()==1);return{generation};} +bool CpuMemoryUnchanged(const CpuMemoryStamp& stamp){return stamp.generation==generation;} +struct Device { + bool fail_texture=false,fail_native=false,fail_buffer=false; + std::unique_ptr createTexture(const plume::RenderTextureDesc& desc){ + if(fail_texture)return nullptr;auto texture=std::make_unique(); + texture->width=desc.width;texture->height=desc.height;if(fail_native)texture->d3d=nullptr;return texture;} + std::unique_ptr createBuffer(plume::RenderBufferDesc desc){ + return fail_buffer ? nullptr : std::make_unique(desc.size);} +} device; +struct Region {u32 left=0,top=0,width=67,height=35,x=0,y=0;}; +enum class ColorResolveDestination {kDepthFloat32}; +struct HostDevice { + static inline std::recursive_mutex mutex; + static inline u32 next_descriptor=100,registrations=0; + static inline bool fail_descriptor=false,fail_resolve=false; + static inline std::vector> retired; + static auto LockRecording(){return std::unique_lock(mutex);} + static auto Device(){return &device;} + static u32 RegisterTexture(plume::RenderTexture*,plume::RenderTextureView*){ + if(fail_descriptor)return ~0u;++registrations;return next_descriptor++;} + static void RetireResource(std::shared_ptr buffer){retired.push_back(std::move(buffer));} + static bool ResolveHdrColor(plume::RenderTexture*,u32,plume::RenderTexture* dst,u32 w,u32 h,float, + ColorResolveDestination,bool,const Region*){ + if(fail_resolve)return false;dst->contents.assign(RowPitch(plume::RenderFormat::R32_FLOAT,w)*h,0x5A);return true;} + static void SnapshotTexture(plume::RenderTexture*,const std::string&,bool){} +}; +std::unordered_map> textures; +auto FindTexture(u32 address){auto it=textures.find(address);return it==textures.end()?nullptr:it->second;} +auto AdoptTexture(u32 address){return FindTexture(address);} +bool DepthAliasesEnabled(){return true;} +struct Alias {u32 physical_base=0x1000;}; Alias AliasLayout(const TextureResource&){return{};} +u64 g_depth_alias_generation=0; +std::unordered_map> g_depth_resolves; +''' + harness += bodies + '\n' + upload + harness += r''' +bool ResolveDepth(const std::shared_ptr& adopted_destination) { + auto source=std::make_shared();source->surface=true; + std::unique_ptr region=std::make_unique(); + u32 destination_level=0,destination_slice=0,resolve_flags=4,destination_texture=adopted_destination->guest_address; +''' + harness += resolve_branch + ' return false;\n}\n' + harness += r''' +u32 BoundShaderTextureMask(int){return 0;} +enum class ShaderStage {kVertex};u32 BoundShaderTextureMask(ShaderStage){return 0;} +bool BindTexture(u32 address,plume::RenderCommandList* commands,u32& selected) { + struct{std::array textures{};}bindings;bindings.textures[15]=address; + for(u32 i=15;i<16;++i){ +''' + harness += bind + r''' + selected=texture.descriptor_index; + } return true; +} +void Fill(TextureResource& resource,u32 endian,bool tiled) { + resource.guest_fetch.endianness=endian;resource.guest_fetch.tiled=tiled; + auto layout=rex::graphics::texture_util::GetGuestTextureLayout(resource.guest_fetch.dimension, + resource.guest_fetch.pitch,resource.width,resource.height,1,tiled,22,false,true,0); + guest_bytes.assign(layout.base.level_data_extent_bytes,0xCD); + const u32 xors[]={0,1,3,2}; + for(u32 y=0;y(&raw); + for(u32 b=0;b<4;++b)guest_bytes[(offset+b)^xors[endian]]=bytes[b]; + } + ++generation; +} +void CheckPixels(const TextureResource& resource) { + const auto& bytes=resource.sampled_texture->contents;const u32 pitch=RowPitch(plume::RenderFormat::R32_FLOAT,resource.width); + for(u32 y=0;y>23))*std::ldexp(1.0f,-24); + assert(value==sdk_value);assert(std::abs(double(value)-double(depth)/16777215.0)<0.00000006); + } + for(u32 y=0;y();textures[resource->guest_address]=resource;Fill(*resource,endian,tiled); + plume::RenderCommandList commands;u32 selected=~0u; + assert(BindTexture(resource->guest_address,&commands,selected));assert(commands.copies==1); + assert(selected==resource->sampled_descriptor_index && selected!=resource->descriptor_index); + assert(!resource->resolved_on_host && resource->sampled_valid && resource->guest_uploaded); + assert((resource->sampled_view->desc.componentMapping.lanes==std::array{plume::RenderSwizzle::R,plume::RenderSwizzle::R,plume::RenderSwizzle::ONE,plume::RenderSwizzle::ONE})); + CheckPixels(*resource); + assert((commands.barriers_seen==std::vector{plume::RenderTextureLayout::COPY_DEST,plume::RenderTextureLayout::SHADER_READ})); + const auto copies=commands.copies;const auto before=translations; + assert(UploadTextureResource(resource->guest_address,&commands,false));assert(commands.copies==copies && translations==before); + // Required verification catches direct guest writes even when no watch generation changed. + guest_bytes[0]^=1;assert(UploadTextureResource(resource->guest_address,&commands,true));assert(commands.copies==copies+1); + const auto destroyed=plume::RenderBuffer::destroyed;assert(!HostDevice::retired.empty()); + HostDevice::retired.clear();assert(plume::RenderBuffer::destroyed>destroyed); + // Actual host depth-atlas resolve branch takes authority; stale CPU memory cannot replace it. + assert(ResolveDepth(resource));assert(resource->resolved_on_host && resource->sampled_valid); + const auto resolved_bytes=resource->sampled_texture->contents;readable=false;++generation; + assert(UploadTextureResource(resource->guest_address,&commands,true)); + assert(resource->sampled_texture->contents==resolved_bytes);readable=true; + } + // A watched generation change must also refresh the real first-use mirror. + { + auto resource=std::make_shared();textures[resource->guest_address]=resource;Fill(*resource,2,true); + plume::RenderCommandList commands;assert(UploadTextureResource(resource->guest_address,&commands,false)); + guest_bytes[0]^=0x80;++generation; + assert(UploadTextureResource(resource->guest_address,&commands,false));assert(commands.copies==2); + float value;std::memcpy(&value,resource->sampled_texture->contents.data(),4); + assert(value==UnpackDepth24(0x800000)); + } + // Exact atlas dimensions exercise the full non-square SDK tiled address range. + { + auto resource=std::make_shared();resource->width=960;resource->height=3840;resource->guest_fetch.pitch=30; + textures[resource->guest_address]=resource;Fill(*resource,2,true); + plume::RenderCommandList commands;assert(UploadTextureResource(resource->guest_address,&commands,false));CheckPixels(*resource); + } + // Every computed byte address must remain within the resident source span. + { + std::array src{},dst{}; + assert(!CopyTextureDepth24(src,dst,1,1,4,0,[](u32,u32,u32)->int64_t{return -1;})); + assert(!CopyTextureDepth24(src,dst,1,1,4,3,[](u32,u32,u32)->int64_t{return 2;})); + assert(!CopyTextureDepth24(src,dst,1,1,4,0,[](u32,u32,u32)->int64_t{return INT64_MAX;})); + assert(!CopyTextureDepth24(src,dst,2,1,4,0,[](u32,u32,u32)->int64_t{return 0;})); + } + // Invalid source/extent never reaches translation; FLOAT24 and unsupported shapes remain rejected. + for(int failure=0;failure<13;++failure){ + auto resource=std::make_shared();textures[resource->guest_address]=resource;Fill(*resource,2,true); + if(failure==0)readable=false; + if(failure==1)rex::graphics::texture_util::oversized_slice=true; + if(failure==2)resource->guest_format=23; + if(failure==3)resource->levels=2; + if(failure==4)resource->guest_fetch.stacked=1; + if(failure==5)resource->guest_fetch.dimension=rex::graphics::xenos::DataDimension::k3D; + if(failure==6)device.fail_texture=true; + if(failure==7)device.fail_native=true; + if(failure==8)plume::fail_view=true; + if(failure==9)HostDevice::fail_descriptor=true; + if(failure==10)device.fail_buffer=true; + if(failure==11)plume::fail_map=true; + const auto before=translations;plume::RenderCommandList commands;u32 selected=~0u; + assert(!BindTexture(resource->guest_address,failure==12?nullptr:&commands,selected)); + assert(!resource->sampled_valid && !resource->guest_uploaded && commands.copies==0 && selected==~0u); + if(failure<2||failure==12)assert(translations==before); + if(failure>=6 && failure<=9)assert(!resource->sampled_texture && resource->sampled_descriptor_index==~0u); + readable=true;rex::graphics::texture_util::oversized_slice=false; + device.fail_texture=device.fail_native=device.fail_buffer=false;plume::fail_view=plume::fail_map=HostDevice::fail_descriptor=false; + if(failure>=6 && failure<=11){ + const auto descriptor=resource->sampled_descriptor_index;const auto registrations=HostDevice::registrations; + assert(BindTexture(resource->guest_address,&commands,selected));assert(resource->sampled_valid && resource->guest_uploaded); + if(failure>=10){assert(selected==descriptor && HostDevice::registrations==registrations);} + } + } + // Simulate the existing relocation/pool-copy invalidation boundary flags; next upload refreshes this mirror. + auto resource=std::make_shared();textures[resource->guest_address]=resource;Fill(*resource,2,true); + plume::RenderCommandList commands;assert(UploadTextureResource(resource->guest_address,&commands,false)); + resource->guest_uploaded=false;resource->upload_source_key_valid=false;resource->cpu_stamp={};Fill(*resource,2,true); + resource->guest_fetch.base_address=0xE0001;assert(UploadTextureResource(resource->guest_address,&commands,false));assert(commands.copies==2); + auto view=ResolveTextureResource(resource->guest_address);assert(view.format==plume::RenderFormat::R32_FLOAT && !view.depth); + HostDevice::retired.clear(); + std::cout<<"Production UNORM24 depth upload, descriptor refresh and host-authority regressions passed\n"; +} +''' + args.output.mkdir(parents=True, exist_ok=True) + fixture = args.output.resolve() / 'test.cpp' + binary = args.output.resolve() / 'test.exe' + fixture.write_text(harness) + subprocess.run([args.compiler, '-std=c++20', '-UNDEBUG', '-Wall', '-Wextra', + '-I', str(root / 'rexlego/src/gpu_native'), str(fixture), '-o', str(binary)], + check=True, timeout=45) + subprocess.run([str(binary)], check=True, timeout=10) + (args.output / 'verification.json').write_text(json.dumps({ + 'production_source_sha256': hashlib.sha256(source_path.read_bytes()).hexdigest(), + 'sdk_depth_loader_sha256': hashlib.sha256(sdk_path.read_bytes()).hexdigest(), + 'decoder_sha256': hashlib.sha256((root / 'rexlego/src/gpu_native/texture_depth_upload.h').read_bytes()).hexdigest(), + 'actual_bodies': ['UploadGuestDepthTexture', 'EnsureDepthSamplingMirror', + 'ResolveTextureResource', 'UploadTextureResource entry guards', + 'ResolveTextureFromSurface depth atlas branch', 'BindConstants upload/refresh'], + 'result': 'passed'}, indent=2)+'\n') + + +if __name__ == '__main__': + main() diff --git a/tools/gpu-aot/test_texture_source_residency.py b/tools/gpu-aot/test_texture_source_residency.py new file mode 100644 index 0000000..c76113c --- /dev/null +++ b/tools/gpu-aot/test_texture_source_residency.py @@ -0,0 +1,82 @@ +"""Execute the production texture-upload residency guard with synthetic heaps.""" + +import argparse +from pathlib import Path +import subprocess + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + text = (root / "rexlego/src/gpu_native/textures.cpp").read_text() + begin = text.index(" const bool source_key_valid =", text.index("bool UploadTextureResource(")) + end = text.index(" if (source_key_valid) {", begin) + guard = text[begin:end] + harness = r''' +#include +#include +#include +#include +using u32 = uint32_t; +bool base_readable = true, mip_readable = true; +u32 residency_queries = 0; +bool ReadableUploadSpan(u32 address, u32 size) { + ++residency_queries; + assert(size != 0); + if (address == 0x1000) return base_readable; + if (address == 0x2000) return mip_readable; + return false; +} +bool LongProbeEnabled() { return false; } +bool LongProbeOnce(uint64_t) { return false; } +template void LongProbeEvent(Args&&...) {} +bool SourceResident(bool volume, bool alpha4, bool capture_pending, + u32 mip_address, u32 mip_size) { + struct { u32 base_address, mip_address; } fetch{1, mip_address}; + struct { struct { u32 level_data_extent_bytes = 64; } base; } layout; + const u32 guest_address = 0x3000; +''' + harness += guard + r''' + return true; +} +int main() { + // 2D/cube uploads have no special-path flags. These must reject unreadable + // memory even when capture is off, just like volumes and DXT3A. + for (auto flags : std::array, 4>{{ + {false, false, false}, {true, false, false}, + {false, true, false}, {false, false, true}}}) { + base_readable = false; + mip_readable = true; + residency_queries = 0; + assert(!SourceResident(flags[0], flags[1], flags[2], 2, 64)); + assert(residency_queries == 1); + base_readable = true; + mip_readable = false; + assert(!SourceResident(flags[0], flags[1], flags[2], 2, 64)); + assert(!SourceResident(flags[0], flags[1], flags[2], 0, 64)); + // Base-only uploads do not require mip memory. + residency_queries = 0; + assert(SourceResident(flags[0], flags[1], flags[2], 0, 0)); + assert(residency_queries == 1); + mip_readable = true; + residency_queries = 0; + assert(SourceResident(flags[0], flags[1], flags[2], 2, 64)); + assert(residency_queries == 2); + } + std::cout << "Texture-upload production residency guard regressions passed\n"; +} +''' + args.output.mkdir(parents=True, exist_ok=True) + source = args.output.resolve() / "test.cpp" + binary = args.output.resolve() / "test.exe" + source.write_text(harness) + subprocess.run([args.compiler, "-std=c++20", "-UNDEBUG", str(source), "-o", str(binary)], + check=True, timeout=45) + subprocess.run([str(binary)], check=True, timeout=10) + + +if __name__ == "__main__": + main() diff --git a/tools/gpu-aot/test_texture_upload_capture.py b/tools/gpu-aot/test_texture_upload_capture.py index 0ff6a31..5867d76 100644 --- a/tools/gpu-aot/test_texture_upload_capture.py +++ b/tools/gpu-aot/test_texture_upload_capture.py @@ -4,9 +4,11 @@ from pathlib import Path import struct import subprocess +import shutil p=argparse.ArgumentParser(description=__doc__) p.add_argument('output',type=Path) +p.add_argument('--compiler',default='clang++') a=p.parse_args() a.output.mkdir(parents=True,exist_ok=True) root=Path(__file__).resolve().parents[2] @@ -21,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +32,14 @@ #include #include using u8=uint8_t;using u32=uint32_t;using u64=uint64_t; +#ifndef _WIN32 +int _putenv_s(const char* name,const char* value){return setenv(name,value,1);} +int _dupenv_s(char** value,size_t* size,const char* name){ + const char* input=std::getenv(name);*value=nullptr;*size=0; + if(input){*size=std::strlen(input)+1;*value=static_cast(std::malloc(*size));std::memcpy(*value,input,*size);} + return 0; +} +#endif std::atomic g_probe_frame{0}; namespace plume {enum class RenderFormat {BC1_UNORM,BC3_UNORM,R8G8B8A8_UNORM};} enum class ShaderStage {kVertex,kPixel}; @@ -39,9 +50,33 @@ u32 RowPitch(plume::RenderFormat,u32){return 256;} u32 RowCount(plume::RenderFormat f,u32 h){return f==plume::RenderFormat::R8G8B8A8_UNORM?h:(h+3)/4;} #define REXLOG_INFO(...) ((void)0) +int warnings=0; +#define REXLOG_WARN(...) (++warnings) '''+function+r''' int main(int argc,char**argv){ _putenv_s("LEGO_DUMP_TEXTURE_UPLOADS",argv[1]); + if(argc>4){ + _putenv_s("LEGO_DUMP_TEXTURE_UPLOADS_PIXEL_SHADER",argv[3]); + TextureResource r{plume::RenderFormat::BC3_UNORM,42}; + std::vector data(256*7,0xAB); + if(argv[4][0]=='0'){ + for(int i=0;i<3;++i)DumpTextureUpload(r,data.data()); + if(warnings!=1 || !TextureCapture().dumped.empty() || TextureCapture().written_bytes || + std::filesystem::exists(argv[1]))return 20; + return 0; + } + const u64 hash=std::strtoull(argv[3],nullptr,16); + ps=hash^1; + DumpTextureUpload(r,data.data()); + if(!TextureCapture().dumped.empty() || TextureCapture().written_bytes || + std::filesystem::exists(argv[1]))return 21; + ps=hash; + if(!TextureUploadCapturePending(r))return 22; + DumpTextureUpload(r,data.data()); + if(warnings || TextureCapture().dumped.size()!=1 || !TextureCapture().written_bytes || + !std::filesystem::is_regular_file(std::filesystem::path(argv[1])/"42.dds"))return 23; + return 0; + } if(argc>2){ _putenv_s("LEGO_DUMP_TEXTURE_UPLOADS_ALL_LEVELS","1"); _putenv_s("LEGO_DUMP_TEXTURE_UPLOADS_TRIGGER",argv[2]); @@ -101,9 +136,9 @@ } ''') exe=a.output/'test.exe' -subprocess.run(['clang++','-std=c++20','-O2',str(cpp),'-o',str(exe)],check=True) +subprocess.run([a.compiler,'-std=c++20','-O2',str(cpp),'-o',str(exe)],check=True,timeout=45) data=a.output/'dds' -subprocess.run([str(exe.resolve()),str(data.resolve())],check=True) +subprocess.run([str(exe.resolve()),str(data.resolve())],check=True,timeout=10) assert not (data/'999.dds').exists() for name,code,row_bytes,rows in [('1',b'DXT5',32,2),('2',b'DXT1',16,2),('3',b'\0'*4,20,7)]: b=(data/(name+'.dds')).read_bytes() @@ -115,7 +150,7 @@ all_data=a.output/'all-mips' trigger=a.output/'arm-capture' trigger.unlink(missing_ok=True) -subprocess.run([str(exe.resolve()),str(all_data.resolve()),str(trigger.resolve())],check=True) +subprocess.run([str(exe.resolve()),str(all_data.resolve()),str(trigger.resolve())],check=True,timeout=10) b=(all_data/'armed/42.dds').read_bytes() assert struct.unpack_from(' Date: Wed, 7 Oct 2026 03:46:26 +0200 Subject: [PATCH 04/40] Validate index ranges and upload native draw vertex windows Reject overflowing or out-of-bounds indexed draws before GPU submission. Extend opt-in range uploads to nonindexed draws and QuadList expansion, retaining stream byte order, signed base vertices, query scopes and fenced upload lifetimes. Validation: actual stream selection, index guards and QuadList byte-equivalence fixtures passed from the staged commit snapshot, including the excluded-QuadList negative control. The native build and 51-check portable suite passed. Live paused allocation probes used bounded ranges rather than whole mesh-pool copies; no isolated frame-rate or pixel-equivalence claim is made. --- rexlego/src/gpu_native/buffer_window.h | 8 + rexlego/src/gpu_native/draw.cpp | 100 ++++--- rexlego/src/gpu_native/index_draw.h | 32 ++ tools/gpu-aot/test_draw_index_bounds.py | 138 +++++++++ .../gpu-aot/test_nonindexed_buffer_windows.py | 177 +++++++++++ tools/gpu-aot/test_quad_buffer_windows.py | 276 ++++++++++++++++++ 6 files changed, 693 insertions(+), 38 deletions(-) create mode 100644 rexlego/src/gpu_native/index_draw.h create mode 100644 tools/gpu-aot/test_draw_index_bounds.py create mode 100644 tools/gpu-aot/test_nonindexed_buffer_windows.py create mode 100644 tools/gpu-aot/test_quad_buffer_windows.py diff --git a/rexlego/src/gpu_native/buffer_window.h b/rexlego/src/gpu_native/buffer_window.h index 75ab2f7..1296792 100644 --- a/rexlego/src/gpu_native/buffer_window.h +++ b/rexlego/src/gpu_native/buffer_window.h @@ -16,4 +16,12 @@ constexpr VertexBufferWindow DrawVertexWindow(uint32_t min_index, uint32_t max_i offset % 4 || length % 4) return {}; return {uint32_t(offset), uint32_t(length)}; } + +// Nonindexed vertices form a contiguous range. Widen before adding count so +// an overflowing range cannot turn into a small apparently valid upload. +constexpr bool NonIndexedVertexRange(uint32_t start, uint32_t count, uint32_t& last) { + if (!count || uint64_t(start) + count - 1 > UINT32_MAX) return false; + last = uint32_t(uint64_t(start) + count - 1); + return true; +} } diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp index a9b199d..1aea608 100644 --- a/rexlego/src/gpu_native/draw.cpp +++ b/rexlego/src/gpu_native/draw.cpp @@ -25,6 +25,7 @@ #include "gpu_native/buffers.h" #include "gpu_native/buffer_window.h" +#include "gpu_native/index_draw.h" #include "gpu_native/alpha_test.h" #include "gpu_native/blend_state.h" #include "gpu_native/queries.h" @@ -661,6 +662,16 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, } return false; } + BufferResourceView index_metadata; + IndexDrawWindow index_window; + if (indexed) { + index_metadata = InspectBufferResource(bindings.index_buffer, BufferKind::kIndex); + index_window = DrawIndexWindow(start, count, + index_metadata.guest_format == 1 ? 2 : 4, index_metadata.length); + // An invalid window must reject the draw, rather than disabling range + // uploads and falling through to an unchecked whole-buffer submission. + if (!index_metadata.mirror_address || !index_window) return false; + } timing.Next(); auto* commands = HostDevice::BeginFrameCommands(); if (!commands) { @@ -853,18 +864,15 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, }(); std::array vertex_windows{}; bool use_windows = false; - u32 min_index = 0, index_offset = 0, index_bytes = 0; - if (window_uploads && indexed && count && - primitive_type != static_cast(rex::graphics::xenos::PrimitiveType::kQuadList)) { - const auto index = InspectBufferResource(bindings.index_buffer, BufferKind::kIndex); - const u32 element_size = index.guest_format == 1 ? 2 : 4; - if (index.mirror_address && u64(start) * element_size <= index.length && - u64(count) * element_size <= index.length - u64(start) * element_size) { - index_offset = start * element_size; - index_bytes = count * element_size; + u32 min_index = 0; + const u32 index_offset = index_window.offset, index_bytes = index_window.length; + if (window_uploads && count) { + u32 max_index = 0; + if (indexed) { + const auto& index = index_metadata; + const u32 element_size = index.guest_format == 1 ? 2 : 4; const auto* source = REX_KERNEL_MEMORY()->TranslateVirtual(index.mirror_address + index_offset); min_index = UINT32_MAX; - u32 max_index = 0; for (u32 i = 0; i < count; ++i) { const u32 value = element_size == 2 ? u32(reinterpret_cast(source)[i]) : @@ -874,26 +882,29 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, } // D3D12's signed base vertex must represent -min_index exactly. use_windows = min_index <= INT32_MAX; - for (u32 i = 0; use_windows && i < kNativeVertexStreams; ++i) { - const auto& binding = bindings.vertex_streams[i]; - if (!binding.buffer) continue; // Synthetic/optional zero stream. - const bool used = std::any_of(declaration.elements, - declaration.elements + declaration.element_count, - [i](const auto& e) { return e.slotIndex == i; }); - if (!used) continue; - for (u32 e = 0; e < declaration.element_count; ++e) { - const auto& element = declaration.elements[e]; - if (element.slotIndex != i) continue; - const u32 width = plume::RenderFormatSize(element.format); - if (!width || element.alignedByteOffset > binding.stride || - width > binding.stride - element.alignedByteOffset) use_windows = false; - } - if (!use_windows) break; - const auto metadata = InspectBufferResource(binding.buffer, BufferKind::kVertex); - vertex_windows[i] = DrawVertexWindow(min_index, max_index, base_vertex, - binding.stride, binding.offset, metadata.length); - if (!vertex_windows[i].length) use_windows = false; + } else { + min_index = start; + use_windows = NonIndexedVertexRange(start, count, max_index); + } + for (u32 i = 0; use_windows && i < kNativeVertexStreams; ++i) { + const auto& binding = bindings.vertex_streams[i]; + if (!binding.buffer) continue; // Synthetic/optional zero stream. + const bool used = std::any_of(declaration.elements, + declaration.elements + declaration.element_count, + [i](const auto& e) { return e.slotIndex == i; }); + if (!used) continue; + for (u32 e = 0; e < declaration.element_count; ++e) { + const auto& element = declaration.elements[e]; + if (element.slotIndex != i) continue; + const u32 width = plume::RenderFormatSize(element.format); + if (!width || element.alignedByteOffset > binding.stride || + width > binding.stride - element.alignedByteOffset) use_windows = false; } + if (!use_windows) break; + const auto metadata = InspectBufferResource(binding.buffer, BufferKind::kVertex); + vertex_windows[i] = DrawVertexWindow(min_index, max_index, indexed ? base_vertex : 0, + binding.stride, binding.offset, metadata.length); + if (!vertex_windows[i].length) use_windows = false; } } for (u32 i = 0; i < kNativeVertexStreams; ++i) { @@ -1023,27 +1034,34 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, } if (primitive_type == static_cast(rex::graphics::xenos::PrimitiveType::kQuadList)) { - if (count % 4 != 0) return false; + const u32 expanded_count = ExpandedQuadIndexCount(count); + u32 nonindexed_last = 0; + if (!expanded_count || (!indexed && !NonIndexedVertexRange(start, count, nonindexed_last))) + return false; const auto source = indexed - ? ResolveBufferResourceView(bindings.index_buffer, BufferKind::kIndex) + ? (use_windows ? + ResolveBufferResourceWindow(bindings.index_buffer, BufferKind::kIndex, index_offset, index_bytes) : + ResolveBufferResourceView(bindings.index_buffer, BufferKind::kIndex)) : BufferResourceView{}; + const u32 source_start = use_windows ? 0 : start; const u32 element_size = source.guest_format == 1 ? 2 : 4; - if (indexed && (!source.buffer || u64(start + u64(count)) * element_size > source.length)) + if (indexed && (!source.buffer || !DrawIndexWindow(source_start, count, element_size, source.length))) return false; auto expanded = std::shared_ptr(HostDevice::Device()->createBuffer( - plume::RenderBufferDesc::IndexBuffer(u64(count / 4) * 6 * sizeof(u32), + plume::RenderBufferDesc::IndexBuffer(u64(expanded_count) * sizeof(u32), plume::RenderHeapType::UPLOAD)).release()); if (!expanded) return false; auto* output = static_cast(expanded->map()); const auto* input = indexed ? static_cast(source.buffer->map()) : nullptr; if (!output || (indexed && !input)) { if (output) expanded->unmap(); + if (input) source.buffer->unmap(); return false; } constexpr u32 order[] = {0, 1, 2, 0, 2, 3}; for (u32 quad = 0; quad < count / 4; ++quad) { for (u32 corner = 0; corner < 6; ++corner) { - const u32 index = start + quad * 4 + order[corner]; + const u32 index = source_start + quad * 4 + order[corner]; output[quad * 6 + corner] = !indexed ? index : element_size == 2 ? reinterpret_cast(input)[index] : reinterpret_cast(input)[index]; @@ -1052,18 +1070,22 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, if (indexed) source.buffer->unmap(); expanded->unmap(); const plume::RenderIndexBufferView index_view(expanded->at(0), - u64(count / 4) * 6 * sizeof(u32), plume::RenderFormat::R32_UINT); + u64(expanded_count) * sizeof(u32), plume::RenderFormat::R32_UINT); commands->setIndexBuffer(&index_view); { QueryDrawScope query_scope(commands); - commands->drawIndexedInstanced(count / 4 * 6, 1, 0, indexed ? base_vertex : 0, 0); + commands->drawIndexedInstanced(expanded_count, 1, 0, + use_windows ? (indexed ? -i32(min_index) : 0) : (indexed ? base_vertex : 0), 0); } HostDevice::RetireResource(std::move(expanded)); } else if (indexed) { const auto index = use_windows ? ResolveBufferResourceWindow(bindings.index_buffer, BufferKind::kIndex, index_offset, index_bytes) : ResolveBufferResourceView(bindings.index_buffer, BufferKind::kIndex); - if (!index.buffer) { + // Header adoption can replace metadata; check the view actually bound to + // this draw too. A range upload starts at zero in its own index buffer. + if (!index.buffer || !DrawIndexWindow(use_windows ? 0 : start, count, + index.guest_format == 1 ? 2 : 4, index.length)) { return false; } const plume::RenderFormat index_format = @@ -1077,7 +1099,9 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, use_windows ? -i32(min_index) : base_vertex, 0); } else { QueryDrawScope query_scope(commands); - commands->drawInstanced(count, 1, start, 0); + // Current native vertex recompilation consumes input attributes, not + // SV_VertexID. Each used stream starts at the original first vertex. + commands->drawInstanced(count, 1, use_windows ? 0 : start, 0); } timing.Next(); static const bool trace_stages = [] { diff --git a/rexlego/src/gpu_native/index_draw.h b/rexlego/src/gpu_native/index_draw.h new file mode 100644 index 0000000..ba0ffef --- /dev/null +++ b/rexlego/src/gpu_native/index_draw.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace legodimensions::gpu_native { + +struct IndexDrawWindow { + uint32_t offset = 0; + uint32_t length = 0; + explicit constexpr operator bool() const { return length != 0; } +}; + +// Six triangle indices replace every four QuadList vertices. The host draw +// count is uint32_t even though buffer allocation sizes use uint64_t. +constexpr uint32_t ExpandedQuadIndexCount(uint32_t count) { + const uint64_t expanded = uint64_t(count / 4) * 6; + return count && count % 4 == 0 && expanded <= UINT32_MAX ? uint32_t(expanded) : 0; +} + +// This interval is independent of signed base_vertex: it covers index fetches, +// while base_vertex affects the subsequent vertex buffer addresses. +constexpr IndexDrawWindow DrawIndexWindow(uint32_t start, uint32_t count, + uint32_t element_bytes, + uint32_t buffer_length) { + if (!count || (element_bytes != 2 && element_bytes != 4)) return {}; + const uint64_t offset = uint64_t(start) * element_bytes; + const uint64_t length = uint64_t(count) * element_bytes; + if (offset > buffer_length || length > buffer_length - offset) return {}; + return {uint32_t(offset), uint32_t(length)}; +} + +} // namespace legodimensions::gpu_native diff --git a/tools/gpu-aot/test_draw_index_bounds.py b/tools/gpu-aot/test_draw_index_bounds.py new file mode 100644 index 0000000..2d9911c --- /dev/null +++ b/tools/gpu-aot/test_draw_index_bounds.py @@ -0,0 +1,138 @@ +"""Run production indexed-draw guards with an asset-free command recorder.""" +import argparse +from pathlib import Path +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('output', type=Path) +p.add_argument('--compiler', default='clang++') +a = p.parse_args() +a.output.mkdir(parents=True, exist_ok=True) +root = Path(__file__).resolve().parents[2] +source = (root / 'rexlego/src/gpu_native/draw.cpp').read_text() +preflight_start = source.index(' BufferResourceView index_metadata;') +preflight_end = source.index(' timing.Next();', preflight_start) +issue_start = source.index(' } else if (indexed) {') +opening = source.index('{', issue_start) +depth, end = 1, opening + 1 +while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 +issue = source[opening + 1:end - 1] +code = r''' +#include +#include +#include +#include +#include "gpu_native/index_draw.h" +using namespace legodimensions::gpu_native; +using u32=uint32_t;using i32=int32_t; +struct FakeBuffer {FakeBuffer* at(u32 offset){assert(offset==0);return this;}} buffer; +struct BufferResourceView { + FakeBuffer* buffer=nullptr; + u32 length=0,guest_format=0,mirror_address=0; +}; +enum class BufferKind{kIndex}; +struct DrawBindings{u32 index_buffer=123;}; +BufferResourceView metadata,full_view,window_view; +u32 requested_offset=0,requested_length=0; +BufferResourceView InspectBufferResource(u32 guest,BufferKind){assert(guest==123);return metadata;} +BufferResourceView ResolveBufferResourceView(u32 guest,BufferKind){assert(guest==123);return full_view;} +BufferResourceView ResolveBufferResourceWindow(u32 guest,BufferKind,u32 offset,u32 length){ + assert(guest==123);requested_offset=offset;requested_length=length;return window_view; +} +namespace plume { +enum class RenderFormat{R16_UINT,R32_UINT}; +struct RenderIndexBufferView { + FakeBuffer* buffer; + u32 length; + RenderFormat format; + RenderIndexBufferView(FakeBuffer* b,u32 l,RenderFormat f):buffer(b),length(l),format(f){} +}; +} +struct CommandRecorder { + u32 bindings=0,draws=0,count=0,start=0; + i32 base=0; + plume::RenderFormat format=plume::RenderFormat::R32_UINT; + void setIndexBuffer(const plume::RenderIndexBufferView* view){++bindings;format=view->format;} + void drawIndexedInstanced(u32 n,u32 instances,u32 s,i32 b,u32 first_instance){ + assert(instances==1&&first_instance==0);++draws;count=n;start=s;base=b; + } +}; +u32 query_scopes=0; +struct QueryDrawScope{explicit QueryDrawScope(CommandRecorder*){++query_scopes;}}; +bool Preflight(bool indexed,u32 start,u32 count,const DrawBindings& bindings){ +''' + source[preflight_start:preflight_end] + r''' + return true; +} +bool Issue(bool use_windows,u32 start,u32 count,i32 base_vertex,u32 min_index, + u32 index_offset,u32 index_bytes,const DrawBindings& bindings,CommandRecorder* commands){ +''' + issue + r''' + return true; +} +int main(){ + const DrawBindings bindings; + for(u32 format:{1u,2u,0u}) { + const u32 width=format==1?2:4; + metadata={nullptr,16*width,format,0x10000}; + assert(Preflight(true,15,1,bindings)); + assert(!Preflight(true,15,2,bindings)); + assert(!Preflight(true,16,1,bindings)); + assert(!Preflight(true,UINT32_MAX,1,bindings)); + assert(!Preflight(true,1,UINT32_MAX,bindings)); + metadata.mirror_address=0;assert(!Preflight(true,0,1,bindings)); + metadata={};assert(Preflight(false,UINT32_MAX,UINT32_MAX,bindings)); + full_view={&buffer,16*width,format,0x10000}; + CommandRecorder commands;query_scopes=0; + assert(Issue(false,15,1,-7,0,0,0,bindings,&commands)); + assert(commands.draws==1&&commands.bindings==1&&query_scopes==1); + assert(commands.count==1&&commands.start==15&&commands.base==-7); + assert(commands.format==(format==1?plume::RenderFormat::R16_UINT:plume::RenderFormat::R32_UINT)); + for(auto parameters:{std::pair{15u,2u},std::pair{16u,1u}, + std::pair{UINT32_MAX,1u},std::pair{1u,UINT32_MAX}}){ + commands={};query_scopes=0; + assert(!Issue(false,parameters.first,parameters.second,19,0,0,0,bindings,&commands)); + assert(commands.bindings==0&&commands.draws==0&&query_scopes==0); + } + // A range upload's original start lies outside its small host view. It + // must bind start=0 and retain the established -min_index vertex rebase. + window_view={&buffer,6*width,format,0x12000};commands={};query_scopes=0; + assert(Issue(true,100,6,-7,200,100*width,6*width,bindings,&commands)); + assert(requested_offset==100*width&&requested_length==6*width); + assert(commands.draws==1&&commands.start==0&&commands.count==6&&commands.base==-200); + window_view.length-=1;commands={};query_scopes=0; + assert(!Issue(true,100,6,-7,200,100*width,6*width,bindings,&commands)); + assert(commands.bindings==0&&commands.draws==0&&query_scopes==0); + // A one-index buffer may occupy just 2/4 bytes; no whole-buffer padding + // requirement belongs in index range validation. + full_view={&buffer,width,format,0x10000};commands={}; + assert(Issue(false,0,1,-1,0,0,0,bindings,&commands)); + full_view.buffer=nullptr;commands={}; + assert(!Issue(false,0,1,0,0,0,0,bindings,&commands)); + assert(!commands.bindings&&!commands.draws); + } + std::mt19937 random(0x83FC6A58); + for(u32 n=0;n<20000;++n){ + const u32 width=n%2?2:4; + const u32 start=random(),count=random(),length=random(); + const bool expected=count&&((uint64_t(start)+count)*width<=length); + const auto window=DrawIndexWindow(start,count,width,length); + assert(bool(window)==expected); + if(window){assert(window.offset==uint64_t(start)*width);assert(window.length==uint64_t(count)*width);} + } + assert(!DrawIndexWindow(0,1,3,32)); + assert(!DrawIndexWindow(0,0,2,32)); + assert(!DrawIndexWindow(UINT32_MAX,UINT32_MAX,4,UINT32_MAX)); + const auto upper=DrawIndexWindow(1073741822,1,4,UINT32_MAX); + assert(upper&&upper.offset==4294967288u&&upper.length==4); +} +''' +cpp = a.output / 'draw-index-bounds.cpp' +cpp.write_text(code) +exe = a.output / 'draw-index-bounds.exe' +subprocess.run([a.compiler, '-std=c++20', '-UNDEBUG', + '-I' + str(root / 'rexlego/src'), str(cpp), '-o', str(exe)], + check=True, timeout=45) +subprocess.run([str(exe.resolve())], check=True, timeout=10) +print('PASS: production index metadata/submission guards; 16/32-bit full/window ' + 'views, signed base vertices and overflow; rejected draws record no GPU commands') diff --git a/tools/gpu-aot/test_nonindexed_buffer_windows.py b/tools/gpu-aot/test_nonindexed_buffer_windows.py new file mode 100644 index 0000000..59770bb --- /dev/null +++ b/tools/gpu-aot/test_nonindexed_buffer_windows.py @@ -0,0 +1,177 @@ +"""Exercise production stream-window selection, binding and draw rebasing.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('output', type=Path) +p.add_argument('--compiler', default='clang++') +p.add_argument('--shader-hlsl', type=Path, + help='Optional real captured vertex HLSL directory: verify no vertex-ID semantics') +p.add_argument('--verify-regression', action='store_true') +a = p.parse_args() +root = Path(__file__).resolve().parents[2] +source = (root / 'rexlego/src/gpu_native/draw.cpp').read_text() +start = source.index(' std::array vertex_windows{};') +end = source.index(' if (mesh_trace) {', start) +streams = source[start:end] +issue_start = source.index(' commands->drawInstanced(count, 1,') +issue = source[issue_start:source.index('\n', issue_start)] +code = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include "gpu_native/buffer_window.h" +#include "gpu_native/index_draw.h" +#include "gpu_native/vertex_byte_order.h" +using namespace legodimensions::gpu_native; +using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using i32=int32_t; +using be_u16=uint16_t;using be_u32=uint32_t; +constexpr u32 kNativeVertexStreams=4; +namespace rex::graphics::xenos {enum class PrimitiveType:u32{kQuadList=13};} +struct Buffer {std::vector bytes;u8* at(u32 n){assert(n<=bytes.size());return bytes.data()+n;}}; +namespace plume { +struct RenderVertexBufferView {u8* address;u32 length;RenderVertexBufferView():address(nullptr),length(0){} + RenderVertexBufferView(u8* a,u32 n):address(a),length(n){} }; +struct RenderInputSlot {RenderInputSlot()=default;RenderInputSlot(u32,u32){} }; +u32 RenderFormatSize(u32 n){return n;} +} +enum class BufferKind{kVertex,kIndex}; +struct BufferResourceView {Buffer* buffer=nullptr;u32 length=0,guest_format=0,mirror_address=0;}; +struct Binding {u32 buffer=0,stride=0,offset=0;}; +struct Bindings {std::array vertex_streams{};}; +struct Element {u32 slotIndex=0,alignedByteOffset=0,format=0;}; +struct Declaration {const Element* elements=nullptr;u32 element_count=0;u64 reversed_byte_elements=0;}; +std::array sources; +std::vector> uploads; +struct Request {u32 guest,offset,length,stride,phase;std::vector reversed;}; +std::vector requests; +u32 full_resolves=0,translations=0; +BufferResourceView InspectBufferResource(u32 guest,BufferKind){assert(guest&&guest();b->bytes.assign(bytes.begin()+offset,bytes.begin()+offset+length); + auto* result=b.get();uploads.push_back(std::move(b));return {result,length,0,guest*0x10000+offset}; +} +BufferResourceView ResolveBufferResourceView(u32 guest,BufferKind,const VertexByteOrder&){ + ++full_resolves;if(!guest)return {};assert(guestT TranslateVirtual(u32){++translations;assert(false&&"nonindexed draws must not read indices");return nullptr;}} memory; +#define REX_KERNEL_MEMORY() (&memory) +struct Commands { + std::array views; + u32 start=~0u,count=0,draws=0; + void setVertexBuffers(u32 first,const plume::RenderVertexBufferView* v,u32 count,const plume::RenderInputSlot*){ + assert(first==0&&count==4);std::copy(v,v+count,views.begin());} + void drawInstanced(u32 n,u32 instances,u32 s,u32 first){assert(instances==1&&first==0);++draws;count=n;start=s;} +}; +Buffer zero; +bool Draw(bool window_uploads,bool indexed,u32 primitive_type,u32 start,u32 count,i32 base_vertex, + const Bindings& bindings,const Declaration& declaration,Commands* commands){ + std::array views; + std::array slots; + std::array vertex_buffers; + auto* null_buffer=&zero;IndexDrawWindow index_window;BufferResourceView index_metadata; +''' + streams + '\n' + issue + r''' + return use_windows; +} +void reset(){requests.clear();uploads.clear();full_resolves=0;} +int main(){ + zero.bytes.resize(256); + for(u32 guest=1;guest elements{{{0,0,12},{0,16,4},{1,4,4}}}; + Declaration declaration{elements.data(),3,2};Commands commands; + assert(Draw(true,false,6,123,7,INT32_MIN,bindings,declaration,&commands)); + assert(commands.start==0&&commands.count==7&&requests.size()==2&&full_resolves==0&&translations==0); + assert(requests[0].offset==12+123*28&&requests[0].length==7*28&&requests[0].phase==0); + assert(requests[0].reversed==std::vector{16}); + assert(requests[1].offset==32+123*16&&requests[1].length==7*16); + assert(commands.views[2].address==zero.at(0)&&commands.views[3].address==zero.at(0)); + for(u32 slot=0;slot<2;++slot)for(u32 vertex=0;vertex<7;++vertex) + for(const auto& e:elements)if(e.slotIndex==slot){const auto b=bindings.vertex_streams[slot]; + assert(std::memcmp(commands.views[slot].address+vertex*b.stride+e.alignedByteOffset, + sources[b.buffer].at(b.offset+(123+vertex)*b.stride+e.alignedByteOffset),e.format)==0);} + reset();commands={};assert(!Draw(false,false,6,123,7,0,bindings,declaration,&commands)); + assert(commands.start==123&&requests.empty()&&full_resolves==2); + reset();commands={};assert(Draw(true,false,13,123,8,0,bindings,declaration,&commands)); + assert(requests.size()==2&&full_resolves==0); // Actual quad issue/index expansion has its own fixture. + // A bad later used stream disables the range path for EVERY used stream. + for(u32 bad:{0u,3u,UINT32_MAX}){ + auto changed=bindings;changed.vertex_streams[1].stride=bad;reset();commands={}; + assert(!Draw(true,false,6,123,7,0,changed,declaration,&commands)); + assert(commands.start==123&&requests.empty()&&full_resolves==2); + } + auto changed=bindings;changed.vertex_streams[1].offset=32760;reset();commands={}; + assert(!Draw(true,false,6,0,1,0,changed,declaration,&commands)&&requests.empty()); + changed=bindings;changed.vertex_streams[0].offset=1;reset();commands={}; + assert(!Draw(true,false,6,0,1,0,changed,declaration,&commands)&&requests.empty()); + auto bad_elements=elements;bad_elements[2].alignedByteOffset=15;reset();commands={}; + assert(!Draw(true,false,6,0,1,0,bindings,{bad_elements.data(),3,2},&commands)&&requests.empty()); + for(auto range:{std::pair{0u,0u},std::pair{UINT32_MAX,2u},std::pair{1u,UINT32_MAX}}){ + reset();commands={};assert(!Draw(true,false,6,range.first,range.second,0,bindings,declaration,&commands)); + assert(requests.empty()&&commands.start==range.first); + } + changed=bindings;changed.vertex_streams[1].buffer=0;reset();commands={}; + assert(Draw(true,false,6,123,7,0,changed,declaration,&commands)); + assert(requests.size()==1&&commands.views[1].address==zero.at(0)&&commands.start==0); + std::mt19937 random(123136); + for(u32 i=0;i<10000;++i){const u32 start=random(),count=random();u32 last=0; + const bool expected=count&&u64(start)+count-1<=UINT32_MAX; + assert(NonIndexedVertexRange(start,count,last)==expected); + if(expected)assert(last==u64(start)+count-1);} +} +''' +a.output.mkdir(parents=True, exist_ok=True) + +def build(name, contents): + cpp = a.output / (name + '.cpp') + exe = a.output / (name + ('.exe' if os.name == 'nt' else '')) + cpp.write_text(contents) + subprocess.run([a.compiler, '-std=c++20', '-UNDEBUG', '-I' + str(root / 'rexlego/src'), + str(cpp), '-o', str(exe)], check=True, timeout=45) + return exe.resolve() + +subprocess.run([str(build('nonindexed-windows', code))], check=True, timeout=10) +negative = None +if a.verify_regression: + old = code.replace('if (window_uploads && count) {', 'if (window_uploads && indexed && count) {') + assert old != code + result = subprocess.run([str(build('indexed-only', old))], capture_output=True, text=True, timeout=10) + assert result.returncode != 0 + negative = 'original indexed-only eligibility fails real nonindexed production selection' +hlsl = [] +if a.shader_hlsl: + for path in sorted(a.shader_hlsl.glob('vs-*.hlsl')): + data = path.read_bytes() + assert b'SV_VertexID' not in data and b'SV_InstanceID' not in data, path + hlsl.append({'path': str(path.resolve()), 'sha256': hashlib.sha256(data).hexdigest()}) + assert hlsl, 'No captured vertex HLSL found' +(a.output / 'verification.json').write_text(json.dumps({ + 'result': 'PASS', 'production_stream_body_sha256': hashlib.sha256(streams.encode()).hexdigest(), + 'production_issue_statement': issue.strip(), 'negative_control': negative, + 'vertex_hlsl_without_id_semantics': hlsl, + 'checks': ['all used streams', 'unused/missing streams', 'bounds/alignment/overflow fallback', + 'packed reversal origin', 'nonindexed base ignored', 'first/last attribute equivalence', + 'default flag unchanged', 'QuadList stream eligibility', '10000 widened-range cases'], + 'boundaries': 'Actual draw selection/binding/issue snippets and range helpers; fake upload driver. ' + 'Run buffer_variants and draw_upload for actual conversion/invalidation/fence lifetime bodies.' +}, indent=2) + '\n') +print('PASS production nonindexed selection/binding/rebase, all used streams and bounds') diff --git a/tools/gpu-aot/test_quad_buffer_windows.py b/tools/gpu-aot/test_quad_buffer_windows.py new file mode 100644 index 0000000..2bbe27c --- /dev/null +++ b/tools/gpu-aot/test_quad_buffer_windows.py @@ -0,0 +1,276 @@ +"""Execute actual stream selection and QuadList expansion with real byte converters.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess + + +def block(text, signature): + start = text.index(signature) + opening = text.index('{', start) + depth, end = 1, opening + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('output', type=Path) +p.add_argument('--compiler', default='clang++') +p.add_argument('--verify-regression', action='store_true') +p.add_argument('--shader-hlsl', type=Path) +a = p.parse_args() +root = Path(__file__).resolve().parents[2] +draw = (root / 'rexlego/src/gpu_native/draw.cpp').read_text() +buffers = (root / 'rexlego/src/gpu_native/buffers.cpp').read_text() +begin = draw.index(' BufferResourceView index_metadata;', draw.index('bool DispatchDraw(')) +preflight = draw[begin:draw.index(' timing.Next();', begin)] +begin = draw.index(' std::array vertex_windows{};') +streams = draw[begin:draw.index(' if (mesh_trace) {', begin)] +quad = block(draw, ' if (primitive_type == static_cast(rex::graphics::xenos::PrimitiveType::kQuadList)) {') +swap = block(buffers, 'void ByteSwapElements(') +code = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "gpu_native/buffer_window.h" +#include "gpu_native/index_draw.h" +#include "gpu_native/vertex_upload.h" +using namespace legodimensions::gpu_native; +using u8=uint8_t;using u16=uint16_t;using u32=uint32_t;using u64=uint64_t;using i32=int32_t; +struct be_u16 {u8 bytes[2];operator u16()const{return u16(u16(bytes[0])<<8|bytes[1]);}}; +struct be_u32 {u8 bytes[4];operator u32()const{return u32(bytes[0])<<24|u32(bytes[1])<<16|u32(bytes[2])<<8|bytes[3];}}; +constexpr u32 kNativeVertexStreams=4; +namespace rex::graphics::xenos {enum class PrimitiveType:u32{kQuadList=13};} +enum class BufferKind{kVertex,kIndex}; +struct Buffer;struct BufferAddress {Buffer* buffer;u32 offset;}; +struct Buffer { + std::vector bytes;bool fail_map=false;u32 maps=0,unmaps=0; + BufferAddress at(u32 n){assert(n<=bytes.size());return {this,n};} + void* map(){++maps;return fail_map?nullptr:bytes.data();}void unmap(){++unmaps;} +}; +namespace plume { +using RenderBuffer=Buffer; +enum class RenderHeapType{UPLOAD};enum class RenderFormat{R32_UINT}; +struct RenderBufferDesc {u64 size;static auto IndexBuffer(u64 size,RenderHeapType){return RenderBufferDesc{size};}}; +struct RenderIndexBufferView {Buffer* buffer;u64 length; + RenderIndexBufferView(BufferAddress b,u64 n,RenderFormat):buffer(b.buffer),length(n){assert(b.offset==0);}}; +struct RenderVertexBufferView {Buffer* buffer=nullptr;u32 offset=0,length=0; + RenderVertexBufferView()=default;RenderVertexBufferView(BufferAddress b,u32 n):buffer(b.buffer),offset(b.offset),length(n){}}; +struct RenderInputSlot {RenderInputSlot()=default;RenderInputSlot(u32,u32){}}; +u32 RenderFormatSize(u32 n){return n;} +} +struct BufferResourceView {Buffer* buffer=nullptr;u32 length=0,guest_format=0,mirror_address=0;}; +struct Binding {u32 buffer=0,stride=0,offset=0;}; +struct Bindings {std::array vertex_streams{};u32 index_buffer=3;}; +struct Element {u32 slotIndex=0,alignedByteOffset=0,format=0;}; +struct Declaration {const Element* elements;u32 element_count;u64 reversed_byte_elements;}; +std::array sources; +u32 index_format=1;bool short_index_view=false,fail_index_map=false,fail_output_map=false,fail_allocation=false; +std::vector> uploads; +std::vector> retired; +std::vector operations; +u64 uploaded_vertex_bytes=0;u32 window_calls=0,full_calls=0; +''' + swap + r''' +auto Upload(u32 guest,BufferKind kind,u32 offset,u32 size,const VertexByteOrder& order) { + assert(guest&&guest();b->bytes.resize(size); + const auto canonical=CanonicalVertexByteOrder(order); + const bool fused=kind==BufferKind::kVertex&&WriteAlignedVertexUpload(b->bytes.data(),bytes.data()+offset,size,canonical); + if(!fused){ByteSwapElements(b->bytes.data(),bytes.data()+offset,size,kind==BufferKind::kIndex&&index_format==1?2:4); + if(kind==BufferKind::kVertex)assert(ApplyVertexByteOrder(b->bytes.data(),size,canonical));} + if(kind==BufferKind::kVertex)uploaded_vertex_bytes+=size; + b->fail_map=kind==BufferKind::kIndex&&fail_index_map; + auto* result=b.get();uploads.push_back(std::move(b)); + return BufferResourceView{result,short_index_view&&kind==BufferKind::kIndex?1u:size, + kind==BufferKind::kIndex?index_format:0u,(guest<<24)+offset}; +} +BufferResourceView InspectBufferResource(u32 guest,BufferKind kind){assert(guest&&guestT TranslateVirtual(u32 address){const u32 guest=address>>24,offset=address&0xFFFFFF; + assert(guest&&guest(sources[guest].bytes.data()+offset);}} memory; +#define REX_KERNEL_MEMORY() (&memory) +struct Device {auto createBuffer(plume::RenderBufferDesc desc){ + if(fail_allocation)return std::unique_ptr{};auto b=std::make_unique(); + assert(desc.size<1024*1024);b->bytes.resize(desc.size);b->fail_map=fail_output_map;return b;}} device; +struct HostDevice {static auto Device(){return &device;} + static void RetireResource(std::shared_ptr b){operations.emplace_back("retire");retired.push_back(std::move(b));}}; +struct Commands { + std::array views; + std::vector indices;u32 draws=0,count=0;i32 base=0; + void setVertexBuffers(u32 first,const plume::RenderVertexBufferView* v,u32 n,const plume::RenderInputSlot*){ + assert(first==0&&n==4);std::copy(v,v+n,views.begin());} + void setIndexBuffer(const plume::RenderIndexBufferView* view){ + assert(view&&view->buffer&&view->length%4==0);indices.resize(view->length/4); + std::memcpy(indices.data(),view->buffer->bytes.data(),view->length);} + void drawIndexedInstanced(u32 n,u32 instances,u32 start,i32 b,u32 first){ + assert(instances==1&&start==0&&first==0&&n==indices.size());++draws;count=n;base=b;operations.emplace_back("issue");} +}; +struct QueryDrawScope {explicit QueryDrawScope(Commands*){operations.emplace_back("query_begin");} + ~QueryDrawScope(){operations.emplace_back("query_end");}}; +Buffer zero; +bool Draw(bool window_uploads,bool indexed,u32 start,u32 count,i32 base_vertex, + const Bindings& bindings,const Declaration& declaration,Commands* commands,bool& window_result){ + const u32 primitive_type=13; +''' + preflight + r''' + std::array views; + std::array slots; + std::array vertex_buffers; + auto* null_buffer=&zero; +''' + streams + r''' + window_result=use_windows; +''' + quad + r''' + return true; +} +void Reset(){uploads.clear();retired.clear();operations.clear();uploaded_vertex_bytes=0;window_calls=full_calls=0; + short_index_view=fail_index_map=fail_output_map=fail_allocation=false;} +void SetIndices(u32 format,u32 start,const std::vector& indices){ + index_format=format;const u32 size=format==1?2:4;sources[3].bytes.assign((start+indices.size()+2)*size,0xCC); + for(u32 i=0;i>(8*(size-1-byte))); +} +struct Result {std::vector> attributes;std::vector expanded;u64 bytes;bool windows;}; +Result Run(bool flag,bool indexed,u32 start,u32 count,i32 base,const Bindings& bindings,const Declaration& decl){ + Reset();Commands commands;bool windows=false;assert(Draw(flag,indexed,start,count,base,bindings,decl,&commands,windows)); + assert(commands.draws==1&&commands.count==count/4*6); + assert(operations==std::vector({"query_begin","issue","query_end","retire"})); + assert(retired.size()==1);const auto retained=retired.front();assert(retained->unmaps==1); + Result result{{},commands.indices,uploaded_vertex_bytes,windows}; + for(u32 index:commands.indices){const int64_t vertex=int64_t(index)+commands.base;assert(vertex>=0); + for(u32 e=0;ebytes.begin()+view.offset+offset, + view.buffer->bytes.begin()+view.offset+offset+element.format); + }} + // Expanded upload stays retained beyond query end; no actual GPU-fence claim. + assert(retained->bytes.size()==commands.indices.size()*4); + return result; +} +int main(){ + zero.bytes.resize(256); + for(u32 guest:{1u,2u,4u}){sources[guest].bytes.resize(65536); + for(u32 i=0;i<65536;++i)sources[guest].bytes[i]=u8((i*17+guest)%251);} + Bindings bindings;bindings.vertex_streams[0]={1,28,12};bindings.vertex_streams[1]={2,36,20}; + bindings.vertex_streams[3]={4,0,1}; // Bound stale unused stream. + std::array elements{{{0,0,12},{0,16,4},{1,0,12},{1,12,4},{2,0,4}}}; + Declaration decl{elements.data(),u32(elements.size()),(u64{1}<<1)|(u64{1}<<3)}; + for(u32 count:{4u,8u,20u,64u})for(u32 start:{0u,3u,97u}){ + const auto old=Run(false,false,start,count,INT32_MIN,bindings,decl); + const auto next=Run(true,false,start,count,INT32_MIN,bindings,decl); + assert(!old.windows&&next.windows&&old.attributes==next.attributes&&next.bytes indices{14,17,18,15,16,13,12,19};SetIndices(format,3,indices); + const auto old=Run(false,true,3,8,base,bindings,decl); + const auto next=Run(true,true,3,8,base,bindings,decl); + assert(next.windows&&!old.windows&&old.attributes==next.attributes&&old.expanded==next.expanded); + assert(next.bytes==8*(28+36)); + } + SetIndices(0,2,{0x80000010,0x80000011,0x80000012,0x80000013}); + auto high=Run(true,true,2,4,INT32_MIN,bindings,decl);assert(!high.windows); + // Actual captured paused UI QuadList offsets, strides and vertex counts. + for(auto shape:std::array,8>{{{224,20,4},{468,36,24},{1476,36,20}, + {2364,36,24},{3228,36,28},{4236,36,64},{6540,36,24},{7404,36,32}}}){ + auto ui=bindings;ui.vertex_streams[0]={1,shape[1],shape[0]};ui.vertex_streams[1]={}; + auto ui_elements=elements;ui_elements[0]={0,0,12};ui_elements[1]={0,16,4}; + Declaration ui_decl{ui_elements.data(),2,u64{1}<<1}; + const auto old=Run(false,false,0,shape[2],0,ui,ui_decl),next=Run(true,false,0,shape[2],0,ui,ui_decl); + assert(next.windows&&old.attributes==next.attributes&&next.bytes==u64(shape[1])*shape[2]); + } + // Whole-draw fallback retains original bytes for valid layouts that cannot + // use an aligned range. A synthetic zero stream never disables other streams. + for(u32 offset:{1u,3u}){auto changed=bindings;changed.vertex_streams[1].offset=offset; + const auto old=Run(false,false,7,8,0,changed,decl),next=Run(true,false,7,8,0,changed,decl); + assert(!next.windows&&old.attributes==next.attributes&&next.bytes==old.bytes);} + auto changed=bindings;changed.vertex_streams[1].buffer=0; + assert(Run(true,false,7,8,0,changed,decl).windows); + for(u32 invalid_stride:{0u,3u,UINT32_MAX}){changed=bindings;changed.vertex_streams[1].stride=invalid_stride; + Reset();Commands commands;bool windows=false; + // Only selection/binding is examined for malformed layouts; do not simulate + // a GPU read of invalid bytes. Packed conversion may reject these layouts. + auto constant_elements=elements;constant_elements[3].slotIndex=0; + // Keep packed conversions off the invalid-stride stream so the actual + // whole-buffer fallback is observable without a simulated GPU fetch. + Draw(true,false,0,4,0,changed,{constant_elements.data(),3,2},&commands,windows);assert(!windows); + } + for(u32 count:{0u,3u,0xFFFFFFFCu}){Reset();Commands commands;bool windows=false; + assert(!Draw(false,false,0,count,0,bindings,decl,&commands,windows));assert(retired.empty()&&operations.empty());} + Reset();Commands commands;bool windows=false; + assert(!Draw(false,false,UINT32_MAX,4,0,bindings,decl,&commands,windows)); + SetIndices(1,3,{14,17,18,15,16,13,12,19}); + for(unsigned failure=0;failure<4;++failure){Reset();commands={};windows=false; + short_index_view=failure==0;fail_allocation=failure==1;fail_output_map=failure==2;fail_index_map=failure==3; + assert(!Draw(true,true,3,8,0,bindings,decl,&commands,windows)); + assert(commands.draws==0&&retired.empty()&&operations.empty()); + for(const auto& upload:uploads)if(upload->maps&&!upload->fail_map)assert(upload->unmaps==upload->maps); + } + std::mt19937 random(15213); + for(unsigned i=0;i<10000;++i){const u32 count=random();const u64 expanded=u64(count/4)*6; + const auto expected=count&&count%4==0&&expanded<=UINT32_MAX?u32(expanded):0; + assert(ExpandedQuadIndexCount(count)==expected);} + std::cout<<"PASS actual QuadList stream/vertex-byte/index expansion equivalence, queries and retained upload order\n"; +} +''' +a.output.mkdir(parents=True, exist_ok=True) + + +def build(name, contents): + cpp = a.output / (name + '.cpp') + exe = a.output / (name + '.exe') + cpp.write_text(contents) + subprocess.run([a.compiler, '-std=c++20', '-UNDEBUG', '-I' + str(root / 'rexlego/src'), + str(cpp), '-o', str(exe)], check=True, timeout=45) + return exe.resolve() + + +subprocess.run([str(build('quad-windows', code))], check=True, timeout=10) +negative = None +if a.verify_regression: + old = code.replace('if (window_uploads && count) {', + 'if (window_uploads && count && primitive_type != 13) {', 1) + assert old != code + result = subprocess.run([str(build('old-quad-exclusion', old))], + capture_output=True, text=True, timeout=10) + assert result.returncode != 0 and 'next.windows' in result.stderr, result.stderr + negative = 'Original QuadList exclusion fails actual full-vs-range attribute-byte equality case window expectation' +hlsl = [] +if a.shader_hlsl: + for path in sorted(a.shader_hlsl.glob('vs-*.hlsl')): + data = path.read_bytes() + assert b'SV_VertexID' not in data and b'SV_InstanceID' not in data, path + hlsl.append({'path': str(path.resolve()), 'sha256': hashlib.sha256(data).hexdigest()}) + assert hlsl +(a.output / 'verification.json').write_text(json.dumps({ + 'result': 'PASS', 'production_sha256': { + 'preflight': hashlib.sha256(preflight.encode()).hexdigest(), + 'stream_selection_binding': hashlib.sha256(streams.encode()).hexdigest(), + 'quad_expansion_submission': hashlib.sha256(quad.encode()).hexdigest(), + 'ByteSwapElements': hashlib.sha256(swap.encode()).hexdigest()}, + 'negative_control': negative, 'vertex_hlsl_without_id_semantics': hlsl, + 'checks': ['nonindexed start and ignored base', 'indexed16/32 nonzero start and signed base', + 'all expanded-index attribute bytes', 'packed field phase', 'multi/unused/synthetic streams', + 'eight actual paused UI shapes', 'alignment fallback', 'INT32MAX index fallback', + 'count/start overflow', '10000 expanded-count cases', 'source view/allocation/map failures', + 'query scope and retirement order'], + 'boundaries': 'Actual draw preflight/window/binding/QuadList bodies, ByteSwapElements and real vertex upload helpers; ' + 'mock buffer adoption/driver and retained uploads. No GPU fence or runtime speed claim.'}, indent=2) + '\n') From 44b950642f565469d9f0113a2ebeff860763457d Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:57:02 +0200 Subject: [PATCH 05/40] Define an evidence-driven native renderer regression strategy Document the existing whole-session PM4 reference route, isolated legitimate checkpoint testing, shader coverage gaps, and distinct acceptance gates for helper tests, rendered output, performance and campaign integration. --- tools/gpu-aot/ENGINEERING-PLAN.md | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tools/gpu-aot/ENGINEERING-PLAN.md diff --git a/tools/gpu-aot/ENGINEERING-PLAN.md b/tools/gpu-aot/ENGINEERING-PLAN.md new file mode 100644 index 0000000..920da08 --- /dev/null +++ b/tools/gpu-aot/ENGINEERING-PLAN.md @@ -0,0 +1,71 @@ +# Native renderer validation strategy + +The experimental native renderer translates the game's Xbox D3D calls into host +D3D12 operations. Recompiling the CPU executable does not establish correct +texture layouts, shader coverage, depth ownership, render state, or GPU resource +lifetime. Those need separate evidence. + +## Current decisions + +Keep the tested native build as a baseline. Before expanding the renderer or +changing implementation language, use the existing whole-session PM4 reference +route to compare equivalent scenes through the SDK's Xbox GPU implementation. +`gpu_native_pm4` / `LEGO_NATIVE_PM4_REFERENCE` restore the `xenos` plugin and route +native hook wrappers to the original Xbox D3D functions. This requires a matching +`rexgpu-xenos.dll` and its dependencies; an older published plugin is not a valid +substitute for the current runtime. + +The reference is useful independent evidence, not a guarantee of exact console +behavior. A per-draw hybrid is a different project: native and PM4 paths have +different resource ownership and synchronization. Do not switch between them +inside a frame without an explicit ownership design. + +## Regression loop + +1. Clone a legitimate saved checkpoint, profile, installation state, and normal + Toypad tags into a private run. Keep separate writable files and ports. +2. Record executable and dependency hashes, selected renderer, configuration, + shader packs, checkpoint hashes, and the input recipe. +3. Reach the scene with a short known recipe. Confirm camera, characters, and + scene identity before comparing results. A save directory is not a running + process snapshot; current SDK running-thread serialization is incomplete. +4. Capture rendered output and diagnostics for native and reference runs. Use + focused reproductions for failures, and fixed scenes without concurrent + compilation for performance measurements. +5. Turn each confirmed failure into a regression check. Repeat broad gameplay + after the targeted reproduction passes. + +Portable helper and mocked-driver checks remain useful, but they cannot establish +rendered pixel correctness. Add image checks and real GPU execution to that layer. +The full campaign is an integration test after this loop is reliable. + +## Shader coverage + +The static extractor filters named archive entries. Its output count does not +prove coverage of unnamed entries, shaders embedded in other asset types, sparse +installer archives, or programs constructed by the game at runtime. + +An initial audit found no exact physical or instruction-stream matches between +the static-only index and 169 captured runtime containers. Removing container +reflection from identity therefore does not fix these misses. Establish whether +the missing programs exist in unscanned asset entries before designing a new +runtime shader-generation path. + +Use bounded scan reports with explicit unreadable, sparse, malformed, and +unexamined counts. Record exact container and microcode identities. Compile new +coverage offline, then use validated additive packs to avoid rebuild/restart +cycles. A pack-loaded event proves acceptance; successful adoption of a shader +requested immediately before publication is separate recovery evidence. + +## Acceptance gates + +- Correctness: independent reference evidence plus a reproducible failure check. +- Coverage: explicit shader and scene inventories, including known gaps. +- Performance: matched scene, camera, build, flags, and an uncontended measurement + interval; report frame-time distributions and resource work. +- Integration: genuine saves reload, input and Toypad behavior remain correct, + and campaign testing completes without the failures under investigation. + +As of 2026-10-07, the native build has passed 51 portable GPU checks and reached +Oz gameplay. Full campaign completion, universal visual correctness, and smooth +performance have not been established. From 60743e51a47252ab48c8f61b48888fb63b8e95b7 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:36:30 +0200 Subject: [PATCH 06/40] Reject unsupported shader fetch mappings and verify compiler preparation Handle optional reflection in the pinned ABI624 compiler without reconstructing metadata. Reject absent vertex-fetch semantic mappings before release-mode iterator dereference. Apply scratch patches from a fixed repository directory and verify resulting source before publishing provenance. The guard fixture passes from CTest and external directories; skipped-patch negative controls reject incomplete preparation. The actual TU23 image audit accounts for 418 passing and 65 unsupported candidates. --- .../patches/xenosrecomp-empty-reflection.md | 32 ++++++++ .../xenosrecomp-empty-reflection.patch | 50 ++++++++++++ .../xenosrecomp-unmapped-vertex-fetch.patch | 13 +++ tools/gpu-aot/prepare_native_compiler.py | 31 +++++++- tools/gpu-aot/test_unmapped_vertex_fetch.py | 79 +++++++++++++++++++ 5 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 tools/gpu-aot/patches/xenosrecomp-empty-reflection.md create mode 100644 tools/gpu-aot/patches/xenosrecomp-empty-reflection.patch create mode 100644 tools/gpu-aot/patches/xenosrecomp-unmapped-vertex-fetch.patch create mode 100644 tools/gpu-aot/test_unmapped_vertex_fetch.py diff --git a/tools/gpu-aot/patches/xenosrecomp-empty-reflection.md b/tools/gpu-aot/patches/xenosrecomp-empty-reflection.md new file mode 100644 index 0000000..e03f26f --- /dev/null +++ b/tools/gpu-aot/patches/xenosrecomp-empty-reflection.md @@ -0,0 +1,32 @@ +`xenosrecomp-empty-reflection.patch` targets XenosRecomp revision +`339af41df2c23dbe3256c1c377716b81a0e0fe6b`. It also applies after the existing +LEGO compatibility and ABI624/viewport/gradient preparation steps. +`prepare_native_compiler.py` applies it only to the freshly prepared compiler +source and records its SHA-256 in `compiler-inputs.json`. + +Valid runtime shader containers may have `constantTableOffset == 0`. CTAB names +are optional reflection: the compiler already resolves unnamed float constants, +samplers and boolean bits through their register indices, and decodes vertex +inputs/interpolators from the shader metadata. The patch selects an explicit +static empty reflection table for those containers. A present CTAB retains its +original input pointer and all three reflection loops. Literal definitions and +microcode translation are unchanged, including rejection of unsupported +memexport shaders. + +On October 6, 2026, the rebuilt Linux native compiler produced DXIL for the actual +captured shaders `ps-20B156FC6D4EF62D`, `ps-31CEA43B9FE57F6F`, +`vs-34EB25B494B1B971` and `vs-71CEE6F3353CE415`, all without CTAB, plus the +present-CTAB control `ps-025D4B4261D8A4D1`. All five generated HLSL files and +their complete compressed bank were byte-identical to the previous Release +compiler's output. The previous compiler happened to read the zero CTAB offset +in the container header as a zero reflection count when assertions were +disabled; an assertion-enabled build rejected the valid inputs. The new path +does neither. This check used `XENOS_RECOMP_DXIL_ONLY=1`; it makes no SPIR-V +coverage claim. + +Local captured-input hashes, compiler/source/patch hashes and equivalence +results are recorded in +`.local-testing/game-launch/native-compiler/empty-reflection-test/verification.json`. +The supplemental patch was also checked against the exact pinned source and +the prepared source. No native application build or game launch was part of +these checks. diff --git a/tools/gpu-aot/patches/xenosrecomp-empty-reflection.patch b/tools/gpu-aot/patches/xenosrecomp-empty-reflection.patch new file mode 100644 index 0000000..106f4dd --- /dev/null +++ b/tools/gpu-aot/patches/xenosrecomp-empty-reflection.patch @@ -0,0 +1,50 @@ +--- a/XenosRecomp/shader_recompiler.cpp ++++ b/XenosRecomp/shader_recompiler.cpp +@@ -1272,3 +1272,2 @@ + assert((shaderContainer->flags & 0xFFFFFF00) == 0x102A1100); +- assert(shaderContainer->constantTableOffset != NULL); + +@@ -1279,4 +1278,14 @@ + +- const auto constantTableContainer = reinterpret_cast(shaderData + shaderContainer->constantTableOffset); +- constantTableData = reinterpret_cast(&constantTableContainer->constantTable); ++ // CTAB is optional reflection. Without it, retain the register-based ++ // constant/sampler paths and the shader's own vertex/interpolator metadata. ++ // Do not reinterpret the container header as an accidental empty CTAB. ++ static const ConstantTable emptyConstantTable{}; ++ const ConstantTable* constantTable = &emptyConstantTable; ++ if (shaderContainer->constantTableOffset != 0) ++ { ++ const auto constantTableContainer = reinterpret_cast( ++ shaderData + shaderContainer->constantTableOffset); ++ constantTable = &constantTableContainer->constantTable; ++ } ++ constantTableData = reinterpret_cast(constantTable); + +@@ -1289,6 +1298,6 @@ + +- for (uint32_t i = 0; i < constantTableContainer->constantTable.constants; i++) ++ for (uint32_t i = 0; i < constantTable->constants; i++) + { + const auto constantInfo = reinterpret_cast( +- constantTableData + constantTableContainer->constantTable.constantInfo + i * sizeof(ConstantInfo)); ++ constantTableData + constantTable->constantInfo + i * sizeof(ConstantInfo)); + +@@ -1402,6 +1411,6 @@ + +- for (uint32_t i = 0; i < constantTableContainer->constantTable.constants; i++) ++ for (uint32_t i = 0; i < constantTable->constants; i++) + { + const auto constantInfo = reinterpret_cast( +- constantTableData + constantTableContainer->constantTable.constantInfo + i * sizeof(ConstantInfo)); ++ constantTableData + constantTable->constantInfo + i * sizeof(ConstantInfo)); + +@@ -1449,6 +1458,6 @@ + +- for (uint32_t i = 0; i < constantTableContainer->constantTable.constants; i++) ++ for (uint32_t i = 0; i < constantTable->constants; i++) + { + const auto constantInfo = reinterpret_cast( +- constantTableData + constantTableContainer->constantTable.constantInfo + i * sizeof(ConstantInfo)); ++ constantTableData + constantTable->constantInfo + i * sizeof(ConstantInfo)); + diff --git a/tools/gpu-aot/patches/xenosrecomp-unmapped-vertex-fetch.patch b/tools/gpu-aot/patches/xenosrecomp-unmapped-vertex-fetch.patch new file mode 100644 index 0000000..283edbd --- /dev/null +++ b/tools/gpu-aot/patches/xenosrecomp-unmapped-vertex-fetch.patch @@ -0,0 +1,13 @@ +--- a/XenosRecomp/shader_recompiler.cpp ++++ b/XenosRecomp/shader_recompiler.cpp +@@ -192,7 +192,9 @@ + out += " = "; + + auto findResult = vertexElements.find(address); +- assert(findResult != vertexElements.end()); ++ if (findResult == vertexElements.end()) ++ throw std::runtime_error(fmt::format( ++ "unsupported vertex fetch address {} (no semantic input mapping)", address)); + + #ifdef REBLUE_RECOMP + // Wrap each 16-bit-packed semantic in swapFloats() (per-usage mask); TEXCOORD also runs sintTexcoord() for raw-int bindings. diff --git a/tools/gpu-aot/prepare_native_compiler.py b/tools/gpu-aot/prepare_native_compiler.py index c23938d..5059767 100644 --- a/tools/gpu-aot/prepare_native_compiler.py +++ b/tools/gpu-aot/prepare_native_compiler.py @@ -4,6 +4,7 @@ build anything, install archives, or launch the game. """ import argparse +import hashlib import json from pathlib import Path import subprocess @@ -26,14 +27,38 @@ p.error('Use a fresh scratch directory to avoid mixing compiler generations') scripts = root / 'tools/gpu-aot' subprocess.run([sys.executable, str(scripts/'prepare_alpha_candidate.py'), - str(scratch/'alpha'), '--checkout', str(checkout)], check=True) + str(scratch/'alpha'), '--checkout', str(checkout)], check=True, cwd=root) subprocess.run([sys.executable, str(scripts/'prepare_viewport_candidate.py'), - str(scratch/'alpha/source'), str(scratch/'viewport')], check=True) + str(scratch/'alpha/source'), str(scratch/'viewport')], check=True, cwd=root) subprocess.run([sys.executable, str(scripts/'prepare_gradients_candidate.py'), - str(scratch/'viewport/source'), str(scratch/'gradients')], check=True) + str(scratch/'viewport/source'), str(scratch/'gradients')], check=True, cwd=root) +reflection_patch = scripts/'patches/xenosrecomp-empty-reflection.patch' +# This tracked patch applies to the pinned compiler's unchanged reflection +# setup after the ABI/viewport/gradient preparation steps. The absolute scratch +# directory is explicitly selected; no checkout or installed bank is modified. +subprocess.run(['git', 'apply', '--unsafe-paths', '-p2', + '--directory=' + str(scratch/'gradients/source'), + str(reflection_patch)], check=True, cwd=root, timeout=10) +compiler_source = scratch/'gradients/source/shader_recompiler.cpp' +reflection_source = compiler_source.read_text() +if (reflection_source.count('static const ConstantTable emptyConstantTable{};') != 1 or + 'assert(shaderContainer->constantTableOffset != NULL);' in reflection_source): + raise RuntimeError('Optional-reflection patch did not update the prepared compiler') +fetch_patch = scripts/'patches/xenosrecomp-unmapped-vertex-fetch.patch' +subprocess.run(['git', 'apply', '--unsafe-paths', '-p2', + '--directory=' + str(scratch/'gradients/source'), + str(fetch_patch)], check=True, cwd=root, timeout=10) +fetch_source = compiler_source.read_text() +if (fetch_source.count('no semantic input mapping') != 1 or + 'assert(findResult != vertexElements.end());' in fetch_source): + raise RuntimeError('Vertex-fetch patch did not update the prepared compiler') (scratch/'compiler-inputs.json').write_text(json.dumps({ 'xenosrecomp_revision': revision, 'shared_constants_bytes': 624, 'source': str(scratch/'gradients/source'), 'built': False, + 'optional_reflection_patch': str(reflection_patch), + 'optional_reflection_patch_sha256': hashlib.sha256(reflection_patch.read_bytes()).hexdigest(), + 'unmapped_vertex_fetch_patch': str(fetch_patch), + 'unmapped_vertex_fetch_patch_sha256': hashlib.sha256(fetch_patch.read_bytes()).hexdigest(), 'archives_installed': False, 'game_launched': False, }, indent=2), encoding='utf-8') print(scratch/'gradients/source') diff --git a/tools/gpu-aot/test_unmapped_vertex_fetch.py b/tools/gpu-aot/test_unmapped_vertex_fetch.py new file mode 100644 index 0000000..871e997 --- /dev/null +++ b/tools/gpu-aot/test_unmapped_vertex_fetch.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Execute the pinned compiler's actual fetch lookup guard without game assets.""" +import argparse +import json +from pathlib import Path +import re +import subprocess + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('output', type=Path) +parser.add_argument('--compiler', default='clang++') +args = parser.parse_args() +root = Path(__file__).resolve().parents[2] +checkout = root / 'research/reblue/thirdparty/XenosRecomp/XenosRecomp' +source = checkout / 'shader_recompiler.cpp' +output = args.output.resolve() +output.mkdir(parents=True, exist_ok=True) +patched = output / 'shader_recompiler.cpp' +patched.write_bytes(source.read_bytes()) +patch = root / 'tools/gpu-aot/patches/xenosrecomp-unmapped-vertex-fetch.patch' +subprocess.run(['git', 'apply', '--unsafe-paths', '-p2', + '--directory=' + str(output), str(patch)], check=True, timeout=10, + cwd=root) +patched_text = patched.read_text() +if (patched_text.count('no semantic input mapping') != 1 or + 'assert(findResult != vertexElements.end());' in patched_text): + raise RuntimeError('Vertex-fetch patch did not update the fixture source') +body = re.search(r' auto findResult = vertexElements.find\(address\);\n.*?(?=\n#ifdef REBLUE_RECOMP)', + patched_text, re.S) +assert body and 'no semantic input mapping' in body.group() +legacy = re.search(r' auto findResult = vertexElements.find\(address\);\n.*?(?=\n#ifdef REBLUE_RECOMP)', + source.read_text(), re.S) +assert legacy and 'assert(findResult != vertexElements.end());' in legacy.group() +preparation = (root / 'tools/gpu-aot/prepare_native_compiler.py').read_text() +assert patch.name in preparation and 'unmapped_vertex_fetch_patch_sha256' in preparation +fixture = r''' +#define FMT_HEADER_ONLY +#include +#include +#include +#include +#include +#include +uint32_t lookup(const std::map& vertexElements, uint32_t address) { +@BODY@ + // The fixture avoids dereferencing end in the old implementation. A missing + // error still fails, rather than relying on nondeterministic undefined behavior. + return findResult == vertexElements.end() ? 0 : findResult->second; +} +int main() { + for (auto address : {0u, 100u, 0xffffffffu}) { + for (const auto& inputs : {std::map{}, + std::map{{7, 42}}}) { + try { lookup(inputs, address); return 17; } + catch (const std::runtime_error& error) { + if (std::string(error.what()) != fmt::format( + "unsupported vertex fetch address {} (no semantic input mapping)", address)) return 18; + } + } + } + if (lookup({{7,42}},7) != 42 || lookup({{0xffffffffu,99}},0xffffffffu) != 99) return 19; +} +''' +results = {} +for name, block, expected in [('guarded', body.group(), 0), ('legacy', legacy.group(), 17)]: + cpp, exe = output / (name + '.cpp'), output / name + cpp.write_text(fixture.replace('@BODY@', block)) + subprocess.run([args.compiler, '-std=c++20', '-DNDEBUG', + '-I' + str(checkout.parent / 'thirdparty/fmt/include'), + str(cpp), '-o', str(exe)], check=True, timeout=45) + result = subprocess.run([str(exe)], timeout=10) + assert result.returncode == expected, (name, result.returncode, expected) + results[name] = result.returncode +(output / 'verification.json').write_text(json.dumps(dict( + status='passed', production_guard=True, release_asserts_disabled=True, + missing_mapping_cases=6, valid_mapping_cases=2, negative_control=results, + boundary='Fetch lookup guard only; shader instruction decoding and DXC are tested separately with actual image inputs.' +), indent=2) + '\n') +print('unmapped vertex fetch guard PASS; legacy negative control rejected') From b9fd8de16b14e92cfa7ac4a9ccbf48e550154083 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:49:33 +0200 Subject: [PATCH 07/40] Recover shader origins through CRC lookup and bounded executable scans --- tools/gpu-aot/CMakeLists.txt | 15 + tools/gpu-aot/test_embedded_shader_extract.py | 140 +++++++ tools/gpu-aot/test_extract_paths.py | 88 ++++ tools/gpu-aot/test_image_shader_extract.py | 162 ++++++++ tools/gpu-shader-extract/ArchiveOutputPath.cs | 37 ++ .../EmbeddedShaderExtractor.cs | 376 ++++++++++++++++++ .../ImageShaderExtractor.cs | 266 +++++++++++++ tools/gpu-shader-extract/Program.cs | 50 ++- tools/gpu-shader-extract/README.md | 84 ++++ 9 files changed, 1208 insertions(+), 10 deletions(-) create mode 100644 tools/gpu-aot/test_embedded_shader_extract.py create mode 100644 tools/gpu-aot/test_extract_paths.py create mode 100644 tools/gpu-aot/test_image_shader_extract.py create mode 100644 tools/gpu-shader-extract/ArchiveOutputPath.cs create mode 100644 tools/gpu-shader-extract/EmbeddedShaderExtractor.cs create mode 100644 tools/gpu-shader-extract/ImageShaderExtractor.cs diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index 210f5b2..4e9ee76 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -20,3 +20,18 @@ foreach(check IN LISTS CHECKS) endif() add_test(NAME ${check} COMMAND test_${check}) endforeach() + +# These fixtures exercise the real extraction CLI with synthetic inputs. +if(BUILD_TESTING) + find_package(Python3 COMPONENTS Interpreter QUIET) + find_program(LEGO_DOTNET_EXECUTABLE NAMES dotnet) + if(Python3_Interpreter_FOUND AND LEGO_DOTNET_EXECUTABLE) + foreach(check IN ITEMS extract_paths image_shader_extract embedded_shader_extract) + add_test(NAME ${check} + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_${check}.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/${check} + --dotnet ${LEGO_DOTNET_EXECUTABLE}) + set_tests_properties(${check} PROPERTIES TIMEOUT 70) + endforeach() + endif() +endif() diff --git a/tools/gpu-aot/test_embedded_shader_extract.py b/tools/gpu-aot/test_embedded_shader_extract.py new file mode 100644 index 0000000..b4f48bc --- /dev/null +++ b/tools/gpu-aot/test_embedded_shader_extract.py @@ -0,0 +1,140 @@ +"""Actual DAT/HDR/DFLT + extractor CLI regression, with permuted name-tree/CRC indices.""" +import argparse +from html import escape +from pathlib import Path +import subprocess + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('output', type=Path) +parser.add_argument('--dotnet', default='dotnet') +parser.add_argument('--negative-control', choices=('name-tree', 'early-publish', 'decoded-count', 'overlong-output')) +args = parser.parse_args() +root = Path(__file__).resolve().parents[2] +output = args.output.resolve() +output.mkdir(parents=True, exist_ok=True) +sources = list((root / 'tools/gpu-shader-extract').glob('*.cs')) +if args.negative_control: + source = root / 'tools/gpu-shader-extract' / ('Program.cs' if args.negative_control == 'name-tree' else 'EmbeddedShaderExtractor.cs') + text = source.read_text() + if args.negative_control == 'name-tree': + old, new = 'int entryIndex = archive.FindEntry(internalPath);', 'int entryIndex = nameTreeIndex;' + elif args.negative_control == 'early-publish': + old = 'if (produced != size) throw new InvalidDataException("Decoded entry size differs from indexed size");' + new = '// negative control: omit actual final decoded-size guard' + elif args.negative_control == 'decoded-count': + old = 'if (Dflt.DecompressChunk(decoder) != 1 || decoder.OutputOffset != u || decoder.InputOffset > c)' + new = 'if (Dflt.DecompressChunk(decoder) != 1)' + else: + old, new = 'decoder.OutputLength = u+1;', 'decoder.OutputLength = u;' + if text.count(old) != 1: + raise RuntimeError('Actual extraction guard changed; update negative control') + altered = output / source.name + altered.write_text(text.replace(old, new)) + sources = [altered if p == source else p for p in sources] +sources += [root / 'DimensionsModManager/DatArchive.cs', root / 'DimensionsModManager/Dflt.cs'] +items = '\n'.join(f'' for p in sources) +(output / 'fixture.csproj').write_text(f''' +Exenet8.0enable +enablefalseEmbeddedFixtures +{items}''') +(output / 'Fixture.cs').write_text(r''' +using System.Buffers.Binary; +using System.IO.Compression; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using GpuShaderExtract; +using DimensionsModManager; + +internal static class EmbeddedFixtures { + static int checks; + static void Check(bool value,string message) {if(!value)throw new Exception(message);++checks;} + static void Be(byte[] b,int off,uint v)=>BinaryPrimitives.WriteUInt32BigEndian(b.AsSpan(off,4),v); + static void Le(byte[] b,int off,uint v)=>BinaryPrimitives.WriteUInt32LittleEndian(b.AsSpan(off,4),v); + static string Sha(byte[] b)=>Convert.ToHexString(SHA256.HashData(b)).ToLowerInvariant(); + static byte[] Shader(bool vertex=false) { + byte[] b=new byte[288];Be(b,0,vertex?0x102a1111u:0x102a1100u);Be(b,4,256);Be(b,8,32);Be(b,24,36); + Be(b,36,0);Be(b,40,12);for(int i=256;iv.Length)];int p=0;foreach(var v in values){v.CopyTo(b,p);p+=v.Length;}return b;} + static byte[] Packed(byte[] bytes,bool compress=false){byte[] payload=bytes;if(compress){using var o=new MemoryStream();using(var d=new DeflateStream(o,CompressionLevel.SmallestSize,true))d.Write(bytes);payload=o.ToArray();} + var h=new byte[12];"DFLT"u8.CopyTo(h);Le(h,4,(uint)payload.Length);Le(h,8,(uint)bytes.Length);return Join(h,payload);} + static byte[] Stored(byte[] actual,uint declared){var block=new byte[actual.Length+3];block[0]=5;BinaryPrimitives.WriteUInt16LittleEndian(block.AsSpan(1,2),(ushort)actual.Length);actual.CopyTo(block,3); + var h=new byte[12];"DFLT"u8.CopyTo(h);Le(h,4,(uint)block.Length);Le(h,8,declared);return Join(h,block);} + static byte[] Fixed(byte[] actual,uint declared){var bits=new List{1,1,0}; + void Symbol(int value){int length=value<=143?8:value<=255?9:7;int code=value<=143?0x30+value:value<=255?0x190+value-144:value-256;for(int i=length-1;i>=0;--i)bits.Add((code>>i)&1);} + foreach(byte b in actual)Symbol(b);Symbol(256);byte[] payload=new byte[(bits.Count+7)/8];for(int i=0;in.Name))+"\0");int nodes=32+blob.Length+4,table=nodes+names.Length*12+8,crc=table+entries.Length*16; + var h=new byte[crc+entries.Length*4];".CC40TAD"u8.CopyTo(h.AsSpan(4));Be(h,12,unchecked((uint)-11));Be(h,16,2);Be(h,20,(uint)entries.Length);Be(h,24,(uint)names.Length);Be(h,28,(uint)blob.Length);blob.CopyTo(h,32); + int nOff=0;for(int i=0;i(int)typeof(GpuShaderExtract.Program).GetMethod("Main",BindingFlags.NonPublic|BindingFlags.Static)!.Invoke(null,new object[]{args})!; + static JsonElement Read(string p)=>JsonDocument.Parse(File.ReadAllText(p)).RootElement.Clone(); + static JsonElement[] Rows(string p)=>File.ReadLines(p).Select(l=>JsonDocument.Parse(l).RootElement.Clone()).ToArray(); + static int Main(string[] args){try{Run(args);return 0;}catch(Exception e){Console.Error.WriteLine(e);return 1;}} + static void Run(string[] args){string root=Path.Combine(Path.GetFullPath(args[0]),Guid.NewGuid().ToString("N"));Directory.CreateDirectory(root); + var ps=Shader();var vs=Shader(true);string normal=Path.Combine(root,"normal");Directory.CreateDirectory(normal); + string archive=Archive(normal,"PATCH",new[]{new Entry(Array.Empty(),0),new Entry(Array.Empty(),0),new Entry(ps,(uint)ps.Length),new Entry(vs,(uint)vs.Length)},("named_nxg.360_shaders",3),("resource.res",2)); + var parsed=new DatArchive(archive);Check(parsed.EnumerateNames().First().Value==0,"fixture lacks sequential name-tree position"); + Check(parsed.FindEntry("named_nxg.360_shaders")==3&&parsed.FindEntry("resource.res")==2,"fixture CRC permutation not resolved"); + string namedOut=Path.Combine(root,"named-output");Check(Cli("extract",normal,namedOut)==0,"standard extract failed"); + Check(File.ReadAllBytes(Path.Combine(namedOut,"PATCH","named_nxg.360_shaders")).SequenceEqual(vs),"standard extractor read tree position instead of CRC file index"); + string all=Path.Combine(root,"all");Check(Cli("extract","--embedded",normal,all)==0,"all-entry extract failed"); + var index=Read(Path.Combine(all,"embedded-index.json"));Check(index.GetProperty("complete").GetBoolean()&&index.GetProperty("candidates").GetInt32()==2,"all-entry coverage missing"); + var rows=Rows(Path.Combine(all,"containers.jsonl"));Check(rows.Length==2,"wrong origin count"); + foreach(var row in rows){int i=row.GetProperty("entry_index").GetInt32();Check(row.GetProperty("names")[0].GetString()==(i==3?"named_nxg.360_shaders":"resource.res"),"embedded label used tree index");var c=row.GetProperty("candidate");byte[] expected=i==3?vs:ps; + Check(File.ReadAllBytes(Path.Combine(all,c.GetProperty("file").GetString()!)).SequenceEqual(expected),"container bytes altered");Check(c.GetProperty("container_sha256").GetString()==Sha(expected),"full SHA wrong");Check(c.GetProperty("physical").GetProperty("sha256").GetString()==Sha(expected[256..]),"physical SHA wrong");Check(c.GetProperty("instruction").GetProperty("sha256").GetString()==Sha(expected[256..268]),"instruction SHA wrong");} + Check(Rows(Path.Combine(all,"entries.jsonl")).Length==4,"unnamed/zero entries absent");Check(Cli("extract","--embedded",normal,all)==2,"fresh-output guard missing"); + // Unnamed entries, a chunk seam, nested header starts, actual DEFLATE, and duplicate source origins. + const int seam=2*1024*1024-2;var seamBytes=new byte[seam+ps.Length];ps.CopyTo(seamBytes,seam); + var outer=new byte[640];Be(outer,0,0x102a1100);Be(outer,4,608);Be(outer,8,32);Be(outer,24,36);Be(outer,40,12);ps.CopyTo(outer,96); + string mixed=Path.Combine(root,"mixed");Directory.CreateDirectory(mixed); + Archive(mixed,"GAME",new[]{new Entry(seamBytes,(uint)seamBytes.Length),new Entry(Packed(vs),(uint)vs.Length),new Entry(Packed(ps,true),(uint)ps.Length),new Entry(outer,(uint)outer.Length)}); + string mixedOut=Path.Combine(root,"mixed-output");Check(Cli("extract","--embedded",mixed,mixedOut)==0,"valid raw/DFLT entries rejected"); + rows=Rows(Path.Combine(mixedOut,"containers.jsonl"));Check(rows.Length==5,"seam/nested/DFLT origins missing");Check(rows.Any(r=>r.GetProperty("candidate").GetProperty("entry_offset").GetInt64()==seam),"boundary candidate missed");Check(rows.Count(r=>r.GetProperty("entry_index").GetInt32()==3)==2,"nested candidate skipped"); + Check(Read(Path.Combine(mixedOut,"embedded-index.json")).GetProperty("unique_containers").GetInt32()==3,"full-byte duplicate accounting wrong"); + // Failed entries cannot leak provisional candidates, including a fully-readable size mismatch. + string bad=Path.Combine(root,"bad");Directory.CreateDirectory(bad);var badTail=Join(Packed(Join(vs,new byte[64])),new byte[]{1,2,3}); + var seeded=Join(new byte[64],ps); + // Actual custom final stored block returns1 after writing one byte, even + // though its DFLT wrapper declares352. Reused output still contains PS@64. + Archive(bad,"SPARSE",new[]{new Entry(badTail,352),new Entry(ps,(uint)ps.Length+1),new Entry(vs,(uint)vs.Length,10000000),new Entry(ps,(uint)ps.Length),new Entry(Stored(seeded,352),352),new Entry(Stored(new byte[]{0x41},352),352),new Entry(Fixed(Join(seeded,new byte[]{0x99}),352),352)}); + string badOut=Path.Combine(root,"bad-output");Check(Cli("extract","--embedded",bad,badOut)==3,"incomplete scan reported success"); + rows=Rows(Path.Combine(badOut,"containers.jsonl"));Check(rows.Length==2&&rows.All(r=>r.GetProperty("entry_index").GetInt32() is 3 or 4),"malformed entry candidates published before end validation"); + var ledger=Rows(Path.Combine(badOut,"entries.jsonl"));Check(ledger.Length==7&&ledger.Count(r=>r.GetProperty("status").GetString()=="failed-no-candidates-published")==5,"sparse/decoder/size failures not explicit"); + Check(ledger[5].GetProperty("failure").GetString()!.Contains("wrong byte count"),"short decoder output accepted and stale bytes sampled"); + Check(ledger[6].GetProperty("failure").GetString()!.Contains("wrong byte count"),"overlong fixed-Huffman stream accepted after saturated writes"); + Check(ledger[0].GetProperty("withheld_candidates").GetInt32()==1,"malformed tail did not exercise provisional candidate withholding"); + Check(!Read(Path.Combine(badOut,"embedded-index.json")).GetProperty("complete").GetBoolean(),"partial ledger lost status"); + // An unreadable header retains an explicit unknown inventory, while another archive completes. + File.WriteAllBytes(Path.Combine(bad,"BROKEN.DAT"),new byte[8]);string failedHeader=Path.Combine(root,"failed-header");Check(Cli("extract","--embedded",bad,failedHeader)==3,"bad embedded header accepted"); + var summary=Read(Path.Combine(failedHeader,"embedded-index.json"));Check(summary.GetProperty("archives").GetArrayLength()==2&&summary.GetProperty("archives")[0].GetProperty("failure").ValueKind==JsonValueKind.String,"unvisited archive/header absent"); + string empty=Path.Combine(root,"empty");Directory.CreateDirectory(empty);Archive(empty,"EMPTY",new[]{new Entry(Array.Empty(),0)});Check(Cli("extract","--embedded",empty,Path.Combine(root,"empty-out"))==1,"valid zero candidates is not exit1"); + Check(Cli("extract","--embedded",empty,Path.Combine(root,"invalid-options"),"--max-seconds","0")==2,"invalid deadline accepted"); + Console.WriteLine($"actual archive/CLI extraction checks: {checks}"); + } +} +''') +subprocess.run([args.dotnet, 'build', str(output/'fixture.csproj'), '-o', str(output/'bin'), '-m:1', + '--disable-build-servers', '-p:UseSharedCompilation=false', '-v:q'], check=True, timeout=45) +run = subprocess.run([args.dotnet, str(output/'bin/fixture.dll'), str(output/'data')], + timeout=25, capture_output=True, text=True) +print(run.stdout, end='') +print(run.stderr, end='') +if args.negative_control: + oracle = ('standard extractor read tree position instead of CRC file index' + if args.negative_control == 'name-tree' else + 'malformed entry candidates published before end validation') + if run.returncode == 0 or oracle not in run.stderr: + raise RuntimeError('Negative control did not fail at the required extraction oracle') + print(f'{args.negative_control} negative control rejected as expected') +else: + run.check_returncode() diff --git a/tools/gpu-aot/test_extract_paths.py b/tools/gpu-aot/test_extract_paths.py new file mode 100644 index 0000000..eba8746 --- /dev/null +++ b/tools/gpu-aot/test_extract_paths.py @@ -0,0 +1,88 @@ +"""Compile and exercise the extractor's actual output-path helper without assets.""" + +import argparse +from html import escape +from pathlib import Path +import subprocess + + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("output", type=Path) +parser.add_argument("--dotnet", default="dotnet") +args = parser.parse_args() +root = Path(__file__).resolve().parents[2] +helper = root / "tools/gpu-shader-extract/ArchiveOutputPath.cs" +output = args.output.resolve() +output.mkdir(parents=True, exist_ok=True) +project = output / "extract-paths.csproj" +project.write_text(f''' + + Exe + net8.0 + enable + enable + false + + + + + + +''') +(output / "Program.cs").write_text(r''' +using GpuShaderExtract; + +string root = Path.GetFullPath(Path.Combine(args[0], "archive with spaces")); +StringComparison comparison = OperatingSystem.IsWindows() + ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; +int checks = 0; +void Valid(string input, string expected) { + string actual = ArchiveOutputPath.Resolve(root, input); + string destination = Path.GetFullPath(Path.Combine(root, + expected.Replace('/', Path.DirectorySeparatorChar))); + if (!string.Equals(actual, destination, comparison)) + throw new Exception($"Unexpected destination for {input}: {actual}"); + checks++; +} +void Reject(string input) { + try { ArchiveOutputPath.Resolve(root, input); } + catch (InvalidDataException) { checks++; return; } + throw new Exception($"Unsafe path accepted: {input}"); +} +Valid("shaders/default.pso", "shaders/default.pso"); +Valid(@"shaders\default.pso", "shaders/default.pso"); +Valid("shader folders/my shader.pso", "shader folders/my shader.pso"); +Valid("/shaders/default.pso", "shaders/default.pso"); +Valid(@"\shaders\default.pso", "shaders/default.pso"); +Valid(@"/\\shaders/default.pso", "shaders/default.pso"); +Valid("shaders/../default.pso", "default.pso"); +Valid("shaders/./default.pso", "shaders/default.pso"); +Reject("../escape.pso"); +Reject(@"..\escape.pso"); +Reject("shaders/../../escape.pso"); +Reject(@"shaders\..\..\escape.pso"); +Reject(@"shaders/..\../escape.pso"); +Reject("/../escape.pso"); +Reject(@"\..\escape.pso"); +Reject("../archive with spaces-other/escape.pso"); +Reject("C:/escape.pso"); +Reject(@"C:\escape.pso"); +Reject("c:escape.pso"); +Reject(@"\\?\C:\escape.pso"); +Reject("/C:/escape.pso"); +Reject("shaders/default.pso:stream"); +Reject("shaders/invalid\0.pso"); +Reject("."); +Reject("shaders/.."); +Reject(""); +Reject(@"/\\/"); +if (Directory.Exists(args[0])) + throw new Exception("Path validation must not create output directories"); +Console.WriteLine($"PASS: {checks} actual extractor path checks on {(OperatingSystem.IsWindows() ? "Windows" : "Unix")}"); +''') +subprocess.run([args.dotnet, "build", str(project), "-c", "Release", "-o", str(output / "bin"), + "--disable-build-servers", "--ignore-failed-sources", "-m:1", + "-p:UseSharedCompilation=false", "-p:NuGetAudit=false", "--nologo"], + check=True, timeout=45) +subprocess.run([args.dotnet, str(output / "bin/extract-paths.dll"), + str(output / "path-validation-does-not-write")], check=True, timeout=10) diff --git a/tools/gpu-aot/test_image_shader_extract.py b/tools/gpu-aot/test_image_shader_extract.py new file mode 100644 index 0000000..8d3a715 --- /dev/null +++ b/tools/gpu-aot/test_image_shader_extract.py @@ -0,0 +1,162 @@ +"""Asset-free tests linking the actual .NET image extractor and CLI sources.""" +import argparse +from html import escape +from pathlib import Path +import subprocess +try: + import resource +except ImportError: + pass +else: + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('output', type=Path) +parser.add_argument('--dotnet', default='dotnet') +parser.add_argument('--negative-control', nargs='?', const='metadata', choices=('metadata', 'nested'), + help='Verify the fixture rejects bypassed metadata bounds or skipped nested starts') +args = parser.parse_args() +root = Path(__file__).resolve().parents[2] +output = args.output.resolve() +output.mkdir(parents=True, exist_ok=True) +sources = list((root / 'tools/gpu-shader-extract').glob('*.cs')) +if args.negative_control: + helper = root / 'tools/gpu-shader-extract/ImageShaderExtractor.cs' + text = helper.read_text() + if args.negative_control == 'metadata': + guard = '''if (!ConstantsFit(virtualSection, constants) || + !DefinitionsFit(virtualSection, definitions, physicalBytes)) return Result.Invalid;''' + replacement = 'if (false) return Result.Invalid;' + else: + guard = ' ++position;' + replacement = ' position += layout.Total;' + if text.count(guard) != 1: + raise RuntimeError('Actual metadata guard changed; update the focused negative control') + altered = output / 'ImageShaderExtractor.cs' + altered.write_text(text.replace(guard, replacement)) + sources = [altered if path == helper else path for path in sources] +sources += [root / 'DimensionsModManager/DatArchive.cs', root / 'DimensionsModManager/Dflt.cs'] +items = '\n'.join(f'' for path in sources) +(output / 'fixture.csproj').write_text(f''' +Exenet8.0 +enableenable +falseImageFixtures +{items}''') +(output / 'Fixture.cs').write_text(r''' +using System.Buffers.Binary; +using System.Reflection; +using System.Security.Cryptography; +using System.Text.Json; +using GpuShaderExtract; + +internal static class ImageFixtures { + static int checks; + static void Check(bool value, string message) { if (!value) throw new Exception(message); ++checks; } + static void Word(byte[] bytes, int offset, uint value) => BinaryPrimitives.WriteUInt32BigEndian(bytes.AsSpan(offset,4), value); + static string Sha(byte[] bytes) => Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + static byte[] Container(bool vertex, bool marker, bool constants=false, bool definitions=false) { + const int v=256,p=32; var b=new byte[v+p+(marker?4:0)]; + Word(b,0,vertex?0x102a1111u:0x102a1100u);Word(b,4,v);Word(b,8,p);Word(b,24,36); + Word(b,36,0);Word(b,40,12); + if(marker)Word(b,v,p); + for(int i=0;i Check(ImageShaderExtractor.Parse(b,true,out var l)==ImageShaderExtractor.Result.Valid&&l.Total==b.Length,"valid container rejected"); + static void Invalid(byte[] b) => Check(ImageShaderExtractor.Parse(b,true,out _)==ImageShaderExtractor.Result.Invalid,"malformed container accepted"); + static int Cli(params string[] args) => (int)typeof(GpuShaderExtract.Program).GetMethod("Main",BindingFlags.NonPublic|BindingFlags.Static)!.Invoke(null,new object[]{args})!; + static int Main(string[] args) { try { Run(args); return 0; } + catch(Exception error) {Console.Error.WriteLine(error.Message);return 1;} } + static void Run(string[] args) { + string root=Path.Combine(Path.GetFullPath(args[0]),Guid.NewGuid().ToString("N"));Directory.CreateDirectory(root); + foreach(bool vertex in new[]{false,true})foreach(bool marker in new[]{false,true}) + foreach(bool ct in new[]{false,true})foreach(bool defs in new[]{false,true})Valid(Container(vertex,marker,ct,defs)); + var basic=Container(false,false); + for(int n=0;nr.GetProperty("image_offset").GetInt32()).SequenceEqual(new[]{0,start}),"nested/overlap offsets wrong"); + Check(rows[1].GetProperty("container_sha256").GetString()==Sha(inner),"nested/overlap bytes changed"); + } + Check(Cli("extract","--image",file,target,"--image-base","82000000")==2,"existing output overwritten"); + Check(Cli("extract","--image",file,Path.Combine(root,"hex-prefix"),"--image-base","0x82000000")==0,"CLI0xbase differs from loader provenance"); + Check(Cli("extract","--image",file,Path.Combine(root,"no-base"),"--image-base","bad")==2,"guessed invalid base"); + Check(Cli("extract","--image",file,Path.Combine(root,"overflow"),"--image-base","FFFF0000")==2,"guest range overflow accepted"); + foreach(string badProof in new[]{"{}","[]","{\"mapped_image\":{\"bytes\":1}}", + "{\"mapped_image\":{\"sha256\":\"wrong\",\"bytes\":3,\"base\":\"82000000\"}}"}) { + File.WriteAllText(proof,badProof);string reject=Path.Combine(root,"bad-proof-"+checks); + Check(Cli("extract","--image",file,reject,"--image-base","82000000","--image-provenance",proof)==2,"bad provenance accepted"); + Check(!Directory.Exists(reject),"partial output published on provenance failure"); + } + Console.WriteLine($"PASS: {checks} actual parser/stream/CLI checks; no game assets, compilation or rendering claim."); + } +} +''') +subprocess.run([args.dotnet, 'build', str(output/'fixture.csproj'), '-c', 'Release', '-o', str(output/'bin'), + '--disable-build-servers', '--ignore-failed-sources', '-m:1', '-p:UseSharedCompilation=false', + '-p:NuGetAudit=false', '--nologo'], check=True, timeout=45) +result = subprocess.run([args.dotnet, str(output/'bin/fixture.dll'), str(output/'data')], + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15) +print(result.stdout, end='') +print(result.stderr, end='') +if args.negative_control: + oracle = 'malformed' if args.negative_control == 'metadata' else 'nested/overlap starts skipped' + if result.returncode == 0 or oracle not in result.stderr: + raise RuntimeError('Altered production source did not fail at its independent fixture oracle') + print(f'PASS: actual {args.negative_control} negative control failed as expected; no core dump.') +else: + result.check_returncode() diff --git a/tools/gpu-shader-extract/ArchiveOutputPath.cs b/tools/gpu-shader-extract/ArchiveOutputPath.cs new file mode 100644 index 0000000..a6ccb80 --- /dev/null +++ b/tools/gpu-shader-extract/ArchiveOutputPath.cs @@ -0,0 +1,37 @@ +namespace GpuShaderExtract; + +internal static class ArchiveOutputPath +{ + internal static string Resolve(string archiveOutputRoot, string internalPath) + { + // Archive paths use a virtual root and either separator spelling. + // A Windows drive or alternate data stream is never a virtual path, + // including when the extraction tool itself is running on Unix. + string relative = internalPath.Replace('\\', '/').TrimStart('/'); + if (relative.Length == 0 || relative.Contains(':')) + { + throw new InvalidDataException("unsafe archive path: empty path or drive/stream syntax"); + } + + try + { + string root = Path.GetFullPath(archiveOutputRoot); + string rootPrefix = Path.EndsInDirectorySeparator(root) + ? root : root + Path.DirectorySeparatorChar; + string destination = Path.GetFullPath(Path.Combine( + root, relative.Replace('/', Path.DirectorySeparatorChar))); + StringComparison comparison = OperatingSystem.IsWindows() + ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + if (!destination.StartsWith(rootPrefix, comparison)) + { + throw new InvalidDataException("unsafe archive path: destination escapes its archive directory"); + } + return destination; + } + catch (Exception exception) when (exception is ArgumentException or + NotSupportedException or PathTooLongException) + { + throw new InvalidDataException("unsafe archive path: invalid filesystem path", exception); + } + } +} diff --git a/tools/gpu-shader-extract/EmbeddedShaderExtractor.cs b/tools/gpu-shader-extract/EmbeddedShaderExtractor.cs new file mode 100644 index 0000000..983d551 --- /dev/null +++ b/tools/gpu-shader-extract/EmbeddedShaderExtractor.cs @@ -0,0 +1,376 @@ +using System.Buffers.Binary; +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text.Json; +using DimensionsModManager; + +namespace GpuShaderExtract; + +// Unlike filename-based extraction, this visits every archive entry. Candidates +// are exact source containers, not CTAB-stripped runtime reconstructions. +internal static class EmbeddedShaderExtractor +{ + private const int Chunk = 2 * 1024 * 1024; + private const int MaximumHeader = 64 * 1024 * 1024; + private const int MaximumEntries = 1000000; + private const int MaximumNames = 250000; + private const long MaximumDecoded = 64L * 1024 * 1024 * 1024; + private const long MaximumEntry = 1024L * 1024 * 1024; + private const long MaximumCandidateWork = 1024L * 1024 * 1024; + private const long MaximumOutput = 512L * 1024 * 1024; + private const int MaximumRecords = 100000; + private static readonly byte[] Magic = { 0x10, 0x2a, 0x11 }; + private static readonly JsonSerializerOptions Json = new() { WriteIndented = true }; + private static string Sha(ReadOnlySpan bytes) => Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + private static void Line(StreamWriter writer, object value) => writer.WriteLine(JsonSerializer.Serialize(value)); + private sealed class WorkLimit(string message) : IOException(message); + private sealed class Budget(int seconds) + { + private readonly Stopwatch clock = Stopwatch.StartNew(); + internal long Decoded, CandidateBytes, OutputBytes; + internal int Candidates; + internal void Check() + { + if (clock.Elapsed.TotalSeconds > seconds) throw new WorkLimit("Scan deadline reached"); + } + internal void Decode(int count) + { + Check(); + Decoded += count; + if (Decoded > MaximumDecoded) throw new WorkLimit("Decoded work exceeds64GiB"); + } + internal void Candidate(int count) + { + Check(); + CandidateBytes += count; + if (++Candidates > MaximumRecords || CandidateBytes > MaximumCandidateWork) + throw new WorkLimit("Candidate count/work bound reached"); + } + } + + internal static int Run(string[] args) + { + if (args.Length < 2) + { + Console.Error.WriteLine("Usage: gpu-shader-extract extract --embedded [--max-seconds <1..3600>]"); + return 2; + } + try + { + int seconds = 300; + if (args.Length != 2 && (args.Length != 4 || args[2] != "--max-seconds" || + !int.TryParse(args[3], out seconds) || seconds < 1 || seconds > 3600)) + throw new ArgumentException("Invalid --max-seconds; expected1..3600"); + return Extract(Path.GetFullPath(args[0]), Path.GetFullPath(args[1]), seconds); + } + catch (Exception error) when (error is IOException or InvalidDataException or UnauthorizedAccessException or ArgumentException or InvalidOperationException) + { + Console.Error.WriteLine($"Embedded extraction rejected: {error.Message}"); + return 2; + } + } + + private static bool IsArchive(string path) => Path.GetExtension(path).Equals(".DAT", StringComparison.OrdinalIgnoreCase) || + Path.GetExtension(path).Equals(".DAT2", StringComparison.OrdinalIgnoreCase); + + // DatArchive intentionally reads its metadata into memory. Check allocation, + // node bounds and expanded path work before invoking that shared API. + private static (string Sha, bool NamesSupported) Preflight(string path) + { + string header = DatArchive.HdrPathFor(path); + byte[] bytes; + if (File.Exists(header)) + { + long length = new FileInfo(header).Length; + if (length < 12 || length > MaximumHeader) throw new InvalidDataException("Header size outside12B..64MiB"); + bytes = File.ReadAllBytes(header); + } + else + { + using var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + Span prefix = stackalloc byte[8]; file.ReadExactly(prefix); + long offset = BinaryPrimitives.ReadUInt32LittleEndian(prefix); + if ((offset & 0x80000000) != 0) offset = ((offset ^ 0xffffffff) << 8) + 0x100; + uint length = BinaryPrimitives.ReadUInt32LittleEndian(prefix[4..]); + if (length < 8 || length > MaximumHeader - 4 || offset > file.Length || length > file.Length-offset) + throw new InvalidDataException("Sparse/truncated or oversized embedded header"); + bytes = new byte[(int)length+4]; file.Position = offset; file.ReadExactly(bytes.AsSpan(4)); + } + bool modern = bytes.Length >= 32 && bytes.AsSpan(4, 8).SequenceEqual(".CC40TAD"u8); + uint Read(int offset) => modern ? BinaryPrimitives.ReadUInt32BigEndian(bytes.AsSpan(offset,4)) : + BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(offset,4)); + void Fits(long offset, long size) + { + if (offset < 0 || size < 0 || offset > bytes.Length || size > bytes.Length-offset) + throw new InvalidDataException("Archive metadata outside bounded header"); + } + int type = unchecked((int)Read(modern ? 12 : 4)); + uint files = Read(modern ? 20 : 8); + if (files > MaximumEntries) throw new InvalidDataException("Archive exceeds1000000 entries"); + int nodeSize; + long nodes, blob, blobBytes; + uint names; + if (modern) + { + uint version = Read(16); + names = Read(24); blob = 32; blobBytes = Read(28); nodeSize = version >= 2 ? 12 : 10; + nodes = blob + blobBytes + 4; + long table = nodes + (long)names * nodeSize + 8; + Fits(table, (long)files * (type <= -11 ? 16 : type <= -10 ? 12 : 16) + (long)files*4); + } + else + { + if (type >= 0 || type < -9) throw new InvalidDataException("Unsupported classic archive type"); + long namesOffset = 12 + (long)files*16; Fits(namesOffset,4); + names = Read((int)namesOffset); nodeSize = type <= -5 ? 12 : 8; nodes = namesOffset+4; + long relative = nodes+(long)names*nodeSize; Fits(relative,4); + blob = relative+4; blobBytes = Read((int)relative); Fits(blob+blobBytes,(long)files*4); + } + if (names > MaximumNames) throw new InvalidDataException("Archive exceeds250000 name nodes"); + Fits(nodes, (long)names*nodeSize); Fits(blob,blobBytes); + bool supported = modern || type <= -5 || names == 0; + if (supported) + { + var lengths = new int[(int)names+1]; long expanded = 0; + for (int i = modern ? 0 : 1; i < names; ++i) + { + int n = (int)(nodes+(long)i*nodeSize); + uint nameOffset = Read(n+(modern ? 0 : 4)); + int parent = modern ? BinaryPrimitives.ReadUInt16BigEndian(bytes.AsSpan(n+4,2)) : + BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(n+8,2)); + if (modern && nameOffset == uint.MaxValue) continue; + // Preserve the shared reader's classic invalid-node skip rule. + if (!modern && (nameOffset >= blobBytes || parent > names)) continue; + if (nameOffset >= blobBytes || parent > names) throw new InvalidDataException("Invalid name node"); + int start = (int)(blob+nameOffset); + int available = (int)Math.Min(1025, blobBytes-nameOffset); + int length = bytes.AsSpan(start,available).IndexOf((byte)0); + if (length < 0) throw new InvalidDataException("Archive name exceeds1024B or is unterminated"); + int full = lengths[parent]+(lengths[parent] == 0 ? 0 : 1)+length; + if (full > 4096 || (expanded += full) > 16*1024*1024) + throw new InvalidDataException("Expanded archive name work exceeds bound"); + int fileId = modern ? BinaryPrimitives.ReadUInt16BigEndian(bytes.AsSpan(n+(nodeSize==12 ? 10 : 8),2)) : + BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(n+10,2)); + bool folder = modern ? fileId == 0 && i != names-1 : fileId == 0 || fileId >= files; + if (folder) lengths[i] = full; + } + } + return (Sha(bytes), supported); + } + + private sealed class EntryScan(string pending, Budget budget, byte[] buffer) + { + private int carry; + private long baseOffset; + internal readonly List Records = new(); + internal readonly Dictionary Files = new(StringComparer.Ordinal); + internal int Rejected; + internal string DecodedSha = ""; + private long pendingBytes; + private readonly IncrementalHash hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + internal void Feed(ReadOnlySpan input, bool eof) + { + hash.AppendData(input); input.CopyTo(buffer.AsSpan(carry)); + int length = carry+input.Length, position = 0; + while (position+ImageShaderExtractor.HeaderBytes <= length) + { + budget.Check(); + int magic = buffer.AsSpan(position,length-position).IndexOf(Magic); + if (magic < 0) { position = Math.Max(position,length-(ImageShaderExtractor.HeaderBytes-1)); break; } + position += magic; + var result = ImageShaderExtractor.Parse(buffer.AsSpan(position,length-position), eof, out var layout); + if (result == ImageShaderExtractor.Result.Incomplete) break; + if (result == ImageShaderExtractor.Result.Invalid) { ++Rejected; ++position; continue; } + budget.Candidate(layout.Total); + if (Records.Count >= 4096) throw new WorkLimit("Entry exceeds4096 candidate starts"); + var full = buffer.AsSpan(position,layout.Total); + string sha = Sha(full), filename = layout.Stage+"-"+sha+".bin"; + if (Files.TryAdd(filename,layout.Total)) + { + pendingBytes += layout.Total; + if (pendingBytes > MaximumOutput-budget.OutputBytes) + throw new WorkLimit("Provisional entry output exceeds512MiB remaining bound"); + using var file = new FileStream(Path.Combine(pending,filename),FileMode.CreateNew,FileAccess.Write); + file.Write(full); + } + Records.Add(new { entry_offset = baseOffset+position, stage = layout.Stage, flags = layout.Flags.ToString("X8"), + file = filename, bytes = layout.Total, archive_marker = layout.Marker, container_sha256 = sha, + physical = new { offset = layout.PhysicalBase, bytes = layout.PhysicalBytes, + sha256 = Sha(full.Slice(layout.PhysicalBase,layout.PhysicalBytes)) }, + instruction = new { offset = layout.InstructionOffset, bytes = layout.InstructionBytes, + sha256 = Sha(full.Slice(layout.InstructionOffset,layout.InstructionBytes)) } }); + ++position; // All starts, including nested/overlapping containers. + } + carry = length-position; + Buffer.BlockCopy(buffer,position,buffer,0,carry); baseOffset += position; + if (eof) { DecodedSha = Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); hash.Dispose(); } + } + internal void Dispose() => hash.Dispose(); + } + + private static long ReadEntry(FileStream source, (long offset,uint zsize,uint size) entry, EntryScan scan, Budget budget, + byte[] input, byte[] output, DfltContext decoder) + { + var (offset,zsize,size) = entry; + if (offset < 0 || offset > source.Length || zsize > source.Length-offset) + throw new InvalidDataException("Sparse/truncated entry range outside DAT"); + if (size > MaximumEntry || zsize > MaximumEntry) throw new InvalidDataException("Entry exceeds1GiB bounded scan"); + source.Position = offset; + long remaining = zsize, produced = 0; + byte[] header = new byte[12]; + bool packed = false; + if (remaining >= 12) { source.ReadExactly(header); source.Position = offset; packed = header.AsSpan(0,4).SequenceEqual("DFLT"u8); } + while (remaining > 0) + { + budget.Check(); + if (!packed) + { + int count = (int)Math.Min(Chunk,remaining); source.ReadExactly(input.AsSpan(0,count)); + budget.Decode(count); scan.Feed(input.AsSpan(0,count),false); remaining -= count; produced += count; + } + else + { + if (remaining < 12) throw new InvalidDataException("Truncated DFLT chunk header"); + source.ReadExactly(header); remaining -= 12; + int c = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4,4)); + int u = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(8,4)); + if (!header.AsSpan(0,4).SequenceEqual("DFLT"u8) || c < 0 || u < 0 || c > Chunk || u > Chunk || + c > remaining || u > size-produced) throw new InvalidDataException("Invalid or oversized DFLT chunk"); + source.ReadExactly(input.AsSpan(0,c)); remaining -= c; budget.Decode(u); + if (c == u) scan.Feed(input.AsSpan(0,u),false); + else + { + decoder.Reset(); + // The actual stored-block decoder uses Input.Length for a + // copy bound, not InputLength. An exact bounded copy keeps + // previous compressed chunks outside its readable input. + decoder.Input = input.AsSpan(0,c).ToArray(); decoder.InputLength = c; + // WriteByte saturates at OutputLength. One sentinel byte + // makes even a one-byte overlong stream fail the count. + decoder.Output = output; decoder.OutputLength = u+1; + if (Dflt.DecompressChunk(decoder) != 1 || decoder.OutputOffset != u || decoder.InputOffset > c) + throw new InvalidDataException("DFLT decoder rejected chunk or produced wrong byte count"); + scan.Feed(output.AsSpan(0,u),false); + } + produced += u; + } + } + if (produced != size) throw new InvalidDataException("Decoded entry size differs from indexed size"); + scan.Feed(ReadOnlySpan.Empty,true); + return produced; + } + + internal static int Extract(string inputRoot, string outputRoot, int seconds) + { + string[] archives = Directory.Exists(inputRoot) ? Directory.EnumerateFiles(inputRoot,"*",SearchOption.TopDirectoryOnly) + .Where(IsArchive).OrderBy(p=>p,StringComparer.Ordinal).Take(4097).ToArray() : + File.Exists(inputRoot) && IsArchive(inputRoot) ? new[]{inputRoot} : Array.Empty(); + if (archives.Length == 0 || archives.Length > 4096) throw new ArgumentException("Expected1..4096 DAT/DAT2 archives"); + if (File.Exists(outputRoot) || Directory.Exists(outputRoot)) throw new IOException("Use a fresh output directory"); + string parent = Path.GetDirectoryName(outputRoot)!; Directory.CreateDirectory(parent); + string temporary = Path.Combine(parent,"."+Path.GetFileName(outputRoot)+".embedded-"+Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(temporary); + var budget = new Budget(seconds); var summaries = new List(); + var unique = new HashSet(StringComparer.Ordinal); int published = 0; bool complete = true, halted = false; + try + { + using (var ledger = new StreamWriter(Path.Combine(temporary,"entries.jsonl"))) + using (var records = new StreamWriter(Path.Combine(temporary,"containers.jsonl"))) + { + var input = new byte[Chunk]; var unpacked = new byte[Chunk+1]; var decoder = new DfltContext(); + var scanBuffer = new byte[Chunk+2*ImageShaderExtractor.MaximumSectionBytes+4]; + foreach (string path in archives) + { + int files = -1, visited = 0, failed = 0, unnamed = 0; string? failure = null, headerSha = null; + bool namesSupported = false; long length = new FileInfo(path).Length; DateTime modified = File.GetLastWriteTimeUtc(path); + string headerPath = DatArchive.HdrPathFor(path); long? headerLength = File.Exists(headerPath) ? new FileInfo(headerPath).Length : null; + DateTime? headerModified = headerLength is null ? null : File.GetLastWriteTimeUtc(headerPath); + if (halted) failure = "Unvisited after global work/deadline limit"; + else try + { + budget.Check(); (headerSha,namesSupported) = Preflight(path); + var archive = new DatArchive(path); files = archive.FileCount; + var names = new Dictionary>(); + foreach (var name in archive.EnumerateNames()) + { + budget.Check(); int index = archive.FindEntry(name.Key); + if (index < 0 || index >= files) throw new InvalidDataException("Name cannot resolve to an indexed entry"); + if (!names.TryGetValue(index,out var list)) names[index] = list = new List(); + list.Add(name.Key); + } + if (!namesSupported) complete = false; + using var source = new FileStream(path,FileMode.Open,FileAccess.Read,FileShare.Read); + for (int i = 0; i < files; ++i) + { + budget.Check(); names.TryGetValue(i,out var foundNames); + string[] aliases = foundNames?.OrderBy(n=>n,StringComparer.Ordinal).ToArray() ?? Array.Empty(); + if (aliases.Length == 0) ++unnamed; + (long offset,uint zsize,uint size) entry = default; + string pending = Path.Combine(temporary,"pending"); Directory.CreateDirectory(pending); + var scan = new EntryScan(pending,budget,scanBuffer); + try + { + entry = archive.GetEntry(i); + long decoded = ReadEntry(source,entry,scan,budget,input,unpacked,decoder); + // All chunks and the indexed size are validated before ANY entry candidate is published. + long additional = scan.Files.Where(p=>!unique.Contains(p.Key)).Sum(p=>(long)p.Value); + if (additional > MaximumOutput-budget.OutputBytes) throw new WorkLimit("Unique output exceeds512MiB"); + foreach (var file in scan.Files) + if (unique.Add(file.Key)) File.Move(Path.Combine(pending,file.Key),Path.Combine(temporary,file.Key)); + budget.OutputBytes += additional; + foreach (object candidate in scan.Records) + Line(records,new { archive = path, entry_index = i, names = aliases, candidate }); + published += scan.Records.Count; + Line(ledger,new { archive = path, entry_index = i, names = aliases, offset = entry.offset, + stored_bytes = entry.zsize, indexed_bytes = entry.size, status = "decoded-scanned", decoded_bytes = decoded, + decoded_sha256 = scan.DecodedSha, structural_candidates = scan.Records.Count, rejected_magic_candidates = scan.Rejected }); + } + catch (Exception error) when (error is IOException or InvalidDataException or ArgumentException or IndexOutOfRangeException) + { + ++failed; complete = false; + Line(ledger,new { archive = path, entry_index = i, names = aliases, offset = entry.offset, + stored_bytes = entry.zsize, indexed_bytes = entry.size, status = "failed-no-candidates-published", + failure = error.Message, withheld_candidates = scan.Records.Count }); + if (error is WorkLimit) { halted = true; failure = error.Message; } + } + finally { scan.Dispose(); Directory.Delete(pending,true); } + ++visited; if (halted) break; + } + if (new FileInfo(path).Length != length || File.GetLastWriteTimeUtc(path) != modified || + headerLength is not null && (new FileInfo(headerPath).Length != headerLength || File.GetLastWriteTimeUtc(headerPath) != headerModified)) + throw new InvalidOperationException("Archive/header changed during scan; no output published"); + } + catch (Exception error) when (error is IOException or InvalidDataException or ArgumentException or IndexOutOfRangeException or OverflowException) + { + complete = false; failure = error.Message; if (error is WorkLimit) halted = true; + } + if (failure is not null || visited != files) complete = false; + summaries.Add(new { archive = path, bytes = length, modified_utc = modified, header = headerLength is null ? "embedded" : headerPath, + header_sha256 = headerSha, header_bytes = headerLength, entries = files < 0 ? (int?)null : files, visited, failed, + header_sha256_basis = headerLength is null ? "shared-reader-header-with-four-zero-prefix" : "complete-external-header", + unnamed_visited = unnamed, names_inventory = namesSupported ? "FindEntry-CRC-first-with-tree-fallback" : "unavailable-or-unsupported", + unvisited = files < 0 ? new { first = (int?)null, count = (int?)null } : new { first = (int?)(visited < files ? visited : null), count = (int?)(files-visited) }, failure }); + } + } + File.WriteAllText(Path.Combine(temporary,"embedded-index.json"),JsonSerializer.Serialize(new { + version = 1, status = complete ? "complete-structural-scan-not-compiled" : "incomplete-structural-scan-not-compiled", + extractor_assembly = typeof(EmbeddedShaderExtractor).Assembly.Location, + complete, archives = summaries, candidates = published, unique_containers = unique.Count, + decoded_bytes_visited = budget.Decoded, candidate_bytes_visited = budget.CandidateBytes, output_bytes = budget.OutputBytes, + entries_ledger = "entries.jsonl", containers_ledger = "containers.jsonl", + bounds = new { seconds, chunk_bytes = Chunk, header_bytes = MaximumHeader, entry_bytes = MaximumEntry, + decoded_bytes = MaximumDecoded, candidate_work_bytes = MaximumCandidateWork, output_bytes = MaximumOutput, candidates = MaximumRecords }, + limitations = new[] { "Structural candidates are not instruction/compilation validation or campaign coverage.", + "All indexed entries are attempted regardless of names. Sparse, oversized, decoder-failed and unvisited entries are explicit.", + "DFLT chunk limits are deliberate; other compression is not decoded and no metadata is reconstructed.", + "Complete source containers retain reflection and optional physical marker; runtime reflection stripping is a separate transformation.", + "Entry decoded SHA256 plus bounded header SHA256 identify scanned source ranges. Whole archive SHA256 is not computed.", + "SHA256 identities are an offline origin index; use lego_gpu_microcode_index for renderer placement XXH3 identities." } + },Json)); + Directory.Move(temporary,outputRoot); + Console.WriteLine($"embedded structural candidates: {published}; unique: {unique.Count}; complete: {complete}"); + return !complete ? 3 : published == 0 ? 1 : 0; + } + catch { if (Directory.Exists(temporary)) Directory.Delete(temporary,true); throw; } + } +} diff --git a/tools/gpu-shader-extract/ImageShaderExtractor.cs b/tools/gpu-shader-extract/ImageShaderExtractor.cs new file mode 100644 index 0000000..8325fa3 --- /dev/null +++ b/tools/gpu-shader-extract/ImageShaderExtractor.cs @@ -0,0 +1,266 @@ +using System.Buffers.Binary; +using System.Globalization; +using System.Security.Cryptography; +using System.Text.Json; + +namespace GpuShaderExtract; + +// This reads an already decoded/mapped image, not XEX compressed/encrypted bytes. +// Bounds describe the actual ShaderContainer/Shader/CTAB layouts used by the +// recomp compiler. Passing them establishes structural candidates, not valid DXIL. +internal static class ImageShaderExtractor +{ + internal const int HeaderBytes = 0x24; + internal const int MaximumSectionBytes = 4 * 1024 * 1024; + private const int MaximumContainerBytes = 2 * MaximumSectionBytes + 4; + private const int ChunkBytes = 2 * 1024 * 1024; + private const long MaximumImageBytes = 512L * 1024 * 1024; + private const int MaximumCandidates = 10000; + private static readonly byte[] Magic = { 0x10, 0x2a, 0x11 }; + + internal readonly record struct Layout(int Total, int PhysicalBase, int PhysicalBytes, + int InstructionOffset, int InstructionBytes, string Stage, + uint Flags, bool Marker); + internal enum Result { Invalid, Incomplete, Valid } + + private static uint Be(ReadOnlySpan bytes, int offset) => + BinaryPrimitives.ReadUInt32BigEndian(bytes.Slice(offset, 4)); + + private static bool Fits(ulong offset, ulong count, ulong end) => + offset <= end && count <= end - offset; + + private static bool ParseBase(string? text, out uint value) + { + value = 0; + if (text is not null && text.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) text = text[2..]; + return text?.Length == 8 && uint.TryParse(text, NumberStyles.AllowHexSpecifier, + CultureInfo.InvariantCulture, out value); + } + + internal static Result Parse(ReadOnlySpan bytes, bool eof, out Layout layout) + { + layout = default; + if (bytes.Length < HeaderBytes) return eof ? Result.Invalid : Result.Incomplete; + uint flags = Be(bytes, 0), virtualBytes = Be(bytes, 4), physicalBytes = Be(bytes, 8); + uint shader = Be(bytes, 24); + bool pixel = (flags & 1) == 0; + uint metadataBytes = pixel ? 32u : 36u; + if ((flags & 0xffffff00) != 0x102a1100 || Be(bytes, 28) != 0 || Be(bytes, 32) != 0 || + virtualBytes < HeaderBytes || virtualBytes > MaximumSectionBytes || + physicalBytes < 8 || physicalBytes > MaximumSectionBytes || + shader < HeaderBytes || !Fits(shader, metadataBytes, virtualBytes)) return Result.Invalid; + // Carry enough bytes to distinguish the optional archive marker. At EOF, + // a marker-less container may end exactly at virtual + physical bytes. + int minimum = checked((int)(virtualBytes + physicalBytes)); + if (bytes.Length < minimum + 4 && !eof) return Result.Incomplete; + if (bytes.Length < minimum) return Result.Invalid; + bool marker = Fits(virtualBytes, 4, (ulong)bytes.Length) && Be(bytes, (int)virtualBytes) == physicalBytes; + int physicalBase = (int)virtualBytes + (marker ? 4 : 0); + int total = physicalBase + (int)physicalBytes; + if (bytes.Length < total) return eof ? Result.Invalid : Result.Incomplete; + uint instruction = Be(bytes, (int)shader), instructionBytes = Be(bytes, (int)shader + 4); + if (instructionBytes < 8 || instructionBytes % 4 != 0 || + !Fits(instruction, instructionBytes, physicalBytes)) return Result.Invalid; + ReadOnlySpan virtualSection = bytes[..(int)virtualBytes]; + uint interpolators = (Be(bytes, (int)shader + 20) >> 5) & 31; + ulong elements = pixel ? interpolators : + (ulong)Be(bytes, (int)shader + 24) + Be(bytes, (int)shader + 28) + interpolators; + if (!Fits((ulong)shader + metadataBytes, elements * 4, virtualBytes)) return Result.Invalid; + uint constants = Be(bytes, 16), definitions = Be(bytes, 20); + if (!ConstantsFit(virtualSection, constants) || + !DefinitionsFit(virtualSection, definitions, physicalBytes)) return Result.Invalid; + layout = new Layout(total, physicalBase, (int)physicalBytes, + physicalBase + (int)instruction, (int)instructionBytes, + pixel ? "ps" : "vs", flags, marker); + return Result.Valid; + } + + private static bool ConstantsFit(ReadOnlySpan bytes, uint offset) + { + if (offset == 0) return true; // Reflection is genuinely optional. + if (offset < HeaderBytes || !Fits(offset, 32, (ulong)bytes.Length)) return false; + int table = (int)offset + 4; // ConstantInfo offsets are relative to ConstantTable. + uint count = Be(bytes, table + 12), info = Be(bytes, table + 16); + if (count > 4096 || !Fits((ulong)table + info, (ulong)count * 20, (ulong)bytes.Length)) return false; + for (uint i = 0; i < count; ++i) + { + int entry = checked(table + (int)info + (int)i * 20); + ulong name = (ulong)table + Be(bytes, entry); + if (!Fits(name, 1, (ulong)bytes.Length)) return false; + int available = Math.Min(1024, bytes.Length - (int)name); + if (bytes.Slice((int)name, available).IndexOf((byte)0) < 0) return false; + } + return true; + } + + private static bool DefinitionsFit(ReadOnlySpan bytes, uint offset, uint physicalBytes) + { + if (offset == 0) return true; + if (offset < HeaderBytes || !Fits(offset, 24, (ulong)bytes.Length)) return false; + int current = (int)offset + 20; + while (Fits((ulong)current, 4, (ulong)bytes.Length) && Be(bytes, current) != 0) + { + if (!Fits((ulong)current, 8, (ulong)bytes.Length)) return false; + uint count = BinaryPrimitives.ReadUInt16BigEndian(bytes.Slice(current + 2, 2)); + ulong values = ((ulong)count + 3) / 4 * 16; + if (!Fits(Be(bytes, current + 4), values, physicalBytes)) return false; + current += 8; + } + if (!Fits((ulong)current, 4, (ulong)bytes.Length)) return false; + current += 4; + while (Fits((ulong)current, 4, (ulong)bytes.Length) && Be(bytes, current) != 0) + { + uint count = BinaryPrimitives.ReadUInt16BigEndian(bytes.Slice(current + 2, 2)); + // The real compiler reads count values at +4 and advances 2+count words. + ulong advance = 8UL + (ulong)count * 4; + if (!Fits((ulong)current, advance, (ulong)bytes.Length)) return false; + current += (int)advance; + } + return Fits((ulong)current, 4, (ulong)bytes.Length); + } + + private static string Sha(ReadOnlySpan bytes) => + Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + + internal static int Run(string[] args) + { + if (args.Length < 4) + { + Console.Error.WriteLine("Usage: gpu-shader-extract extract --image --image-base <8hex> [--image-provenance ]"); + return 2; + } + try + { + string image = Path.GetFullPath(args[0]), output = Path.GetFullPath(args[1]); + uint? imageBase = null; + string? provenancePath = null; + for (int i = 2; i < args.Length; ++i) + { + string option = args[i]; + if (++i >= args.Length) throw new ArgumentException($"Missing value for {option}"); + if (option == "--image-base" && imageBase is null && ParseBase(args[i], out uint value)) imageBase = value; + else if (option == "--image-provenance" && provenancePath is null) provenancePath = Path.GetFullPath(args[i]); + else throw new ArgumentException($"Unknown, repeated or invalid option: {option}"); + } + if (imageBase is null) throw new ArgumentException("--image-base is required; image load address is never guessed"); + return Extract(image, output, imageBase.Value, provenancePath) == 0 ? 1 : 0; + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or + ArgumentException or JsonException or InvalidDataException) + { + Console.Error.WriteLine($"Image extraction rejected: {exception.Message}"); + return 2; + } + } + + internal static int Extract(string image, string output, uint imageBase, string? provenancePath) + { + var before = new FileInfo(image); + long imageBytes = before.Length; + DateTime modified = before.LastWriteTimeUtc; + if (imageBytes < HeaderBytes || imageBytes > MaximumImageBytes || + (ulong)imageBase + (ulong)imageBytes > 0x100000000UL) + throw new InvalidDataException("Image is empty, exceeds512MiB, or escapes its32-bit guest range"); + if (Directory.Exists(output) || File.Exists(output)) throw new IOException("Use a fresh output directory"); + JsonElement? suppliedProvenance = null; + if (provenancePath is not null) + { + if (new FileInfo(provenancePath).Length > 4*1024*1024) throw new InvalidDataException("Provenance exceeds4MiB"); + using var document = JsonDocument.Parse(File.ReadAllBytes(provenancePath)); + suppliedProvenance = document.RootElement.Clone(); + } + string parent = Path.GetDirectoryName(output)!; + Directory.CreateDirectory(parent); + string temporary = Path.Combine(parent, "." + Path.GetFileName(output) + ".image-extract-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(temporary); + try + { + var buffer = new byte[ChunkBytes + MaximumContainerBytes]; + int carry = 0, rejected = 0, count = 0; + long fileOffset = 0, candidateBytes = 0; + var records = new List(); + var unique = new HashSet(StringComparer.Ordinal); + using var imageHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + using (var source = new FileStream(image, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + bool eof; + do + { + int read = source.Read(buffer, carry, Math.Min(ChunkBytes, buffer.Length-carry)); + imageHash.AppendData(buffer.AsSpan(carry, read)); + eof = read == 0; + int length = carry + read, position = 0; + while (position + HeaderBytes <= length) + { + int magic = buffer.AsSpan(position, length-position).IndexOf(Magic); + if (magic < 0) { position = Math.Max(position, length-(HeaderBytes-1)); break; } + position += magic; + Result status = Parse(buffer.AsSpan(position, length-position), eof, out Layout layout); + if (status == Result.Incomplete) break; + if (status == Result.Invalid) { ++position; ++rejected; continue; } + if (++count > MaximumCandidates) throw new InvalidDataException("Image exceeds10000 structural candidates"); + candidateBytes += layout.Total; + if (candidateBytes > MaximumImageBytes) + throw new InvalidDataException("Overlapping candidate bytes exceed512MiB work/output bound"); + var full = buffer.AsSpan(position, layout.Total); + string sha = Sha(full), filename = layout.Stage + "-" + sha + ".bin"; + if (unique.Add(filename)) File.WriteAllBytes(Path.Combine(temporary, filename), full.ToArray()); + var physical = full.Slice(layout.PhysicalBase, layout.PhysicalBytes); + var instruction = full.Slice(layout.InstructionOffset, layout.InstructionBytes); + records.Add(new { image_offset = fileOffset+position, + guest_address = ((ulong)imageBase+(ulong)fileOffset+(ulong)position).ToString("X8"), + stage = layout.Stage, flags = layout.Flags.ToString("X8"), bytes = layout.Total, + archive_marker = layout.Marker, file = filename, container_sha256 = sha, + container_prefix8 = Convert.ToHexString(full[..8]), + physical = new { offset = layout.PhysicalBase, bytes = layout.PhysicalBytes, + sha256 = Sha(physical), prefix8 = Convert.ToHexString(physical[..8]) }, + instruction = new { offset = layout.InstructionOffset, bytes = layout.InstructionBytes, + sha256 = Sha(instruction), prefix8 = Convert.ToHexString(instruction[..8]) } }); + // Visit every possible header start, including containers + // embedded in a structurally-valid enclosing candidate. + ++position; + } + carry = length-position; + Buffer.BlockCopy(buffer, position, buffer, 0, carry); + fileOffset += position; + } while (!eof); + } + string imageSha = Convert.ToHexString(imageHash.GetHashAndReset()).ToLowerInvariant(); + before.Refresh(); + if (before.Length != imageBytes || before.LastWriteTimeUtc != modified) + throw new IOException("Image changed during extraction"); + if (suppliedProvenance is JsonElement proof) + { + if (proof.ValueKind != JsonValueKind.Object || + !proof.TryGetProperty("mapped_image", out var mapped) || mapped.ValueKind != JsonValueKind.Object || + !mapped.TryGetProperty("sha256", out var sha) || sha.ValueKind != JsonValueKind.String || sha.GetString() != imageSha || + !mapped.TryGetProperty("bytes", out var size) || size.ValueKind != JsonValueKind.Number || + !size.TryGetInt64(out long recordedSize) || recordedSize != imageBytes || + !mapped.TryGetProperty("base", out var address) || address.ValueKind != JsonValueKind.String || + !ParseBase(address.GetString(), out uint recordedBase) || recordedBase != imageBase) + throw new InvalidDataException("Supplied mapped_image provenance differs from actual image/base"); + } + var index = new { version = 1, status = "structural-candidates-not-compiled", image = new { + path = image, bytes = imageBytes, sha256 = imageSha, guest_base = imageBase.ToString("X8") }, + supplied_loader_provenance = suppliedProvenance, + supplied_loader_provenance_identity_matched = suppliedProvenance is not null, + candidates = count, candidate_bytes_visited = candidateBytes, + unique_containers = unique.Count, rejected_magic_candidates = rejected, + trailing_unexamined_bytes = carry, records, + limits = new[] { "Input must already be faithfully decoded and patched. This tool does not decode XEX or authenticate a provenance author.", + "Structural bounds do not establish instruction semantics, successful compilation, or campaign shader coverage.", + "Complete container bytes and optional physical markers are preserved; metadata is never reconstructed.", + "Every possible header start is searched, including overlapping/nested containers; truncated trailing bytes cannot contain a complete header.", + "SHA256/prefix8 identities are an offline origin index. Generate the renderer placement XXH3 index with lego_gpu_microcode_index." } }; + File.WriteAllText(Path.Combine(temporary, "image-index.json"), JsonSerializer.Serialize(index, new JsonSerializerOptions { WriteIndented = true })); + Directory.Move(temporary, output); + Console.WriteLine($"image structural candidates: {count}; unique containers: {unique.Count}; sha256: {imageSha}"); + return count; + } + catch + { + if (Directory.Exists(temporary)) Directory.Delete(temporary, true); + throw; + } + } +} diff --git a/tools/gpu-shader-extract/Program.cs b/tools/gpu-shader-extract/Program.cs index b19f13d..c1ce690 100644 --- a/tools/gpu-shader-extract/Program.cs +++ b/tools/gpu-shader-extract/Program.cs @@ -12,10 +12,20 @@ internal static class Program private static int Main(string[] args) { + if (args.Length >= 2 && args[0] == "extract" && args[1] == "--embedded") + { + return EmbeddedShaderExtractor.Run(args[2..]); + } + if (args.Length >= 2 && args[0] == "extract" && args[1] == "--image") + { + return ImageShaderExtractor.Run(args[2..]); + } if (args.Length < 2 || (args[0] != "inventory" && args[0] != "extract")) { Console.Error.WriteLine( - "Usage: gpu-shader-extract [output-dir]"); + "Usage: gpu-shader-extract [output-dir]\n" + + " gpu-shader-extract extract --image --image-base <8hex> [--image-provenance ]\n" + + " gpu-shader-extract extract --embedded [--max-seconds <1..3600>]"); return 2; } @@ -38,10 +48,14 @@ private static int Main(string[] args) int extractedEntries = 0; int skippedEntries = 0; - foreach (string datPath in Directory.EnumerateFiles(gameDir, "*.DAT", + foreach (string datPath in Directory.EnumerateFiles(gameDir, "*", SearchOption.TopDirectoryOnly) + .Where(path => + Path.GetExtension(path).Equals(".DAT", StringComparison.OrdinalIgnoreCase) || + Path.GetExtension(path).Equals(".DAT2", StringComparison.OrdinalIgnoreCase)) .OrderBy(path => path, - StringComparer.OrdinalIgnoreCase)) + StringComparer.OrdinalIgnoreCase) + .ThenBy(path => path, StringComparer.Ordinal)) { string hdrPath = DatArchive.HdrPathFor(datPath); if (!File.Exists(hdrPath)) @@ -58,7 +72,7 @@ private static int Main(string[] args) Console.WriteLine($"{Path.GetFileName(datPath)}: {archive.FileCount} entries, " + $"{names.Length} named"); - foreach ((string internalPath, int entryIndex) in names) + foreach ((string internalPath, int nameTreeIndex) in names) { string extension = Path.GetExtension(internalPath); extension = string.IsNullOrEmpty(extension) ? "" : extension.ToLowerInvariant(); @@ -70,9 +84,30 @@ private static int Main(string[] args) continue; } + string archiveName = Path.GetFileNameWithoutExtension(datPath); + string destination; + try + { + destination = ArchiveOutputPath.Resolve( + Path.Combine(outputDir!, archiveName), internalPath); + } + catch (InvalidDataException exception) + { + Console.Error.WriteLine( + $"skip {Path.GetFileName(datPath)}:{internalPath}: {exception.Message}"); + skippedEntries++; + continue; + } + byte[] data; try { + // New-format tree values are sequential name positions, + // not necessarily DAT indices. The shared lookup resolves + // the archive CRC table before its tree fallback. + int entryIndex = archive.FindEntry(internalPath); + if (entryIndex < 0 || entryIndex >= archive.FileCount) + throw new InvalidDataException($"No file-table entry for name-tree index {nameTreeIndex}"); data = archive.ReadEntryDataDecompressed(entryIndex); } catch (Exception exception) when (exception is IOException or @@ -89,11 +124,6 @@ InvalidDataException or skippedEntries++; continue; } - string archiveName = Path.GetFileNameWithoutExtension(datPath); - string safeRelative = internalPath.Replace('\\', Path.DirectorySeparatorChar) - .Replace('/', Path.DirectorySeparatorChar) - .TrimStart(Path.DirectorySeparatorChar); - string destination = Path.Combine(outputDir!, archiveName, safeRelative); Directory.CreateDirectory(Path.GetDirectoryName(destination)!); File.WriteAllBytes(destination, data); extractedEntries++; @@ -111,7 +141,7 @@ InvalidDataException or if (args[0] == "extract") { Console.WriteLine($"extracted shader candidates: {extractedEntries}"); - Console.WriteLine($"skipped unreadable candidates: {skippedEntries}"); + Console.WriteLine($"skipped candidates: {skippedEntries}"); return extractedEntries == 0 ? 1 : 0; } return 0; diff --git a/tools/gpu-shader-extract/README.md b/tools/gpu-shader-extract/README.md index 2e577b1..51ac067 100644 --- a/tools/gpu-shader-extract/README.md +++ b/tools/gpu-shader-extract/README.md @@ -13,3 +13,87 @@ gpu-shader-extract extract `extract` writes only likely shader-container files (`.vso`, `.pso`, related extensions, and paths containing `shader`) beneath one directory per archive. The extracted directory can be passed directly to XenosRecomp. +Names resolve through the shared reader's CRC-first `FindEntry` lookup. In new +archives, the name tree's sequential values can differ from DAT file-table +indices; using those values directly extracts the wrong payload. In particular, +TU23's named `*_nxg.360_shaders` can contain the source programs whose runtime +copies omit optional reflection. Keep the original metadata during extraction. + +To audit every indexed entry, including unnamed entries and containers embedded +in other resources, use the opt-in streaming mode: + +```powershell +gpu-shader-extract extract --embedded --max-seconds 300 +``` + +This scans a single `.DAT`/`.DAT2` or all such files directly beneath a directory, +using the actual archive reader and DFLT decoder. Raw and decoded data are read +in 2 MiB chunks; no entire archive entry is allocated. It searches every possible +container start, including nested/overlapping records, with the same metadata, +reflection and physical-range validator used by image mode. Exact complete +source containers share files by SHA256; reflection is never stripped or rebuilt. +Each entry's output remains provisional until its entire compression framing +and declared decoded size pass validation. +This checks DFLT chunk headers, decoder success and exact produced byte counts, +including short and overlong output. It retains the shared decoder's bitstream +semantics, including zero padding; it is not a new strict bitstream validator. + +`embedded-index.json` reports completion, per-archive inventory, bounds and +unvisited ranges. `entries.jsonl` records every attempted file-table index, +CRC-first resolved name aliases, source range, decoded SHA256 and explicit +failures. `containers.jsonl` records each accepted source entry/decoded offset +and full/physical/instruction SHA256 identities. Names inherit the shared +reader's tree fallback when CRC lookup fails; the index states this policy. +Bounded header identities and decoded entry hashes are recorded; this mode +does **not** hash an entire multi-gigabyte archive. + +The default deadline is 300 seconds, adjustable from 1 to 3600. Limits include +64 MiB headers, 1 GiB stored/decoded entries, 2 MiB DFLT chunks, 64 GiB total +decoded work, 100000 candidate starts, 4096 starts per entry, 1 GiB candidate +hash work and 512 MiB output. Unsupported name layouts, sparse archives, +oversized entries, decoder failures and deadline exhaustion are explicit; +an incomplete scan publishes its ledger with exit **3**, never success. +Exit 0 is a completed scan with structural candidates; 1 is a completed scan +without candidates; 2 rejects arguments/input/output. Successful extraction +still requires separate instruction decoding and shader compilation. It does +not establish rendered correctness or whole-campaign coverage. + +Executable-resident shaders can also be extracted without starting the game: + +```powershell +gpu-shader-extract extract --image mapped-image.bin executable-shaders --image-base 82000000 --image-provenance loader-provenance.json +``` + +The input must contain the **mapped, decoded executable image after applying +the intended title update**. Passing an encrypted/compressed `Default.xex` or +its patch file does not substitute for SDK loading. The load address must be +supplied explicitly. Producing this image with the SDK loader is a separate +step; this tool does not load or execute guest code. + +Optional loader provenance must contain a matching record: + +```json +{"mapped_image":{"base":"0x82000000","bytes":45613056,"sha256":""}} +``` + +Additional source XEX, update and loader identities are retained in the output +index. Matching the supplied record checks image identity, not the provenance +author's authenticity. + +Image extraction scans all image bytes with bounded memory, validates container, +shader metadata, optional reflection and literal-definition ranges, and writes +complete containers beneath a **fresh** output directory. The four-byte archive +physical-size marker is retained when present. Identical full containers share +one file; every discovered image/guest address remains in `image-index.json`. +That index includes SHA256 and eight-byte prefixes for full containers, physical +sections and instruction streams. The existing `lego_gpu_microcode_index` tool +can produce the renderer's XXH3 placement lookup from these files. + +Input is limited to 512 MiB, each section to 4 MiB and discovered records to 10000. +Every possible header start is searched, including nested and overlapping +containers. Total candidate bytes are capped at 512 MiB to bound repeated +hashing and output. These are bounded +**structural candidates**: shader compilation must succeed separately before a +bank is used. Neither extraction nor compilation establishes full campaign +coverage or rendered correctness. Exit 1 means no image candidates were found; +exit 2 means rejected arguments, bounds, provenance or output. From 3f36efdfdacb2f6b7c94a9cc882f62e35cd61044 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:50:50 +0200 Subject: [PATCH 08/40] Summarize generic frame CSVs and preserve constant stage timings --- tools/gpu-aot/summarize_frame_metrics.py | 5 +++-- tools/gpu-aot/test_frame_metrics.py | 26 +++++++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/tools/gpu-aot/summarize_frame_metrics.py b/tools/gpu-aot/summarize_frame_metrics.py index e3b6c43..0b96d8c 100644 --- a/tools/gpu-aot/summarize_frame_metrics.py +++ b/tools/gpu-aot/summarize_frame_metrics.py @@ -43,10 +43,11 @@ def stats(data): for key in ['cpu_resource_wait_skips','bindings_ms','begin_ms','pipeline_ms','issue_ms', 'tail_ms','buffer_converted_bytes','frame_slot_wait_calls','frame_slot_wait_ms', 'acquire_cpu_ms','present_submit_cpu_ms','swap_present_cpu_ms', - 'buffer_watch_hits','buffer_watch_audits','buffer_watch_mismatches'] + [k for k in data[0] if k.startswith(('sync_', 'callbacks_'))]: + 'buffer_watch_hits','buffer_watch_audits','buffer_watch_mismatches'] + [k for k in data[0] if k.startswith(('sync_', 'callbacks_', 'constants_'))]: if key in data[0]:result[key+'_mean']=mean(key) return result -record_path=a.file.with_name(a.file.name.replace('-frames.csv','-process.json')) +record_path=(a.file.with_name(a.file.name[:-len('-frames.csv')]+'-process.json') + if a.file.name.endswith('-frames.csv') else a.file.with_suffix('.process.json')) record=json.loads(record_path.read_text(encoding='utf-8-sig')) if record_path.exists() else None heavy_capture=bool(record and (record.get('mode')=='snapshot' or any(record.get(k) for k in ['longProbe','logoUploads','logoCaptureRenderDocDll','captureMissing','meshTrace','auditTextureWatch','auditBufferWatch','portraitTrace']))) diff --git a/tools/gpu-aot/test_frame_metrics.py b/tools/gpu-aot/test_frame_metrics.py index a3992e6..8522549 100644 --- a/tools/gpu-aot/test_frame_metrics.py +++ b/tools/gpu-aot/test_frame_metrics.py @@ -14,19 +14,24 @@ fields=['frame','interval_ms','texture_ms','constants_ms','vertices_ms', 'buffer_hash_ms','present_cpu_ms','draw_calls','frame_slot_wait_ms', 'acquire_cpu_ms','present_submit_cpu_ms','swap_present_cpu_ms', - 'buffer_watch_hits','buffer_watch_audits','buffer_watch_mismatches'] + 'buffer_watch_hits','buffer_watch_audits','buffer_watch_mismatches', + 'constants_shadow_ms','constants_upload_ms','constants_binding_ms', + 'constants_texture_binding_ms','constants_restore_ms'] with path.open('w',newline='') as f: w=csv.DictWriter(f,fieldnames=fields);w.writeheader() for n in range(1,101): w.writerow(dict.fromkeys(fields,0)|{'frame':n,'interval_ms':n,'draw_calls':505, 'frame_slot_wait_ms':2,'acquire_cpu_ms':3,'present_submit_cpu_ms':4, - 'swap_present_cpu_ms':5,'buffer_watch_audits':6}) + 'swap_present_cpu_ms':5,'buffer_watch_audits':6, + 'constants_shadow_ms':n,'constants_upload_ms':n*2, + 'constants_binding_ms':n*3,'constants_texture_binding_ms':n*4, + 'constants_restore_ms':n*5}) for interval in ('nan','inf',0,-1): w.writerow(dict.fromkeys(fields,0)|{'frame':101,'interval_ms':interval}) f.write('102,1000,') record=path.with_name('fixture-process.json') -def report(*args): - return json.loads(subprocess.check_output([sys.executable,str(script),str(path),*args],text=True)) +def report(*args,metric_path=path): + return json.loads(subprocess.check_output([sys.executable,str(script),str(metric_path),*args],text=True)) record.write_text(json.dumps({'mode':'timing','auditBufferWatch':True})) r=report('--tail-seconds','0.1') s=r['whole_run'] @@ -35,6 +40,9 @@ def report(*args): assert s['over_16_67ms_percent']==84 and s['over_33_33ms_percent']==67 assert s['frame_slot_wait_ms_mean']==2 and s['swap_present_cpu_ms_mean']==5 assert s['buffer_watch_audits_mean']==6 and s['buffer_watch_mismatches_mean']==0 +for multiplier, key in enumerate(fields[-5:], 1): + assert s[key+'_mean']==50.5*multiplier + assert r['tail'][key+'_mean']==100*multiplier assert r['tail']['frames']==1 and not r['clean_performance_probe'] record.write_text(json.dumps({'mode':'timing','portraitTrace':True})) assert not report()['clean_performance_probe'] @@ -42,7 +50,15 @@ def report(*args): assert report()['clean_performance_probe'] r=report('--after-frame','90','--through-frame','95') assert r['whole_run']['frames']==5 and r['latest_frame']==95 +assert r['whole_run']['constants_shadow_ms_mean']==93 assert report('--after-frame','100')['whole_run']=={} record.unlink() assert report()['clean_performance_probe'] is None -print('PASS: CLI frame budgets, percentile tails, malformed/partial rows, window selection, audit exclusion') +generic=path.with_name('frames.csv') +generic.write_bytes(path.read_bytes()) +generic.with_suffix('.process.json').unlink(missing_ok=True) +r=report(metric_path=generic) +assert r['whole_run']['frames']==100 and r['clean_performance_probe'] is None +generic.with_suffix('.process.json').write_text(json.dumps({'mode':'snapshot'})) +assert report(metric_path=generic)['clean_performance_probe'] is False +print('PASS: CLI frame budgets, percentile tails, constant phase means, malformed/partial rows, window selection, audit exclusion') From a42d25a80522796794a4131829547180a60bd736 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:55:02 +0200 Subject: [PATCH 09/40] Record shader provenance limits and gameplay performance gates --- tools/gpu-aot/ENGINEERING-PLAN.md | 41 ++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/tools/gpu-aot/ENGINEERING-PLAN.md b/tools/gpu-aot/ENGINEERING-PLAN.md index 920da08..f4fc689 100644 --- a/tools/gpu-aot/ENGINEERING-PLAN.md +++ b/tools/gpu-aot/ENGINEERING-PLAN.md @@ -39,6 +39,25 @@ Portable helper and mocked-driver checks remain useful, but they cannot establis rendered pixel correctness. Add image checks and real GPU execution to that layer. The full campaign is an integration test after this loop is reliable. +The first matched Vorton captures have no confirmed renderer discrepancy. The +matching PM4 plugin requires the RTV render-target path on this Intel/Proton +setup; its default ROV path produced grossly incorrect output. Record that +selection instead of treating either reference configuration as ground truth. + +A static unpaused native Vorton sample measured 14.20 FPS across 444 complete +frame intervals, with 81.50 ms p95 and about 543 draws per frame. LongProbe and +detailed timing were enabled, so this is a failing instrumented performance +baseline, not an instrumentation-free benchmark. Paused UI results must never +substitute for gameplay performance. Trace-disabled and per-present-only probes +will quantify instrumentation cost before choosing an optimization. + +The measured command-slot waits averaged 14.87 ms per frame across 8.34 waits. +Guest kickoffs reuse the three-slot command ring independently of presentation. +A bounded larger-pool experiment must preserve every submission, query, +callback, fence and descriptor-retirement ticket; increasing swap-chain latency +or removing completion waits is not equivalent. Keep the current default until +GPU runs demonstrate a useful gain without regressions. + ## Shader coverage The static extractor filters named archive entries. Its output count does not @@ -47,9 +66,25 @@ installer archives, or programs constructed by the game at runtime. An initial audit found no exact physical or instruction-stream matches between the static-only index and 169 captured runtime containers. Removing container -reflection from identity therefore does not fix these misses. Establish whether -the missing programs exist in unscanned asset entries before designing a new -runtime shader-generation path. +reflection from identity therefore does not fix these misses. + +A bounded pass visited all 19,000 GAME.DAT entry indices without finding those +169 programs. Its legacy decoder did not verify actual produced byte counts, +so the compressed-entry negative result remains provisional. A faithful +SDK-loaded and TU23-patched executable image contained byte-identical metadata, +physical sections, and instruction +streams for 96 of them. Executable-resident shaders were missing from the +DAT-only input inventory. Include the patched image in offline extraction before +asking gameplay to discover this coverage again. + +The remaining 73 observed programs are exactly the 73 containers without CTAB +reflection. Their instruction streams did not match the scanned image. A bounded +PATCH.DAT scan subsequently traced 54 of them to ordinary named shader assets: +physical sections, instructions, and fixed shader metadata match byte for byte. +The engine's XDK repacking path removes reflection. The standard extractor was +reading name-tree positions as entry indices; new-format archives require the +authoritative CRC-based `FindEntry` lookup. Fix this association before judging +coverage from an extraction count. Nineteen observed origins remain unproved. Use bounded scan reports with explicit unreadable, sparse, malformed, and unexamined counts. Record exact container and microcode identities. Compile new From b6bebb1b1da7eca0b30d69ffcb8bf53b50955690 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:09:33 +0200 Subject: [PATCH 10/40] Distinguish cadence-only frame probes from detailed timing --- tools/gpu-aot/summarize_frame_metrics.py | 3 +++ tools/gpu-aot/test_frame_metrics.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/tools/gpu-aot/summarize_frame_metrics.py b/tools/gpu-aot/summarize_frame_metrics.py index 0b96d8c..7210a0a 100644 --- a/tools/gpu-aot/summarize_frame_metrics.py +++ b/tools/gpu-aot/summarize_frame_metrics.py @@ -45,6 +45,9 @@ def stats(data): 'acquire_cpu_ms','present_submit_cpu_ms','swap_present_cpu_ms', 'buffer_watch_hits','buffer_watch_audits','buffer_watch_mismatches'] + [k for k in data[0] if k.startswith(('sync_', 'callbacks_', 'constants_'))]: if key in data[0]:result[key+'_mean']=mean(key) + if 'detailed_timing_enabled' in data[0]: + modes={r['detailed_timing_enabled'] for r in data} + result['detailed_timing_enabled']=bool(next(iter(modes))) if len(modes)==1 else None return result record_path=(a.file.with_name(a.file.name[:-len('-frames.csv')]+'-process.json') if a.file.name.endswith('-frames.csv') else a.file.with_suffix('.process.json')) diff --git a/tools/gpu-aot/test_frame_metrics.py b/tools/gpu-aot/test_frame_metrics.py index 8522549..aa39315 100644 --- a/tools/gpu-aot/test_frame_metrics.py +++ b/tools/gpu-aot/test_frame_metrics.py @@ -61,4 +61,17 @@ def report(*args,metric_path=path): assert r['whole_run']['frames']==100 and r['clean_performance_probe'] is None generic.with_suffix('.process.json').write_text(json.dumps({'mode':'snapshot'})) assert report(metric_path=generic)['clean_performance_probe'] is False +cadence=path.with_name('cadence-frames.csv') +with cadence.open('w',newline='') as f: + w=csv.DictWriter(f,fieldnames=fields+['detailed_timing_enabled']);w.writeheader() + for n in range(1,4): + w.writerow(dict.fromkeys(fields,0)|{'frame':n,'interval_ms':20,'draw_calls':543, + 'detailed_timing_enabled':0}) +r=report(metric_path=cadence)['whole_run'] +assert r['mean_fps']==50 and r['draws_mean']==543 +assert r['detailed_timing_enabled'] is False +with cadence.open('a',newline='') as f: + csv.DictWriter(f,fieldnames=fields+['detailed_timing_enabled']).writerow( + dict.fromkeys(fields,0)|{'frame':4,'interval_ms':20,'detailed_timing_enabled':1}) +assert report(metric_path=cadence)['whole_run']['detailed_timing_enabled'] is None print('PASS: CLI frame budgets, percentile tails, constant phase means, malformed/partial rows, window selection, audit exclusion') From fdf068524cef810a7198ba02d433bafe66f0cc23 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:07:22 +0200 Subject: [PATCH 11/40] Pin public Plume and validate native D3D12 dependency fixes --- rexlego/CMakeLists.txt | 11 ++ thirdparty/plume | 2 +- tools/gpu-aot/CMakeLists.txt | 18 ++ tools/gpu-aot/patches/plume-lego.patch | 110 +++++++++++ tools/gpu-aot/prepare_plume.py | 106 +++++++++++ tools/gpu-aot/test_plume_command_creation.py | 165 +++++++++++++++++ tools/gpu-aot/test_prepare_plume.py | 181 +++++++++++++++++++ 7 files changed, 592 insertions(+), 1 deletion(-) create mode 100644 tools/gpu-aot/patches/plume-lego.patch create mode 100644 tools/gpu-aot/prepare_plume.py create mode 100644 tools/gpu-aot/test_plume_command_creation.py create mode 100644 tools/gpu-aot/test_prepare_plume.py diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt index 1fc435a..d6b68e6 100644 --- a/rexlego/CMakeLists.txt +++ b/rexlego/CMakeLists.txt @@ -42,6 +42,17 @@ if(LEGODIMENSIONS_NATIVE_GPU) endif() enable_language(C) + find_package(Python3 COMPONENTS Interpreter REQUIRED) + execute_process( + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/prepare_plume.py" --check + RESULT_VARIABLE LEGO_PLUME_PREPARE_RESULT + OUTPUT_VARIABLE LEGO_PLUME_PREPARE_OUTPUT + ERROR_VARIABLE LEGO_PLUME_PREPARE_ERROR + TIMEOUT 30) + if(NOT LEGO_PLUME_PREPARE_RESULT STREQUAL "0") + message(FATAL_ERROR "Native Plume dependency needs preparation. Run python tools/gpu-aot/prepare_plume.py.\n${LEGO_PLUME_PREPARE_OUTPUT}${LEGO_PLUME_PREPARE_ERROR}") + endif() set(PLUME_D3D12_AGILITY_SDK_ENABLED OFF CACHE BOOL "" FORCE) add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/plume" "${CMAKE_CURRENT_BINARY_DIR}/thirdparty/plume" diff --git a/thirdparty/plume b/thirdparty/plume index ceaeeee..e0c8871 160000 --- a/thirdparty/plume +++ b/thirdparty/plume @@ -1 +1 @@ -Subproject commit ceaeeee0f7e914886542f041a52ecb40b76c8aaf +Subproject commit e0c8871b930dc1a6544bf457f2348c456b9d7e13 diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index 4e9ee76..3a30bd1 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -21,6 +21,24 @@ foreach(check IN LISTS CHECKS) add_test(NAME ${check} COMMAND test_${check}) endforeach() +# Validate the public native-renderer dependency with real source and fake COM. +if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") + find_package(Python3 COMPONENTS Interpreter QUIET) + if(Python3_Interpreter_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/plume_d3d12.cpp") + add_test(NAME plume_command_creation + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_plume_command_creation.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/plume_command_creation + --compiler ${CMAKE_CXX_COMPILER}) + set_tests_properties(plume_command_creation PROPERTIES TIMEOUT 60) + if(UNIX AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/.git") + add_test(NAME prepare_plume + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_prepare_plume.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/prepare_plume) + set_tests_properties(prepare_plume PROPERTIES TIMEOUT 60) + endif() + endif() +endif() + # These fixtures exercise the real extraction CLI with synthetic inputs. if(BUILD_TESTING) find_package(Python3 COMPONENTS Interpreter QUIET) diff --git a/tools/gpu-aot/patches/plume-lego.patch b/tools/gpu-aot/patches/plume-lego.patch new file mode 100644 index 0000000..6da4cc5 --- /dev/null +++ b/tools/gpu-aot/patches/plume-lego.patch @@ -0,0 +1,110 @@ +diff --git a/plume_d3d12.cpp b/plume_d3d12.cpp +index dcae4e3..7d7315a 100644 +--- a/plume_d3d12.cpp ++++ b/plume_d3d12.cpp +@@ -1416,24 +1416,32 @@ namespace plume { + } + + bool D3D12SwapChain::resize() { +- getWindowSize(width, height); ++ uint32_t newWidth = 0, newHeight = 0; ++ getWindowSize(newWidth, newHeight); + + // Don't resize the swap chain at all if the window doesn't have a valid size. +- if ((width == 0) || (height == 0)) { ++ if ((newWidth == 0) || (newHeight == 0)) { + return false; + } + + for (uint32_t i = 0; i < desc.textureCount; i++) { +- textures[i].d3d->Release(); +- textures[i].d3d = nullptr; ++ if (textures[i].d3d != nullptr) { ++ textures[i].d3d->Release(); ++ textures[i].d3d = nullptr; ++ } + } + + HRESULT res = d3d->ResizeBuffers(0, 0, 0, DXGI_FORMAT_UNKNOWN, swapChainFlags); + if (FAILED(res)) { + fprintf(stderr, "ResizeBuffers failed with error code 0x%lX.\n", res); ++ // DXGI retains the old buffers on a failed resize. Reacquire them ++ // so a retry does not operate on released or null references. ++ setTextures(); + return false; + } + ++ width = newWidth; ++ height = newHeight; + setTextures(); + return true; + } +@@ -2679,7 +2687,11 @@ namespace plume { + } + + std::unique_ptr D3D12CommandQueue::createCommandList() { +- return std::make_unique(this); ++ auto commandList = std::make_unique(this); ++ if (commandList->d3d == nullptr || commandList->commandAllocator == nullptr) { ++ return nullptr; ++ } ++ return commandList; + } + + std::unique_ptr D3D12CommandQueue::createSwapChain(const RenderSwapChainDesc &desc) { +@@ -2724,7 +2736,19 @@ namespace plume { + assert(fence != nullptr); + + D3D12CommandFence *interfaceFence = static_cast(fence); +- WaitForSingleObjectEx(interfaceFence->fenceEvent, INFINITE, FALSE); ++ const uint64_t targetValue = interfaceFence->fenceValue - 1; ++ // The auto-reset event may still belong to an earlier submission, or ++ // already have been consumed by another wait. The fence value is the ++ // completion authority; only wait while the submitted value is pending. ++ for (;;) { ++ const uint64_t completed = interfaceFence->d3d->GetCompletedValue(); ++ if (completed == UINT64_MAX || completed >= targetValue) { ++ return; ++ } ++ if (WaitForSingleObjectEx(interfaceFence->fenceEvent, INFINITE, FALSE) != WAIT_OBJECT_0) { ++ return; ++ } ++ } + } + + // D3D12Buffer +@@ -3939,7 +3963,11 @@ namespace plume { + } + + std::unique_ptr D3D12Device::createCommandQueue(RenderCommandListType type) { +- return std::make_unique(this, type); ++ auto commandQueue = std::make_unique(this, type); ++ if (commandQueue->d3d == nullptr) { ++ return nullptr; ++ } ++ return commandQueue; + } + + std::unique_ptr D3D12Device::createBuffer(const RenderBufferDesc &desc) { +@@ -3968,11 +3996,19 @@ namespace plume { + } + + std::unique_ptr D3D12Device::createCommandFence() { +- return std::make_unique(this); ++ auto fence = std::make_unique(this); ++ if (fence->d3d == nullptr || fence->fenceEvent == 0) { ++ return nullptr; ++ } ++ return fence; + } + + std::unique_ptr D3D12Device::createCommandSemaphore() { +- return std::make_unique(this); ++ auto semaphore = std::make_unique(this); ++ if (semaphore->d3d == nullptr) { ++ return nullptr; ++ } ++ return semaphore; + } + + std::unique_ptr D3D12Device::createFramebuffer(const RenderFramebufferDesc &desc) { diff --git a/tools/gpu-aot/prepare_plume.py b/tools/gpu-aot/prepare_plume.py new file mode 100644 index 0000000..3e7fa44 --- /dev/null +++ b/tools/gpu-aot/prepare_plume.py @@ -0,0 +1,106 @@ +"""Apply or verify the native renderer's fixes on the public pinned Plume source.""" + +import argparse +from pathlib import Path +import re +import subprocess + + +PIN = "e0c8871b930dc1a6544bf457f2348c456b9d7e13" + + +def verify_patched_source(checkout, patch): + """Check every new-side hunk, independently of git apply's exit status. + + Git may skip patches outside its current prefix and still return success. + Matching the actual hunks also allows unrelated local edits to survive. + """ + target = None + hunks = 0 + expected = None + lines = [] + + def finish(): + nonlocal hunks + if expected is None: + return + if len(lines) != expected or not lines or target is None: + raise ValueError("Invalid native Plume patch hunk") + path = checkout / target + if path.resolve() != path or not path.is_file(): + raise ValueError(f"Native Plume source verification failed: invalid source path {target}") + if "\n".join(lines) + "\n" not in path.read_text(): + raise ValueError(f"Native Plume source verification failed: missing patched hunk in {target}") + hunks += 1 + + for line in patch.read_text().splitlines(): + if line.startswith("diff --git "): + finish() + expected = None + lines = [] + target = None + elif line.startswith("+++ b/"): + target = Path(line[6:]) + if target.is_absolute() or ".." in target.parts: + raise ValueError("Invalid native Plume patch target") + elif line.startswith("@@ "): + finish() + header = re.fullmatch(r"@@ -\d+(?:,\d+)? \+\d+(?:,(\d+))? @@.*", line) + if header is None: + raise ValueError("Invalid native Plume patch header") + expected = int(header.group(1) or "1") + lines = [] + elif expected is not None: + if line.startswith((" ", "+")): + lines.append(line[1:]) + elif not line.startswith(("-", "\\ No newline")): + raise ValueError("Invalid native Plume patch content") + finish() + if hunks == 0: + raise ValueError("Native Plume patch has no verifiable hunks") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + root = Path(__file__).resolve().parents[2] + parser.add_argument("checkout", type=Path, nargs="?", default=root / "thirdparty/plume") + parser.add_argument("--check", action="store_true", help="Verify only; do not modify the checkout") + args = parser.parse_args() + checkout = args.checkout.resolve() + patch = root / "tools/gpu-aot/patches/plume-lego.patch" + + def git(*arguments, check=False): + return subprocess.run(["git", "-C", str(checkout), *arguments], + cwd=checkout, check=check, capture_output=True, text=True, timeout=30) + + if not checkout.is_dir(): + parser.error(f"Plume checkout does not exist: {checkout}") + repository = git("rev-parse", "--show-toplevel") + if repository.returncode or Path(repository.stdout.strip()).resolve() != checkout: + parser.error("Plume checkout must be the repository root, not a subdirectory") + revision = git("rev-parse", "HEAD", check=True).stdout.strip() + if revision != PIN: + parser.error(f"Expected public Plume {PIN}, got {revision}; select the recorded dependency revision") + if git("apply", "--reverse", "--check", str(patch)).returncode == 0: + try: + verify_patched_source(checkout, patch) + except ValueError as error: + parser.error(str(error)) + print(f"PASS: public Plume {PIN} with native fence/resize/command-allocation fixes") + return + if args.check: + parser.error("Native Plume fixes are missing or incomplete; run prepare_plume.py without --check") + applicability = git("apply", "--check", str(patch)) + if applicability.returncode: + parser.error("Cannot apply native Plume fixes without conflicting with existing edits:\n" + applicability.stderr.strip()) + git("apply", str(patch), check=True) + git("apply", "--reverse", "--check", str(patch), check=True) + try: + verify_patched_source(checkout, patch) + except ValueError as error: + parser.error(str(error)) + print(f"Applied native fence/resize/command-allocation fixes to public Plume {PIN}") + + +if __name__ == "__main__": + main() diff --git a/tools/gpu-aot/test_plume_command_creation.py b/tools/gpu-aot/test_plume_command_creation.py new file mode 100644 index 0000000..ef569e1 --- /dev/null +++ b/tools/gpu-aot/test_plume_command_creation.py @@ -0,0 +1,165 @@ +"""Run production Plume command constructors/factories with failing fake COM APIs.""" +import argparse +from pathlib import Path +import subprocess + + +def function(source, signature): + start = source.index(signature) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + parser.add_argument("--source", type=Path) + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + path = args.source or root / "thirdparty/plume/plume_d3d12.cpp" + if not path.is_file(): + parser.error("Initialize public thirdparty/plume and run prepare_plume.py before this fixture") + production = path.read_text() + bodies = "\n".join(function(production, signature) for signature in ( + "D3D12CommandList::D3D12CommandList(", "D3D12CommandList::~D3D12CommandList(", + "D3D12CommandFence::D3D12CommandFence(", "D3D12CommandFence::~D3D12CommandFence(", + "D3D12CommandSemaphore::D3D12CommandSemaphore(", "D3D12CommandSemaphore::~D3D12CommandSemaphore(", + "D3D12CommandQueue::D3D12CommandQueue(", "D3D12CommandQueue::~D3D12CommandQueue(", + "std::unique_ptr D3D12CommandQueue::createCommandList(", + "std::unique_ptr D3D12Device::createCommandQueue(", + "std::unique_ptr D3D12Device::createCommandFence(", + "std::unique_ptr D3D12Device::createCommandSemaphore(", + )) + harness = r''' +#include +#include +#include +#include +#include +using HRESULT=long;using HANDLE=std::uintptr_t;using UINT64=uint64_t; +#define FAILED(value) ((value)<0) +#define IID_PPV_ARGS(value) (value) +constexpr int FALSE=0,D3D12_FENCE_FLAG_NONE=0; +constexpr int D3D12_COMMAND_QUEUE_PRIORITY_NORMAL=0,D3D12_COMMAND_QUEUE_FLAG_NONE=0; +using D3D12_COMMAND_LIST_TYPE=int; +constexpr int D3D12_COMMAND_LIST_TYPE_DIRECT=0,D3D12_COMMAND_LIST_TYPE_COMPUTE=1,D3D12_COMMAND_LIST_TYPE_COPY=2; +struct D3D12_COMMAND_QUEUE_DESC {int Priority=0,Flags=0,NodeMask=0,Type=0;}; +enum class Failure {None,Allocator,List,Fence,Event,Queue}; +Failure failure=Failure::None; +struct Native { + static inline int live=0; + Native(){++live;}virtual ~Native(){--live;} + void Release(){delete this;} +}; +struct ID3D12CommandAllocator:Native {}; +struct ID3D12GraphicsCommandList:Native { + bool closed=false; + templateHRESULT QueryInterface(T** out){*out=nullptr;return -1;} + HRESULT Close(){closed=true;return 0;} +}; +struct ID3D12Fence:Native {UINT64 initial;explicit ID3D12Fence(UINT64 value):initial(value){}}; +struct ID3D12CommandQueue:Native {int type;explicit ID3D12CommandQueue(int value):type(value){}}; +int live_events=0; +HANDLE CreateEvent(void*,int manual,int initial,void*){ + assert(!manual&&!initial);if(failure==Failure::Event)return 0; + ++live_events;return 1; +} +void CloseHandle(HANDLE event){assert(event==1&&live_events>0);--live_events;} +struct FakeDevice { + HRESULT CreateCommandAllocator(int,ID3D12CommandAllocator** out){ + if(failure==Failure::Allocator)return -1;*out=new ID3D12CommandAllocator;return 0; + } + HRESULT CreateCommandList(int,int,ID3D12CommandAllocator* allocator,void*,ID3D12GraphicsCommandList** out){ + assert(allocator);if(failure==Failure::List)return -1;*out=new ID3D12GraphicsCommandList;return 0; + } + HRESULT CreateFence(UINT64 initial,int flags,ID3D12Fence** out){ + assert(flags==0);if(failure==Failure::Fence)return -1;*out=new ID3D12Fence(initial);return 0; + } + HRESULT CreateCommandQueue(const D3D12_COMMAND_QUEUE_DESC* desc,ID3D12CommandQueue** out){ + if(failure==Failure::Queue)return -1;*out=new ID3D12CommandQueue(desc->Type);return 0; + } +}; +namespace plume { +enum class RenderCommandListType {DIRECT,COMPUTE,COPY,UNKNOWN}; +struct RenderCommandList {virtual ~RenderCommandList()=default;}; +struct RenderCommandFence {virtual ~RenderCommandFence()=default;}; +struct RenderCommandSemaphore {virtual ~RenderCommandSemaphore()=default;}; +struct RenderCommandQueue {virtual ~RenderCommandQueue()=default;}; +struct D3D12Device;struct D3D12CommandQueue; +struct D3D12CommandList:RenderCommandList { + ID3D12GraphicsCommandList* d3d=nullptr; + ID3D12GraphicsCommandList* d3dV1=nullptr; + ID3D12GraphicsCommandList* d3dV4=nullptr; + ID3D12CommandAllocator* commandAllocator=nullptr;D3D12CommandQueue* queue=nullptr; + D3D12CommandList(D3D12CommandQueue*);~D3D12CommandList()override; +}; +struct D3D12CommandFence:RenderCommandFence { + ID3D12Fence* d3d=nullptr;D3D12Device* device=nullptr;HANDLE fenceEvent=0;UINT64 fenceValue=0; + D3D12CommandFence(D3D12Device*);~D3D12CommandFence()override; +}; +struct D3D12CommandSemaphore:RenderCommandSemaphore { + ID3D12Fence* d3d=nullptr;D3D12Device* device=nullptr;UINT64 semaphoreValue=0; + D3D12CommandSemaphore(D3D12Device*);~D3D12CommandSemaphore()override; +}; +struct D3D12CommandQueue:RenderCommandQueue { + ID3D12CommandQueue* d3d=nullptr;D3D12Device* device=nullptr;RenderCommandListType type=RenderCommandListType::UNKNOWN; + D3D12CommandQueue(D3D12Device*,RenderCommandListType);~D3D12CommandQueue()override; + std::unique_ptr createCommandList(); +}; +struct D3D12Device { + FakeDevice* d3d; + std::unique_ptr createCommandQueue(RenderCommandListType); + std::unique_ptr createCommandFence(); + std::unique_ptr createCommandSemaphore(); +}; +''' + tests = r''' +} +int main(){ + using namespace plume;FakeDevice native;D3D12Device device{&native}; + for(auto type:{RenderCommandListType::DIRECT,RenderCommandListType::COMPUTE,RenderCommandListType::COPY}){ + failure=Failure::Queue;assert(!device.createCommandQueue(type));assert(Native::live==0); + failure=Failure::None;auto queue=device.createCommandQueue(type);assert(queue&&Native::live==1); + auto* typed=static_cast(queue.get());assert(typed->d3d&&typed->type==type); + for(auto mode:{Failure::Allocator,Failure::List}){ + failure=mode;assert(!typed->createCommandList()); + assert(Native::live==1); // A failed list releases its partially created allocator. + } + failure=Failure::None;auto commands=typed->createCommandList();assert(commands&&Native::live==3); + auto* list=static_cast(commands.get()); + assert(list->d3d&&list->commandAllocator&&list->d3d->closed); + commands.reset();assert(Native::live==1);queue.reset();assert(Native::live==0); + } + for(auto mode:{Failure::Fence,Failure::Event}){ + failure=mode;assert(!device.createCommandFence()); + assert(Native::live==0&&live_events==0); // Failed event creation releases the COM fence. + } + failure=Failure::None;auto fence=device.createCommandFence();assert(fence); + auto* typed_fence=static_cast(fence.get()); + assert(typed_fence->d3d&&typed_fence->d3d->initial==0&&typed_fence->fenceValue==1); + assert(typed_fence->fenceEvent&&live_events==1&&Native::live==1); + fence.reset();assert(Native::live==0&&live_events==0); + failure=Failure::Fence;assert(!device.createCommandSemaphore());assert(Native::live==0); + failure=Failure::None;auto semaphore=device.createCommandSemaphore();assert(semaphore); + auto* typed_semaphore=static_cast(semaphore.get()); + assert(typed_semaphore->d3d&&typed_semaphore->d3d->initial==1&&typed_semaphore->semaphoreValue==1); + semaphore.reset();assert(Native::live==0&&live_events==0); + std::cout<<"PASS: Plume command factories propagate allocator/list/fence/event/queue failures and clean partial resources.\n"; +} +''' + args.output.mkdir(parents=True, exist_ok=True) + source = args.output.resolve() / "plume-command-creation.cpp" + exe = args.output.resolve() / "plume-command-creation.exe" + source.write_text(harness + bodies + tests) + subprocess.run([args.compiler, "-std=c++20", "-UNDEBUG", str(source), "-o", str(exe)], + check=True, timeout=45) + subprocess.run([str(exe)], check=True, timeout=10) + + +if __name__ == "__main__": + main() diff --git a/tools/gpu-aot/test_prepare_plume.py b/tools/gpu-aot/test_prepare_plume.py new file mode 100644 index 0000000..55af496 --- /dev/null +++ b/tools/gpu-aot/test_prepare_plume.py @@ -0,0 +1,181 @@ +"""Prepare real pinned public Plume in disposable, offline Git checkouts. + +No compiler, assets, downloads, or edits to the source checkout are required. +""" +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +PIN = "e0c8871b930dc1a6544bf457f2348c456b9d7e13" + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + root = Path(__file__).resolve().parents[2] + parser.add_argument("output", type=Path) + parser.add_argument("--checkout", type=Path, default=root / "thirdparty/plume") + parser.add_argument("--preparer", type=Path, default=root / "tools/gpu-aot/prepare_plume.py") + args = parser.parse_args() + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=True) + source = args.checkout.resolve() + preparer = args.preparer.resolve() + patch = root / "tools/gpu-aot/patches/plume-lego.patch" + git_executable = shutil.which("git") + if git_executable is None: + parser.error("Git is required for the offline public-source fixture") + checks = [] + + def command(argv, cwd, *, expect=0, oracle=None): + result = subprocess.run(argv, cwd=cwd, capture_output=True, text=True, timeout=15) + if result.returncode != expect or (oracle is not None and oracle not in result.stdout + result.stderr): + raise AssertionError(f"Unexpected preparation result: expected exit {expect}, oracle {oracle!r}; " + f"got {result.returncode}\n{result.stdout}\n{result.stderr}") + return result + + def git(checkout, *arguments): + return command([git_executable, "-C", str(checkout), *arguments], args.output) + + # Record Git metadata and the dirty source before any sandbox operations. + original = { + "head": git(source, "rev-parse", "HEAD").stdout, + "status": git(source, "status", "--porcelain=v1", "--untracked-files=all").stdout, + "source_sha256": digest(source / "plume_d3d12.cpp"), + } + git(source, "cat-file", "-e", f"{PIN}^{{commit}}") + public_blob = subprocess.run([git_executable, "-C", str(source), "show", f"{PIN}:plume_d3d12.cpp"], + capture_output=True, check=True, timeout=15).stdout + + with tempfile.TemporaryDirectory(prefix="plume-preparation-", dir=args.output) as temporary: + sandbox = Path(temporary) + unrelated = sandbox / "unrelated-working-directory" + unrelated.mkdir() + + def clone(name): + checkout = sandbox / name + # Local object sharing only: no fetch or network access, and no writes + # to the original checkout's index, HEAD, or worktree registration. + command([git_executable, "clone", "--shared", "--no-checkout", str(source), str(checkout)], unrelated) + git(checkout, "checkout", "--detach", PIN) + if (checkout / "plume_d3d12.cpp").read_bytes() != public_blob: + raise AssertionError("Isolated checkout does not contain the actual pinned public source") + return checkout + + def prepare(checkout, *options, expect=0, oracle=None): + return command([sys.executable, str(preparer), str(checkout), *options], unrelated, + expect=expect, oracle=oracle) + + expected = clone("independent-real-git-patch") + git(expected, "apply", "--check", str(patch)) + git(expected, "apply", str(patch)) + expected_sha = digest(expected / "plume_d3d12.cpp") + checks.append("actual pinned public source and independently Git-applied patch") + + checkout = clone("preparer-under-test") + before = digest(checkout / "plume_d3d12.cpp") + prepare(checkout, "--check", expect=2, oracle="Native Plume fixes are missing or incomplete") + if digest(checkout / "plume_d3d12.cpp") != before: + raise AssertionError("Missing-patch --check modified source") + checks.append("missing patch rejected without writes") + + subdirectory = checkout / "nested" + subdirectory.mkdir() + for options in ((), ("--check",)): + prepare(subdirectory, *options, expect=2, oracle="must be the repository root, not a subdirectory") + if digest(checkout / "plume_d3d12.cpp") != before: + raise AssertionError("Subdirectory rejection modified source") + checks.append("apply/check reject repository subdirectory with exact oracle") + + prepare(checkout, oracle="Applied native fence/resize/command-allocation fixes") + if digest(checkout / "plume_d3d12.cpp") != expected_sha: + raise AssertionError("Preparer result differs from independently applied real public patch") + prepare(checkout, "--check", oracle="PASS: public Plume") + prepare(checkout, oracle="PASS: public Plume") + if digest(checkout / "plume_d3d12.cpp") != expected_sha: + raise AssertionError("Repeated preparation changed the patched source") + checks.append("apply/check/repeat from unrelated cwd preserve exact patch result") + + with (checkout / "plume_d3d12.cpp").open("a") as stream: + stream.write("\n// Fixture unrelated local edit outside all patched hunks.\n") + local_sha = digest(checkout / "plume_d3d12.cpp") + prepare(checkout, "--check", oracle="PASS: public Plume") + prepare(checkout, oracle="PASS: public Plume") + if digest(checkout / "plume_d3d12.cpp") != local_sha: + raise AssertionError("Preparation altered an unrelated local edit") + checks.append("unrelated local edits survive check and repeat") + + # Change just one mandatory factory guard; --check must reject a partial patch. + text = (checkout / "plume_d3d12.cpp").read_text() + old = "if (semaphore->d3d == nullptr) {" + if text.count(old) != 1: + raise AssertionError("Actual patched factory guard was not found exactly once") + (checkout / "plume_d3d12.cpp").write_text(text.replace(old, "if (false) {")) + prepare(checkout, "--check", expect=2, oracle="Native Plume fixes are missing or incomplete") + checks.append("partial command-creation patch rejected") + + wrong = clone("wrong-revision") + tree = git(wrong, "rev-parse", f"{PIN}^{{tree}}").stdout.strip() + commit = command([git_executable, "-C", str(wrong), "-c", "user.name=Fixture", + "-c", "user.email=fixture@example.invalid", "commit-tree", tree, + "-p", PIN, "-m", "Synthetic wrong revision; public source is unchanged"], unrelated).stdout.strip() + git(wrong, "checkout", "--detach", commit) + prepare(wrong, expect=2, oracle=f"Expected public Plume {PIN}, got {commit}") + checks.append("wrong revision rejected even with identical public tree") + + noop = clone("successful-no-op-git") + # Execute the actual preparer while making git apply falsely return zero. + # All repository/revision commands still run real Git. Test both a false + # already-applied report and a no-op apply followed by false verification. + wrapper = '''import runpy, subprocess, sys +real_run = subprocess.run +reverse_calls = 0 +mode = sys.argv[1] +preparer, checkout = sys.argv[2:4] +def fake_run(argv, *args, **kwargs): + global reverse_calls + if isinstance(argv, list) and argv[0] == 'git' and 'apply' in argv: + if '--reverse' in argv: + reverse_calls += 1 + if mode == 'apply' and reverse_calls == 1: + return subprocess.CompletedProcess(argv, 1, '', 'fixture: patch not applied') + return subprocess.CompletedProcess(argv, 0, '', '') + return real_run(argv, *args, **kwargs) +subprocess.run = fake_run +sys.argv = [preparer, checkout] +runpy.run_path(preparer, run_name='__main__') +''' + for mode in ("already", "apply"): + command([sys.executable, "-c", wrapper, mode, str(preparer), str(noop)], unrelated, + expect=2, oracle="Native Plume source verification failed: missing patched hunk") + if digest(noop / "plume_d3d12.cpp") != before: + raise AssertionError("No-op Git fixture unexpectedly modified source") + checks.append("false Git success rejected for already-applied and no-op apply paths") + + after = { + "head": git(source, "rev-parse", "HEAD").stdout, + "status": git(source, "status", "--porcelain=v1", "--untracked-files=all").stdout, + "source_sha256": digest(source / "plume_d3d12.cpp"), + } + if original != after: + raise AssertionError("Original Plume checkout changed during the offline fixture") + checks.append("original checkout HEAD/status/source preserved") + report = {"passed": True, "pin": PIN, "actual_public_source": True, "offline": True, + "preparer_sha256": digest(preparer), "patch_sha256": digest(patch), + "public_source_sha256": hashlib.sha256(public_blob).hexdigest(), + "patched_source_sha256": expected_sha, "checks": checks} + (args.output / "verification.json").write_text(json.dumps(report, indent=2) + "\n") + print(f"PASS: {len(checks)} actual-public-source Plume preparation checks; original checkout unchanged") + + +if __name__ == "__main__": + main() From 5145a21690397e7f8addc459ec7a4ab4e4cd707c Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:15:06 +0200 Subject: [PATCH 12/40] Validate GPU completion before retiring native resources Keep frame-ring allocation transactional and reject invalid D3D12 pipeline handles. A void backend wait now permits retirement only after verified fence completion. Validation: seven isolated production-body fixtures passed against the staged source snapshot; native Plume fence/resize fixtures also passed. --- rexlego/src/gpu_native/device.cpp | 118 +++++---- tools/gpu-aot/CMakeLists.txt | 23 ++ tools/gpu-aot/test_completion_callbacks.py | 8 +- tools/gpu-aot/test_cpu_resource_wait.py | 7 +- tools/gpu-aot/test_draw_upload.py | 13 +- tools/gpu-aot/test_frame_ring_transaction.py | 177 ++++++++++++++ tools/gpu-aot/test_pipeline_failures.py | 244 +++++++++++++++++++ tools/gpu-aot/test_query_retirement.py | 7 +- tools/gpu-aot/test_resize_fences.py | 16 +- tools/gpu-aot/test_submission_waits.py | 208 ++++++++++++++++ 10 files changed, 758 insertions(+), 63 deletions(-) create mode 100644 tools/gpu-aot/test_frame_ring_transaction.py create mode 100644 tools/gpu-aot/test_pipeline_failures.py create mode 100644 tools/gpu-aot/test_submission_waits.py diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp index 5e06a41..7c10b29 100644 --- a/rexlego/src/gpu_native/device.cpp +++ b/rexlego/src/gpu_native/device.cpp @@ -271,7 +271,10 @@ bool CreatePipelineLayout(State& state) { plume::RenderShaderStageFlag::PIXEL); layout.end(); state.pipeline_layout = layout.create(state.device.get()); - if (!state.pipeline_layout) { + if (!state.pipeline_layout || + !static_cast(state.pipeline_layout.get())->rootSignature) { + state.pipeline_layout.reset(); + REXLOG_ERROR("Native GPU: D3D12 rejected the root signature"); return false; } @@ -321,6 +324,10 @@ bool CreatePresentPipeline(State& state) { desc.renderTargetBlend[0] = plume::RenderBlendDesc::Copy(); REXLOG_INFO("Native GPU init: creating present pipeline"); state.copy_pipeline = state.device->createGraphicsPipeline(desc); + if (state.copy_pipeline && + !static_cast(state.copy_pipeline.get())->d3d) { + state.copy_pipeline.reset(); + } REXLOG_INFO("Native GPU init: present pipeline {}", state.copy_pipeline ? "ready" : "failed"); return state.copy_pipeline != nullptr; @@ -329,33 +336,46 @@ bool CreatePresentPipeline(State& state) { bool CreateFrameRing(State& state) { REXLOG_INFO("Native GPU init: creating frame ring"); const u32 texture_count = state.swap_chain->getTextureCount(); - state.framebuffers.clear(); - state.render_semaphores.clear(); - state.framebuffers.reserve(texture_count); - state.render_semaphores.reserve(texture_count); + if (!texture_count) return false; + // Resize has already discarded the old swap-chain framebuffers. Publish a + // new ring only when every allocation succeeds, so an empty framebuffer + // list remains the retry marker after a partial failure. + decltype(state.framebuffers) framebuffers; + decltype(state.render_semaphores) render_semaphores; + decltype(state.command_lists) command_lists; + decltype(state.frame_fences) frame_fences; + decltype(state.acquire_semaphores) acquire_semaphores; + framebuffers.reserve(texture_count); + render_semaphores.reserve(texture_count); for (u32 i = 0; i < texture_count; ++i) { const plume::RenderTexture* attachment = state.swap_chain->getTexture(i); - state.framebuffers.emplace_back(state.device->createFramebuffer( + framebuffers.emplace_back(state.device->createFramebuffer( plume::RenderFramebufferDesc(&attachment, 1))); - state.render_semaphores.emplace_back( + render_semaphores.emplace_back( state.device->createCommandSemaphore()); - if (!state.framebuffers.back() || !state.render_semaphores.back()) { + if (!framebuffers.back() || !render_semaphores.back()) { return false; } } for (u32 i = 0; i < State::kFramesInFlight; ++i) { - state.command_lists[i] = state.queue->createCommandList(); - state.frame_fences[i] = state.device->createCommandFence(); - state.acquire_semaphores[i] = state.device->createCommandSemaphore(); + command_lists[i] = state.queue->createCommandList(); + frame_fences[i] = state.device->createCommandFence(); + acquire_semaphores[i] = state.device->createCommandSemaphore(); + if (!command_lists[i] || !frame_fences[i] || !acquire_semaphores[i]) { + return false; + } + } + state.framebuffers = std::move(framebuffers); + state.render_semaphores = std::move(render_semaphores); + state.command_lists = std::move(command_lists); + state.frame_fences = std::move(frame_fences); + state.acquire_semaphores = std::move(acquire_semaphores); + for (u32 i = 0; i < State::kFramesInFlight; ++i) { state.frame_submitted[i] = false; state.retired_resources[i].clear(); state.retired_descriptors.CompleteFrame(i, [&](u32 index) { state.texture_slots[index] = false; }); - if (!state.command_lists[i] || !state.frame_fences[i] || - !state.acquire_semaphores[i]) { - return false; - } } state.frame_slot = 0; return true; @@ -495,29 +515,37 @@ void MarkSubmissionLocked(State& state, u32 slot) { state.slot_submission[slot] = ++state.last_submission; } +bool WaitForSubmissionLocked(State& state, u32 slot) { + if (!state.frame_submitted[slot]) return true; + if (state.backend == Backend::kD3D12) { + auto* fence = static_cast(state.frame_fences[slot].get()); + if (fence->d3d->GetCompletedValue() == UINT64_MAX) { + REXLOG_ERROR("Native GPU: cannot wait for submission after device removal"); + return false; + } + } + state.queue->waitForCommandFence(state.frame_fences[slot].get()); + if (state.backend == Backend::kD3D12) { + auto* fence = static_cast(state.frame_fences[slot].get()); + const u64 completed = fence->d3d->GetCompletedValue(); + if (completed == UINT64_MAX || completed < fence->fenceValue - 1) { + REXLOG_ERROR("Native GPU: submission GPU wait failed"); + return false; + } + } + // A void Plume wait can return on device removal or an event failure. Only + // a verified completion permits resource retirement or allocator reuse. + state.completed_submission = std::max(state.completed_submission, + state.slot_submission[slot]); + state.frame_submitted[slot] = false; + return true; +} + bool WaitForSubmittedFramesLocked(State& state) { // DXGI's frame-latency object is a present throttle, not a GPU completion // fence. Rebuilding/releasing the ring requires all submitted GPU users. for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) { - if (!state.frame_submitted[slot]) continue; - if (state.backend == Backend::kD3D12) { - auto* fence = static_cast(state.frame_fences[slot].get()); - if (fence->d3d->GetCompletedValue() == UINT64_MAX) { - REXLOG_ERROR("Native GPU: cannot rebuild frame ring after device removal"); - return false; - } - } - state.queue->waitForCommandFence(state.frame_fences[slot].get()); - if (state.backend == Backend::kD3D12) { - auto* fence = static_cast(state.frame_fences[slot].get()); - const u64 completed = fence->d3d->GetCompletedValue(); - if (completed == UINT64_MAX || completed < fence->fenceValue - 1) { - REXLOG_ERROR("Native GPU: frame-ring GPU wait failed"); - return false; - } - } - state.completed_submission = std::max(state.completed_submission, state.slot_submission[slot]); - state.frame_submitted[slot] = false; + if (!WaitForSubmissionLocked(state, slot)) return false; } RefreshCompletedSubmissionsLocked(state); return true; @@ -546,10 +574,8 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) { const u32 slot = state.frame_slot; if (state.frame_submitted[slot]) { const auto wait_start = std::chrono::steady_clock::now(); - state.queue->waitForCommandFence(state.frame_fences[slot].get()); + if (!WaitForSubmissionLocked(state, slot)) return nullptr; RefreshCompletedSubmissionsLocked(state); - if (state.backend != Backend::kD3D12) - state.completed_submission = std::max(state.completed_submission, state.slot_submission[slot]); const auto wait_ms = std::chrono::duration( std::chrono::steady_clock::now() - wait_start) .count(); @@ -561,7 +587,6 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) { REXLOG_WARN("Native GPU: frame-slot {} GPU wait took {:.2f} ms", slot, wait_ms); } - state.frame_submitted[slot] = false; SaveSnapshots(state.snapshots[slot]); } state.retired_resources[slot].clear(); @@ -851,11 +876,8 @@ bool HostDevice::Synchronize(SyncReason reason) { if (!state.frame_submitted[slot]) { continue; } - state.queue->waitForCommandFence(state.frame_fences[slot].get()); + if (!WaitForSubmissionLocked(state, slot)) return false; RefreshCompletedSubmissionsLocked(state); - if (state.backend != Backend::kD3D12) - state.completed_submission = std::max(state.completed_submission, state.slot_submission[slot]); - state.frame_submitted[slot] = false; state.retired_resources[slot].clear(); state.retired_descriptors.CompleteFrame(slot, [&](u32 index) { state.texture_slots[index] = false; }); @@ -1015,8 +1037,12 @@ bool HostDevice::ResolveHdrColor(plume::RenderTexture* source, u32 descriptor_in desc.renderTargetFormat[0] = formats[format_index]; desc.renderTargetBlend[0] = plume::RenderBlendDesc::Copy(); pipeline = state.device->createGraphicsPipeline(desc); - if (!pipeline || !static_cast(pipeline.get())->d3d) + if (!pipeline || !static_cast(pipeline.get())->d3d) { + pipeline.reset(); + REXLOG_ERROR("Native GPU: D3D12 rejected the color resolve pipeline (format={})", + format_index); return false; + } } plume::RenderFramebufferDesc desc; const plume::RenderTexture* attachments[] = {destination}; @@ -1092,8 +1118,12 @@ bool HostDevice::TransferDepthAlias(plume::RenderTexture* source, u32 descriptor desc.renderTargetFormat[0] = plume::RenderFormat::R8G8B8A8_UNORM; desc.renderTargetBlend[0] = plume::RenderBlendDesc::Copy(); pipeline = state.device->createGraphicsPipeline(desc); - if (!pipeline || !static_cast(pipeline.get())->d3d) + if (!pipeline || !static_cast(pipeline.get())->d3d) { + pipeline.reset(); + REXLOG_ERROR("Native GPU: D3D12 rejected the depth alias pipeline (restore={})", + restore); return false; + } } plume::RenderFramebufferDesc desc; const plume::RenderTexture* attachments[] = {destination}; diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index 3a30bd1..0ddfc7f 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -39,6 +39,29 @@ if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") endif() endif() +# Compile production bodies with deterministic allocation and fence failures. +# These use GNU-style compiler arguments and require no SDK or game assets. +if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") + find_package(Python3 COMPONENTS Interpreter QUIET) + if(Python3_Interpreter_FOUND) + foreach(check IN ITEMS frame_ring_transaction submission_waits pipeline_failures + completion_callbacks cpu_resource_wait draw_upload query_retirement) + add_test(NAME ${check} + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_${check}.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/${check} + --compiler ${CMAKE_CXX_COMPILER}) + set_tests_properties(${check} PROPERTIES TIMEOUT 60) + endforeach() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/plume_d3d12.cpp") + add_test(NAME resize_fences + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_resize_fences.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/resize_fences + --compiler ${CMAKE_CXX_COMPILER}) + set_tests_properties(resize_fences PROPERTIES TIMEOUT 60) + endif() + endif() +endif() + # These fixtures exercise the real extraction CLI with synthetic inputs. if(BUILD_TESTING) find_package(Python3 COMPONENTS Interpreter QUIET) diff --git a/tools/gpu-aot/test_completion_callbacks.py b/tools/gpu-aot/test_completion_callbacks.py index f6a7fe0..d0d6c19 100644 --- a/tools/gpu-aot/test_completion_callbacks.py +++ b/tools/gpu-aot/test_completion_callbacks.py @@ -4,7 +4,7 @@ from pathlib import Path import subprocess -p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args() +p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);p.add_argument('--compiler',default='clang++');a=p.parse_args() root=Path(__file__).resolve().parents[2];a.output.mkdir(parents=True,exist_ok=True) source=(root/'rexlego/src/gpu_native/device.cpp').read_text() def body(sig): @@ -33,7 +33,7 @@ def body(sig): struct D3D12CommandFence {NativeFence native;NativeFence* d3d=&native;u64 fenceValue=1;}; } struct State { - static constexpr u32 kFramesInFlight=3; + static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight; Backend backend=Backend::kD3D12; std::array frame_submitted{}; std::array,3> frame_fences; @@ -97,8 +97,8 @@ def body(sig): } ''') exe=a.output/'completion-callbacks.exe' -subprocess.run(['clang++','-std=c++20','-DNOMINMAX','-I'+str(root/'rexlego/src'),str(h),'-o',str(exe)],check=True) -subprocess.run([str(exe.resolve())],check=True) +subprocess.run([a.compiler,'-std=c++20','-DNOMINMAX','-I'+str(root/'rexlego/src'),str(h),'-o',str(exe)],check=True, timeout=45) +subprocess.run([str(exe.resolve())],check=True, timeout=45) assert source.count('MarkSubmissionLocked(state, slot);')==4 hook=(root/'rexlego/src/gpu_native/hooks_device.cpp').read_text() insert=hook[hook.index('void InsertCallbackHook('):hook.index('} // namespace',hook.index('void InsertCallbackHook('))] diff --git a/tools/gpu-aot/test_cpu_resource_wait.py b/tools/gpu-aot/test_cpu_resource_wait.py index 5c18f23..e0e1d0c 100644 --- a/tools/gpu-aot/test_cpu_resource_wait.py +++ b/tools/gpu-aot/test_cpu_resource_wait.py @@ -4,7 +4,10 @@ import os from pathlib import Path import subprocess -p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args() +p=argparse.ArgumentParser(description=__doc__) +p.add_argument('output',type=Path) +p.add_argument('--compiler', default='clang++') +a=p.parse_args() root=Path(__file__).resolve().parents[2];a.output.mkdir(parents=True,exist_ok=True) def body(path,sig): text=(root/path).read_text();start=text.index(sig);end=text.index('{',start)+1;depth=1 @@ -67,7 +70,7 @@ def body(path,sig): } ''') exe=a.output/'cpu-resource-wait.exe' -subprocess.run(['clang++','-std=c++20',str(h),'-o',str(exe)],check=True) +subprocess.run([a.compiler,'-std=c++20',str(h),'-o',str(exe)],check=True) for enabled in (False,True): env=os.environ.copy();env.pop('LEGO_NATIVE_ASYNC_CPU_RESOURCES',None) if enabled:env['LEGO_NATIVE_ASYNC_CPU_RESOURCES']='1' diff --git a/tools/gpu-aot/test_draw_upload.py b/tools/gpu-aot/test_draw_upload.py index 9952358..c9a9c4c 100644 --- a/tools/gpu-aot/test_draw_upload.py +++ b/tools/gpu-aot/test_draw_upload.py @@ -4,7 +4,7 @@ from pathlib import Path import subprocess -p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);a=p.parse_args() +p=argparse.ArgumentParser(description=__doc__);p.add_argument('output',type=Path);p.add_argument('--compiler',default='clang++');a=p.parse_args() root=Path(__file__).resolve().parents[2];a.output.mkdir(parents=True,exist_ok=True) source=(root/'rexlego/src/gpu_native/device.cpp').read_text() def function(signature): @@ -53,7 +53,7 @@ def function(signature): struct DescriptorRetirement { void CompleteFrame(u32,auto callback){} }; enum class Backend {kD3D12,kVulkan}; struct State { - static constexpr u32 kFramesInFlight=3; + static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight; std::unique_ptr device=std::make_unique(); std::unique_ptr queue=std::make_unique(); std::unique_ptr swap_chain=std::make_unique(); @@ -72,6 +72,11 @@ def function(signature): }; void SaveSnapshots(std::vector&){} void RefreshCompletedSubmissionsLocked(State&){} +bool WaitForSubmissionLocked(State& state,u32 slot){ + state.queue->waitForCommandFence(state.frame_fences[slot].get()); + state.completed_submission=std::max(state.completed_submission,state.slot_submission[slot]); + state.frame_submitted[slot]=false;return true; +} void InitializeNullTextures(State&,plume::RenderCommandList*){} bool RebuildSwapChain(State&){return true;} bool timing_enabled=true; @@ -113,8 +118,8 @@ def function(signature): } ''') exe=a.output/'draw-upload-test.exe' -subprocess.run(['clang++','-std=c++20','-DNOMINMAX',str(h),'-o',str(exe)],check=True) -subprocess.run([str(exe.resolve())],check=True) +subprocess.run([a.compiler,'-std=c++20','-DNOMINMAX',str(h),'-o',str(exe)],check=True,timeout=45) +subprocess.run([str(exe.resolve())],check=True,timeout=10) (a.output/'verification.json').write_text(json.dumps({'actual_production_functions':True,'fake_driver':True, 'checks':['256-byte alignment','550 immutable draws','page growth','independent slots','reset after fence wait','oversize','allocation failure','shutdown'], 'passed':True},indent=2)) diff --git a/tools/gpu-aot/test_frame_ring_transaction.py b/tools/gpu-aot/test_frame_ring_transaction.py new file mode 100644 index 0000000..0b34696 --- /dev/null +++ b/tools/gpu-aot/test_frame_ring_transaction.py @@ -0,0 +1,177 @@ +"""Inject every frame-ring allocation failure into production creation/retry bodies.""" +import argparse +from pathlib import Path +import subprocess + + +def function(source, signature): + start = source.index(signature) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + production = (root / "rexlego/src/gpu_native/device.cpp").read_text() + bodies = "\n".join(function(production, signature) for signature in ( + "bool CreateFrameRing(", "void RefreshCompletedSubmissionsLocked(", + "bool WaitForSubmissionLocked(", "bool WaitForSubmittedFramesLocked(", + "bool RebuildSwapChain(", "plume::RenderCommandList* BeginFrameCommandsLocked(", + )) + harness = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include "gpu_native/descriptor_retirement.h" +using u32=std::uint32_t;using u64=std::uint64_t; +using legodimensions::gpu_native::DescriptorRetirement; +#define REXLOG_INFO(...) ((void)0) +#define REXLOG_ERROR(...) ((void)0) +#define REXLOG_WARN(...) ((void)0) +enum class Backend {kD3D12,kVulkan}; +struct Tracked {static inline int live=0;Tracked(){++live;}virtual ~Tracked(){--live;}}; +struct Allocation { + int attempts=0,fail_at=0; + templatestd::unique_ptr Make(){ + if(++attempts==fail_at)return {};return std::make_unique(); + } +} allocation; +namespace plume { +struct RenderTexture {}; +struct RenderFramebuffer:Tracked {}; +struct RenderCommandSemaphore:Tracked {}; +struct NativeFence {u64 completed=0;u64 GetCompletedValue(){return completed;}}; +struct RenderCommandFence:Tracked {}; +struct D3D12CommandFence:RenderCommandFence { + NativeFence native;NativeFence* d3d=&native;u64 fenceValue=2; +}; +struct RenderCommandList:Tracked {int begins=0;void begin(){++begins;}}; +struct RenderFramebufferDesc {RenderFramebufferDesc(const RenderTexture**,u32 count){assert(count==1);}}; +struct Device { + auto createFramebuffer(RenderFramebufferDesc){return allocation.Make();} + auto createCommandSemaphore(){return allocation.Make();} + auto createCommandFence(){return allocation.Make();} +}; +struct Queue { + int waits=0; + auto createCommandList(){return allocation.Make();} + void waitForCommandFence(RenderCommandFence* base){ + ++waits;auto& fence=*static_cast(base); + fence.native.completed=fence.fenceValue-1; + } +}; +struct Swap { + std::array textures;u32 count=3;int resizes=0;bool resize_pending=true; + u32 getTextureCount(){return count;} + RenderTexture* getTexture(u32 image){return &textures.at(image);} + bool needsResize(){return resize_pending;} + bool resize(){++resizes;resize_pending=false;return true;} + bool isEmpty(){return count==0;} +}; +} +struct State { + static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight;Backend backend=Backend::kD3D12; + std::unique_ptr device=std::make_unique(); + std::unique_ptr queue=std::make_unique(); + std::unique_ptr swap_chain=std::make_unique(); + std::vector> framebuffers; + std::vector> render_semaphores; + std::array,3> command_lists; + std::array,3> frame_fences; + std::array,3> acquire_semaphores; + std::array frame_submitted{};std::array slot_submission{1,2,3}; + u64 completed_submission=0; + std::array>,3> retired_resources; + DescriptorRetirement retired_descriptors{8};std::array texture_slots{true,true,true,true,true,true,true,true}; + std::array,3> snapshots; + bool command_list_open=false;u32 frame_slot=2,draw_upload_page=7,long_wait_log_count=0; + u64 draw_upload_offset=1024,frame_slot_wait_calls=0;double frame_slot_wait_ms=0; + State(){for(u32 i=0;i<3;++i){ + framebuffers.push_back(std::make_unique()); + render_semaphores.push_back(std::make_unique()); + command_lists[i]=std::make_unique(); + frame_fences[i]=std::make_unique(); + acquire_semaphores[i]=std::make_unique(); + }} +}; +bool NativeTextureTimingEnabled(){return true;} +void SaveSnapshots(std::vector& snapshots){snapshots.clear();} +void InitializeNullTextures(State&,plume::RenderCommandList*){} +''' + tests = r''' +int main(){ + constexpr int kAllocations=15; // Three images and three submission slots. + for(int failure=1;failure<=kAllocations;++failure){ + { + State state;const int live=Tracked::live; + auto* old_commands=state.command_lists[2].get(); + auto* old_fence=state.frame_fences[2].get(); + auto* old_acquire=state.acquire_semaphores[2].get(); + auto resource=std::make_shared(1);std::weak_ptr retained=resource; + state.retired_resources[2].push_back(std::move(resource)); + assert(!state.retired_descriptors.Retire(3,1u<<2)); + allocation={0,failure};assert(!CreateFrameRing(state)); + assert(state.framebuffers.size()==3&&state.render_semaphores.size()==3); + assert(state.command_lists[2].get()==old_commands&&state.frame_fences[2].get()==old_fence); + assert(state.acquire_semaphores[2].get()==old_acquire&&state.frame_slot==2); + assert(!retained.expired()&&state.texture_slots[3]&&Tracked::live==live); + } + assert(Tracked::live==0); + { + State state;auto* old_commands=state.command_lists[2].get(); + auto* old_fence=state.frame_fences[2].get(); + state.frame_submitted[2]=true; + // Failed resize creation must leave the empty retry marker. Old allocator + // and fence arrays stay intact, and temporary allocations are destroyed. + allocation={0,failure};assert(!BeginFrameCommandsLocked(state)); + assert(state.framebuffers.empty()&&state.render_semaphores.empty()); + assert(state.command_lists[2].get()==old_commands&&state.frame_fences[2].get()==old_fence); + assert(!state.command_list_open&&state.frame_slot==2&&state.queue->waits==1); + assert(state.swap_chain->resizes==1&&!state.swap_chain->needsResize()); + assert(Tracked::live==9); // Old command lists, fences and acquire semaphores. + // A repeated failure retries even though DXGI no longer requests resize. + allocation={0,failure};assert(!BeginFrameCommandsLocked(state)); + assert(state.swap_chain->resizes==2&&Tracked::live==9); + allocation={0,0};auto* commands=BeginFrameCommandsLocked(state); + assert(commands&&commands==state.command_lists[0].get()&&commands->begins==1); + assert(state.framebuffers.size()==3&&state.render_semaphores.size()==3); + assert(state.command_list_open&&state.frame_slot==0&&state.swap_chain->resizes==3); + assert(state.queue->waits==1&&Tracked::live==15); + } + assert(Tracked::live==0); + } + { + State state;auto* old_commands=state.command_lists[2].get(); + state.swap_chain->count=0;allocation={0,0};assert(!CreateFrameRing(state)); + assert(allocation.attempts==0&&state.command_lists[2].get()==old_commands); + assert(state.framebuffers.size()==3&&state.frame_slot==2); + } + assert(Tracked::live==0); + std::cout<<"PASS: transactional frame-ring creation and retry after every allocation failure.\n"; +} +''' + args.output.mkdir(parents=True, exist_ok=True) + source = args.output.resolve() / "frame-ring-transaction.cpp" + exe = args.output.resolve() / "frame-ring-transaction.exe" + source.write_text(harness + bodies + tests) + subprocess.run([args.compiler, "-std=c++20", "-UNDEBUG", + "-I" + str(root / "rexlego/src"), str(source), "-o", str(exe)], + check=True, timeout=45) + subprocess.run([str(exe)], check=True, timeout=10) + + +if __name__ == "__main__": + main() diff --git a/tools/gpu-aot/test_pipeline_failures.py b/tools/gpu-aot/test_pipeline_failures.py new file mode 100644 index 0000000..5a97da0 --- /dev/null +++ b/tools/gpu-aot/test_pipeline_failures.py @@ -0,0 +1,244 @@ +"""Exercise native pipeline bootstrap and resolve retries with real function bodies.""" +import argparse +from pathlib import Path +import subprocess + + +def function(source, signature): + start = source.index(signature) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + production = (root / "rexlego/src/gpu_native/device.cpp").read_text() + bodies = "\n".join(function(production, signature) for signature in ( + "bool CreatePipelineLayout(", "bool CreatePresentPipeline(", + "bool HostDevice::ResolveHdrColor(", "bool HostDevice::TransferDepthAlias(", + )) + harness = r''' +#include +#include +#include +#include +#include +#include +#include +using u32=std::uint32_t;using i32=std::int32_t; +#define REXLOG_INFO(...) ((void)0) +#define REXLOG_ERROR(...) ((void)0) +enum class Failure {None,NullWrapper,NullHandle}; +namespace plume { +enum class RenderShaderFormat {DXIL}; +enum class RenderComparisonFunction {ALWAYS}; +enum class RenderPrimitiveTopology {TRIANGLE_LIST}; +enum class RenderCullMode {NONE}; +enum class RenderFillMode {SOLID}; +enum class RenderRootDescriptorType {CONSTANT_BUFFER}; +enum class RenderShaderStageFlag {PIXEL}; +enum class RenderFilter {LINEAR}; +enum class RenderMipmapMode {LINEAR}; +enum class RenderTextureAddressMode {CLAMP}; +enum class RenderFormat {UNKNOWN,B8G8R8A8_UNORM,R16G16B16A16_UNORM, + R32G32B32A32_FLOAT,R16G16_UNORM,R32_FLOAT,D32_FLOAT_S8_UINT,R8G8B8A8_UNORM}; +enum class RenderTextureLayout {SHADER_READ,COLOR_WRITE,DEPTH_WRITE}; +enum class RenderBarrierStage {GRAPHICS}; +struct RenderPipelineLayout {virtual ~RenderPipelineLayout()=default;}; +struct D3D12PipelineLayout:RenderPipelineLayout { + int handle=1;int* rootSignature=nullptr; + explicit D3D12PipelineLayout(bool valid){if(valid)rootSignature=&handle;} +}; +struct RenderPipeline {virtual ~RenderPipeline()=default;}; +struct D3D12GraphicsPipeline:RenderPipeline { + static inline int live=0;int handle=1;int* d3d=nullptr; + explicit D3D12GraphicsPipeline(bool valid){++live;if(valid)d3d=&handle;} + ~D3D12GraphicsPipeline(){--live;} +}; +struct RenderShader {}; +struct RenderSampler {}; +struct RenderTexture {}; +struct RenderFramebuffer {}; +struct RenderSamplerDesc { + RenderFilter minFilter{},magFilter{};RenderMipmapMode mipmapMode{}; + RenderTextureAddressMode addressU{},addressV{},addressW{}; +}; +struct RenderBlendDesc {static RenderBlendDesc Copy(){return {};}}; +struct RenderGraphicsPipelineDesc { + RenderPipelineLayout* pipelineLayout=nullptr;RenderShader* vertexShader=nullptr; + RenderShader* pixelShader=nullptr;RenderComparisonFunction depthFunction{}; + bool depthEnabled=false,depthWriteEnabled=false;RenderPrimitiveTopology primitiveTopology{}; + RenderCullMode cullMode{};RenderFillMode fillMode{};u32 renderTargetCount=0; + std::array renderTargetFormat{};std::array renderTargetBlend{}; + RenderFormat depthTargetFormat{}; +}; +struct RenderFramebufferDesc { + const RenderTexture** colorAttachments=nullptr;u32 colorAttachmentsCount=0; + RenderTexture* depthAttachment=nullptr; +}; +struct RenderTextureBarrier { + RenderTextureBarrier(RenderTexture*,RenderTextureLayout){} +}; +struct RenderViewport {RenderViewport(float,float,float,float){}}; +struct RenderRect {RenderRect(u32,u32,u32,u32){}}; +struct RenderDescriptorSet { + int sampler_sets=0; + void setSampler(u32,RenderSampler* sampler){assert(sampler);++sampler_sets;} +}; +struct Device { + Failure layout_failure=Failure::None,pipeline_failure=Failure::None; + int layouts=0,pipelines=0,shaders=0,samplers=0,framebuffers=0,fail_shader_at=0; + std::unique_ptr createPipelineLayout(){ + ++layouts;if(layout_failure==Failure::NullWrapper)return {}; + return std::make_unique(layout_failure!=Failure::NullHandle); + } + std::unique_ptr createGraphicsPipeline(const RenderGraphicsPipelineDesc& desc){ + assert(desc.pipelineLayout&&static_cast(desc.pipelineLayout)->rootSignature); + assert(desc.vertexShader&&desc.pixelShader);++pipelines; + if(pipeline_failure==Failure::NullWrapper)return {}; + return std::make_unique(pipeline_failure!=Failure::NullHandle); + } + auto createShader(const void*,std::size_t,const char*,RenderShaderFormat){ + ++shaders;return shaders==fail_shader_at?std::unique_ptr{}: + std::make_unique(); + } + auto createSampler(const RenderSamplerDesc&){++samplers;return std::make_unique();} + auto createFramebuffer(const RenderFramebufferDesc&){++framebuffers;return std::make_unique();} +}; +struct RenderDescriptorSetBuilder { + void begin(){}void addTexture(u32,u32){}void addSampler(u32,u32){}void end(bool,u32){} + auto create(Device*){return std::make_unique();} +}; +struct RenderPipelineLayoutBuilder { + void begin(bool,bool){}void addDescriptorSet(const RenderDescriptorSetBuilder&){} + void addRootDescriptor(u32,u32,RenderRootDescriptorType){} + void addPushConstant(u32,u32,u32,RenderShaderStageFlag){}void end(){} + auto create(Device* device){return device->createPipelineLayout();} +}; +struct Commands { + int mutations=0,draws=0,pipeline_binds=0; + void barriers(RenderBarrierStage,const RenderTextureBarrier*,u32){++mutations;} + void barriers(RenderBarrierStage,const RenderTextureBarrier&){++mutations;} + void setFramebuffer(RenderFramebuffer*){++mutations;} + void setViewports(RenderViewport){++mutations;}void setScissors(RenderRect){++mutations;} + void setGraphicsPipelineLayout(RenderPipelineLayout* layout){ + assert(layout&&static_cast(layout)->rootSignature);++mutations; + } + void setGraphicsDescriptorSet(RenderDescriptorSet*,u32){++mutations;} + void setPipeline(RenderPipeline* pipeline){ + assert(pipeline&&static_cast(pipeline)->d3d); + ++pipeline_binds;++mutations; + } + void setGraphicsPushConstants(u32,const void*){++mutations;} + void drawInstanced(u32,u32,u32,u32){++draws;++mutations;} +}; +} +constexpr u32 kBindlessTextureCount=65536,kBindlessSamplerCount=1024,kFirstTextureSlot=3; +constexpr unsigned char g_copy_vs_dxil[]{1},g_copy_color_ps_dxil[]{2}, + g_resolve_color_ps_dxil[]{3},g_depth_pack_ps_dxil[]{4},g_depth_restore_ps_dxil[]{5}; +struct ResolveRegion {u32 x,y,width,height,left,top;}; +enum class ColorResolveDestination {Unorm16,Float32,Unorm16x2,kDepthFloat32}; +struct State { + std::unique_ptr device=std::make_unique(); + std::unique_ptr pipeline_layout; + std::unique_ptr texture_descriptors,sampler_descriptors; + std::unique_ptr default_sampler; + std::unique_ptr copy_vertex_shader,copy_pixel_shader,resolve_pixel_shader; + std::unique_ptr copy_pipeline; + std::array,4> resolve_pipelines; + std::array,2> depth_alias_shaders; + std::array,2> depth_alias_pipelines; + std::vector texture_slots; +}; +std::unique_ptr g_state; +std::mutex g_mutex; +plume::Commands commands; +int retired=0; +struct HostDevice { + static auto LockRecording(){static std::recursive_mutex mutex;return std::unique_lock(mutex);} + static plume::Commands* BeginFrameCommands(){return &commands;} + static void RetireResource(std::shared_ptr resource){assert(resource);++retired;} + static bool ResolveHdrColor(plume::RenderTexture*,u32,plume::RenderTexture*,u32,u32,float, + ColorResolveDestination,bool,const ResolveRegion*); + static bool TransferDepthAlias(plume::RenderTexture*,u32,plume::RenderTexture*,u32,u32,bool); +}; +''' + tests = r''' +void Prepare(){ + g_state=std::make_unique();commands={};retired=0; + assert(CreatePipelineLayout(*g_state));assert(CreatePresentPipeline(*g_state)); +} +int main(){ + for(auto failure:{Failure::NullWrapper,Failure::NullHandle}){ + State state;state.device->layout_failure=failure; + assert(!CreatePipelineLayout(state));assert(!state.pipeline_layout); + assert(state.device->samplers==0&&state.device->pipelines==0); + state.device->layout_failure=Failure::None; + assert(CreatePipelineLayout(state));assert(state.default_sampler); + state.device->pipeline_failure=failure; + assert(!CreatePresentPipeline(state));assert(!state.copy_pipeline); + assert(plume::D3D12GraphicsPipeline::live==0); + state.device->pipeline_failure=Failure::None; + assert(CreatePresentPipeline(state));assert(state.copy_pipeline); + } + assert(plume::D3D12GraphicsPipeline::live==0); + for(int shader_failure:{1,2}){ + State state;assert(CreatePipelineLayout(state));state.device->fail_shader_at=shader_failure; + assert(!CreatePresentPipeline(state));assert(!state.copy_pipeline&&state.device->pipelines==0); + } + plume::RenderTexture source,destination; + for(auto failure:{Failure::NullWrapper,Failure::NullHandle}){ + for(u32 format=0;format<4;++format){ + Prepare();auto& state=*g_state;state.device->pipeline_failure=failure; + const auto call=[&]{return HostDevice::ResolveHdrColor(&source,1,&destination,128,64,1, + static_cast(format),true,nullptr);}; + for(int attempt=0;attempt<2;++attempt){ + assert(!call());assert(!state.resolve_pipelines[format]); + assert(commands.mutations==0&&commands.draws==0&&retired==0); + assert(state.device->pipelines==2+attempt&&state.device->framebuffers==0); + assert(plume::D3D12GraphicsPipeline::live==1); // Only the healthy present PSO. + } + state.device->pipeline_failure=Failure::None; + assert(call());assert(commands.draws==1&&commands.pipeline_binds==1&&retired==1); + assert(state.device->pipelines==4&&plume::D3D12GraphicsPipeline::live==2); + assert(call());assert(commands.draws==2&&state.device->pipelines==4&&retired==2); + g_state.reset();assert(plume::D3D12GraphicsPipeline::live==0); + } + for(bool restore:{false,true}){ + Prepare();auto& state=*g_state;state.device->pipeline_failure=failure; + const auto call=[&]{return HostDevice::TransferDepthAlias(&source,1,&destination,128,64,restore);}; + for(int attempt=0;attempt<2;++attempt){ + assert(!call());assert(!state.depth_alias_pipelines[restore]); + assert(commands.mutations==0&&commands.draws==0&&retired==0); + assert(state.device->pipelines==2+attempt&&state.device->framebuffers==0); + assert(plume::D3D12GraphicsPipeline::live==1); + } + state.device->pipeline_failure=Failure::None; + assert(call());assert(commands.draws==1&&commands.pipeline_binds==1&&retired==1); + assert(state.device->pipelines==4&&plume::D3D12GraphicsPipeline::live==2); + assert(call());assert(commands.draws==2&&state.device->pipelines==4&&retired==2); + g_state.reset();assert(plume::D3D12GraphicsPipeline::live==0); + } + } + std::cout<<"PASS: real pipeline bootstrap bodies reject failed handles; color/depth resolve failures retry without binding or drawing.\n"; +} +''' + args.output.mkdir(parents=True, exist_ok=True) + source = args.output.resolve() / "pipeline-failures.cpp" + exe = args.output.resolve() / "pipeline-failures.exe" + source.write_text(harness + bodies + tests) + subprocess.run([args.compiler, "-std=c++20", "-UNDEBUG", str(source), "-o", str(exe)], + check=True, timeout=45) + subprocess.run([str(exe)], check=True, timeout=10) + + +if __name__ == "__main__": + main() diff --git a/tools/gpu-aot/test_query_retirement.py b/tools/gpu-aot/test_query_retirement.py index 071489e..6afdc79 100644 --- a/tools/gpu-aot/test_query_retirement.py +++ b/tools/gpu-aot/test_query_retirement.py @@ -6,6 +6,7 @@ p = argparse.ArgumentParser(description=__doc__) p.add_argument('output', type=Path) +p.add_argument('--compiler', default='clang++') a = p.parse_args() root = Path(__file__).resolve().parents[2] a.output.mkdir(parents=True, exist_ok=True) @@ -77,7 +78,7 @@ def body(path, signature): DeviceImpl device_impl;QueueImpl queue_impl; plume::D3D12Device device{&device_impl};plume::D3D12CommandQueue queue{&queue_impl}; struct State { - static constexpr u32 kFramesInFlight=3; + static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight; std::array frame_submitted{true,false,false}; bool command_list_open=true;u32 frame_slot=1; std::array>,3> retired_resources; @@ -139,8 +140,8 @@ def body(path, signature): source = a.output / 'query-retirement.cpp' source.write_text(harness + retire + '\n' + functions + '\n' + tests) exe = a.output / 'query-retirement-test.exe' -subprocess.run(['clang++', '-std=c++20', '-DNOMINMAX', str(source), '-o', str(exe)], check=True) -subprocess.run([str(exe.resolve())], check=True) +subprocess.run([a.compiler, '-std=c++20', '-DNOMINMAX', str(source), '-o', str(exe)], check=True, timeout=45) +subprocess.run([str(exe.resolve())], check=True, timeout=45) report = {'passed': True, 'actual_native_functions': ['IssueQuery','GetQueryData','ReleaseQuery','HostDevice::RetireResource'], 'checks': ['no query drain', 'fresh generations', 'retain at every submitted/open slot', 'release before END', 'guest header reuse', 'nonfinal refcount', 'fence readiness', diff --git a/tools/gpu-aot/test_resize_fences.py b/tools/gpu-aot/test_resize_fences.py index 756001f..7eb180a 100644 --- a/tools/gpu-aot/test_resize_fences.py +++ b/tools/gpu-aot/test_resize_fences.py @@ -9,6 +9,8 @@ p = argparse.ArgumentParser(description=__doc__) p.add_argument('output', type=Path) +p.add_argument('--compiler', default='clang++') +p.add_argument('--plume-source', type=Path) a = p.parse_args() root = Path(__file__).resolve().parents[2] a.output.mkdir(parents=True, exist_ok=True) @@ -23,11 +25,13 @@ def body(path, signature): end += 1 return source[start:end] -plume = body('thirdparty/plume/plume_d3d12.cpp', +plume_path = a.plume_source or Path('thirdparty/plume/plume_d3d12.cpp') +plume = body(plume_path, 'void D3D12CommandQueue::waitForCommandFence(') -resize = body('thirdparty/plume/plume_d3d12.cpp', 'bool D3D12SwapChain::resize(') +resize = body(plume_path, 'bool D3D12SwapChain::resize(') native = '\n'.join(body('rexlego/src/gpu_native/device.cpp', sig) for sig in ( - 'void RefreshCompletedSubmissionsLocked(', 'bool WaitForSubmittedFramesLocked(', + 'void RefreshCompletedSubmissionsLocked(', 'bool WaitForSubmissionLocked(', + 'bool WaitForSubmittedFramesLocked(', 'bool RebuildSwapChain(')) source = r''' #include @@ -85,7 +89,7 @@ def body(path, signature): } struct SwapChain {bool resize_ok=true,empty=false;int resizes=0;bool resize();bool isEmpty(){return empty;}}; struct State { - static constexpr u32 kFramesInFlight=3; + static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight; Backend backend=Backend::kD3D12; std::array frame_submitted{}; std::array,3> frame_fences; @@ -160,8 +164,8 @@ def body(path, signature): h = a.output / 'resize-fences.cpp' h.write_text(source) exe = a.output / 'resize-fences.exe' -subprocess.run(['clang++', '-std=c++20', '-DNOMINMAX', str(h), '-o', str(exe)], check=True) -subprocess.run([str(exe.resolve())], check=True) +subprocess.run([a.compiler, '-std=c++20', '-UNDEBUG', '-DNOMINMAX', str(h), '-o', str(exe)], check=True, timeout=45) +subprocess.run([str(exe.resolve())], check=True, timeout=10) (a.output / 'verification.json').write_text(json.dumps({ 'actual_production_bodies': True, 'fake_gpu': True, 'passed': True, 'checks': ['completed consumed event', 'stale event and newer fence', 'all three submissions before resize', diff --git a/tools/gpu-aot/test_submission_waits.py b/tools/gpu-aot/test_submission_waits.py new file mode 100644 index 0000000..698a012 --- /dev/null +++ b/tools/gpu-aot/test_submission_waits.py @@ -0,0 +1,208 @@ +"""Exercise native submission waits and retirement with deterministic fences.""" +import argparse +import json +from pathlib import Path +import subprocess + + +def function(source, signature): + start = source.index(signature) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + args.output.mkdir(parents=True, exist_ok=True) + production = (root / "rexlego/src/gpu_native/device.cpp").read_text() + bodies = "\n".join(function(production, signature) for signature in ( + "void RefreshCompletedSubmissionsLocked(", + "void MarkSubmissionLocked(", + "bool WaitForSubmissionLocked(", + "bool WaitForSubmittedFramesLocked(", + "plume::RenderCommandList* BeginFrameCommandsLocked(", + "bool HostDevice::Synchronize(", + )) + harness = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "gpu_native/descriptor_retirement.h" +using u32=std::uint32_t;using u64=std::uint64_t; +using legodimensions::gpu_native::DescriptorRetirement; +#define REXLOG_ERROR(...) ((void)0) +#define REXLOG_WARN(...) ((void)0) +enum class Backend {kD3D12,kVulkan}; +enum class SyncReason {kOther,kCount}; +namespace plume { +struct NativeFence {u64 completed=0;u64 GetCompletedValue(){return completed;}}; +struct RenderCommandFence {virtual ~RenderCommandFence()=default;}; +struct D3D12CommandFence:RenderCommandFence { + NativeFence native;NativeFence* d3d=&native;u64 fenceValue=2; + std::optional wait_result; +}; +struct RenderCommandList { + int begins=0,ends=0;void begin(){++begins;}void end(){++ends;} +}; +struct Queue { + int waits=0,submissions=0; + void waitForCommandFence(RenderCommandFence* base){ + ++waits;auto& fence=*static_cast(base); + if(fence.wait_result)fence.native.completed=*fence.wait_result; + } + void executeCommandLists(RenderCommandList*,RenderCommandFence* base){ + ++submissions;++static_cast(base)->fenceValue; + } +}; +struct Swap {bool needsResize(){return false;}}; +} +struct State { + static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight; + Backend backend=Backend::kD3D12; + std::array frame_submitted{}; + std::array,3> frame_fences; + std::array,3> command_lists; + std::unique_ptr queue=std::make_unique(); + std::unique_ptr swap_chain=std::make_unique(); + std::vector framebuffers{1,2,3}; + std::array slot_submission{1,2,3}; + u64 last_submission=3,completed_submission=0; + std::array>,3> retired_resources; + std::array,3> snapshots; + DescriptorRetirement retired_descriptors{8}; + std::array texture_slots{true,true,true,true,true,true,true,true}; + bool command_list_open=false;u32 frame_slot=0,draw_upload_page=7; + u64 draw_upload_offset=1024,frame_slot_wait_calls=0,sync_calls=0; + u32 long_wait_log_count=0; + double frame_slot_wait_ms=0,sync_ms=0; + std::array sync_reason_calls{}; + std::array sync_reason_ms{}; + State(){for(u32 i=0;i<3;++i){ + frame_fences[i]=std::make_unique(); + command_lists[i]=std::make_unique(); + }} +}; +int saved_snapshots=0; +void SaveSnapshots(std::vector& snapshots){saved_snapshots+=int(snapshots.size());snapshots.clear();} +bool NativeTextureTimingEnabled(){return true;} +void InitializeNullTextures(State&,plume::RenderCommandList*){} +bool RebuildSwapChain(State&){assert(false);return false;} +std::unique_ptr g_state; +std::mutex g_mutex; +struct HostDevice { + static auto LockRecording(){static std::recursive_mutex mutex;return std::unique_lock(mutex);} + static bool Synchronize(SyncReason reason=SyncReason::kOther); +}; +''' + tests = r''' +std::weak_ptr Retain(State& state,u32 slot){ + auto resource=std::make_shared(int(slot)); + state.retired_resources[slot].push_back(resource); + state.frame_submitted[slot]=true; + state.snapshots[slot]={int(slot)}; + assert(!state.retired_descriptors.Retire(3+slot,1u<begins==0&&state.snapshots[0].size()==1); + assert(state.completed_submission==0&&!state.command_list_open); + state.frame_fences[0]->wait_result=1; + assert(BeginFrameCommandsLocked(state)==state.command_lists[0].get()); + assert(!state.frame_submitted[0]&&resource.expired()&&!state.texture_slots[3]); + assert(state.draw_upload_page==0&&state.draw_upload_offset==0); + assert(state.command_lists[0]->begins==1&&state.snapshots[0].empty()); + assert(state.completed_submission==1&&state.command_list_open); + } + for(bool removed_before_wait:{false,true}){ + State state;auto resource=Retain(state,0); + if(removed_before_wait)state.frame_fences[0]->native.completed=UINT64_MAX; + else state.frame_fences[0]->wait_result=UINT64_MAX; + assert(!BeginFrameCommandsLocked(state)); + assert(state.queue->waits==int(!removed_before_wait)); + assert(state.frame_submitted[0]&&!resource.expired()&&state.texture_slots[3]); + assert(state.command_lists[0]->begins==0&&state.completed_submission==0); + } + { + g_state=std::make_unique();auto& state=*g_state; + auto first=Retain(state,0),second=Retain(state,1),third=Retain(state,2); + state.frame_fences[0]->native.completed=1; + assert(!HostDevice::Synchronize()); + assert(!state.frame_submitted[0]&&first.expired()&&!state.texture_slots[3]); + assert(state.frame_submitted[1]&&!second.expired()&&state.texture_slots[4]); + assert(state.frame_submitted[2]&&!third.expired()&&state.texture_slots[5]); + assert(state.snapshots[0].empty()&&state.snapshots[1].size()==1); + assert(state.completed_submission==1&&state.queue->waits==2); + // Retry drains only remaining submissions, not slot 0's consumed event. + state.frame_fences[1]->wait_result=1;state.frame_fences[2]->wait_result=1; + assert(HostDevice::Synchronize()); + assert(second.expired()&&third.expired()&&state.completed_submission==3); + assert(state.queue->waits==4&&state.snapshots[1].empty()&&state.snapshots[2].empty()); + } + for(bool removed_before_wait:{false,true}){ + g_state=std::make_unique();auto& state=*g_state;auto resource=Retain(state,0); + if(removed_before_wait)state.frame_fences[0]->native.completed=UINT64_MAX; + else state.frame_fences[0]->wait_result=UINT64_MAX; + assert(!HostDevice::Synchronize()); + assert(state.frame_submitted[0]&&!resource.expired()&&state.texture_slots[3]); + assert(state.completed_submission==0&&state.snapshots[0].size()==1); + } + { + g_state=std::make_unique();auto& state=*g_state;auto resource=Retain(state,0); + state.command_list_open=true;state.frame_fences[0]->wait_result=2; + assert(HostDevice::Synchronize()); + assert(state.queue->submissions==1&&state.command_lists[0]->ends==1); + assert(!state.command_list_open&&!state.frame_submitted[0]&&resource.expired()); + assert(state.completed_submission==4); + } + { + State state;state.backend=Backend::kVulkan;auto resource=Retain(state,0); + assert(BeginFrameCommandsLocked(state)&&resource.expired()); + } + g_state.reset();assert(!HostDevice::Synchronize()); +} +''' + source = args.output / "submission-waits.cpp" + source.write_text(harness + bodies + tests) + exe = args.output / "submission-waits.exe" + include = str(root / "rexlego/src") + if Path(args.compiler).name.lower() in ("cl", "cl.exe"): + command = [args.compiler, "/nologo", "/std:c++20", "/EHsc", "/UNDEBUG", + "/I" + include, str(source), "/Fe:" + str(exe)] + else: + command = [args.compiler, "-std=c++20", "-UNDEBUG", "-pthread", + "-I" + include, str(source), "-o", str(exe)] + subprocess.run(command, check=True, timeout=45) + subprocess.run([str(exe.resolve())], check=True, timeout=10) + (args.output / "verification.json").write_text(json.dumps({ + "passed": True, "actual_production_bodies": True, "fake_gpu": True, + "checks": ["incomplete wait preserves allocator/resources/descriptors", + "removal before and during wait", "successful retry", + "partial drain retires completed slots only", + "open-list submission before drain"], + "real_GPU_gameplay_not_proven": True, + }, indent=2)) + print("PASS: native submission waits reject failed fences before resource or allocator reuse.") + + +if __name__ == "__main__": + main() From 35632b262fba44cc30283a6a1d7e6b609b3fc381 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:16:57 +0200 Subject: [PATCH 13/40] Add isolated renderer checkpoint regression tools Prepare private writable snapshots with pinned binaries and saves, launch only explicitly, and compare fixed cadence windows without selecting favorable results. Keep rejected evidence and vary one setting per experiment. Validation: 32 synthetic checks passed through staged-source CTest. Preparation, report and launcher dry-run passed with the frozen293 real installation and33 checkpoint files;225 independent reflinks, no fallback copies. No additional game was launched. --- tools/gpu-aot/CMakeLists.txt | 12 + tools/gpu-aot/ENGINEERING-PLAN.md | 36 +- tools/gpu-aot/regression/README.md | 167 +++++ tools/gpu-aot/regression/compare_samples.py | 79 +++ tools/gpu-aot/regression/execute_run.py | 113 +++ tools/gpu-aot/regression/prepare_run.py | 735 ++++++++++++++++++++ tools/gpu-aot/regression/sample_cadence.py | 276 ++++++++ tools/gpu-aot/regression/test_regression.py | 674 ++++++++++++++++++ 8 files changed, 2091 insertions(+), 1 deletion(-) create mode 100644 tools/gpu-aot/regression/README.md create mode 100644 tools/gpu-aot/regression/compare_samples.py create mode 100644 tools/gpu-aot/regression/execute_run.py create mode 100644 tools/gpu-aot/regression/prepare_run.py create mode 100644 tools/gpu-aot/regression/sample_cadence.py create mode 100644 tools/gpu-aot/regression/test_regression.py diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index 0ddfc7f..fd2dd03 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -2,6 +2,18 @@ cmake_minimum_required(VERSION 3.25) project(NativeGpuPortableChecks LANGUAGES CXX) include(CTest) +# The private-run harness uses Linux /proc, flock and reflinks. No game assets +# are required by its synthetic preparation, sampling and comparison checks. +if(BUILD_TESTING AND UNIX) + find_package(Python3 3.11 COMPONENTS Interpreter QUIET) + if(Python3_Interpreter_FOUND) + add_test(NAME renderer_regression + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/regression/test_regression.py) + set_tests_properties(renderer_regression PROPERTIES TIMEOUT 30) + endif() +endif() + + # No SDK, generated game code, game assets, DXC, or graphics device required. set(CHECKS blend_state buffer_header color_resolve_swizzle descriptor_retirement pixel_shader_filter polygon_offset pool_copy portrait_identity present_rect diff --git a/tools/gpu-aot/ENGINEERING-PLAN.md b/tools/gpu-aot/ENGINEERING-PLAN.md index f4fc689..035e552 100644 --- a/tools/gpu-aot/ENGINEERING-PLAN.md +++ b/tools/gpu-aot/ENGINEERING-PLAN.md @@ -58,6 +58,27 @@ callback, fence and descriptor-retirement ticket; increasing swap-chain latency or removing completion waits is not equivalent. Keep the current default until GPU runs demonstrate a useful gain without regressions. +The frozen `293d4aa0f7071b69` build now provides per-present CSV without detailed +per-draw clocks. In matched, warmed Vorton runs, twelve command slots measured +16.432 FPS against 16.389 FPS with three slots: a 0.26% difference, insufficient +to justify changing the default. A colder interval with resource streaming was +rejected and retained. Cheap wait-call counts are not measured wait durations. + +An exact-build userspace CPU profile identified the SDK timer spin loop in 641 +of 2,031 samples (31.56%). A restart-only blocking-timer experiment reduced that +thread's CPU consumption from 6.710 to 0.231 seconds per ten-second interval +(96.56%). Bracketed cadence measurements were 16.389, 17.395 and 16.669 FPS for +spin, blocking and spin. NPC/particle work varied, so the CPU saving is stronger +evidence than the modest observed FPS benefit. Timer defaults remain unchanged; +deadline precision, other workloads and campaign behavior still need validation. + +Use the tracked `regression/` tools for independent writable checkpoint copies, +exact binary/runtime identities, explicit launch ownership, fixed measurement +windows and comparisons that retain rejected outcomes. Their synthetic checks +and a real-build preparation/report/dry-run smoke do not establish rendered +correctness. Add GPU queue timestamp spans before attributing unassigned frame +time to CPU work; blocked time is absent from userspace CPU samples. + ## Shader coverage The static extractor filters named archive entries. Its output count does not @@ -84,7 +105,20 @@ physical sections, instructions, and fixed shader metadata match byte for byte. The engine's XDK repacking path removes reflection. The standard extractor was reading name-tree positions as entry indices; new-format archives require the authoritative CRC-based `FindEntry` lookup. Fix this association before judging -coverage from an extraction count. Nineteen observed origins remain unproved. +coverage from an extraction count. A subsequent complete named TU23 scan, with +actual decoded counts verified, +traced the remaining nineteen as well. All 169 observed runtime origins are now +accounted for: 96 executable-resident and 73 archive-resident. This is coverage +of the observed set, not proof that later scenes contain no additional shaders. +The scan produced 7,209 distinct structural source containers. All 3,270 newly +indexed programs passed guarded compilation. Of these, 2,939 strict non-alias +programs passed the normal compiler/exporter/parser chain in fifteen unpublished +additive packs. Those packs remain bound to an older baseline and need normal +re-export for a new build before adoption. Reflection/instruction aliases were +audited separately: both prelinked specialization variants matched byte for byte +for all 281 directly compared pairs. No alias canonicalization is enabled. +An auxiliary ledger initially read the wrong initializer column for masks; its +corrected ledger matches actual compiler fields and the unchanged pack metadata. Use bounded scan reports with explicit unreadable, sparse, malformed, and unexamined counts. Record exact container and microcode identities. Compile new diff --git a/tools/gpu-aot/regression/README.md b/tools/gpu-aot/regression/README.md new file mode 100644 index 0000000..a6214c8 --- /dev/null +++ b/tools/gpu-aot/regression/README.md @@ -0,0 +1,167 @@ +# Private renderer regression runs + +These Linux/Proton helpers prepare a real saved-game checkpoint for repeatable +renderer experiments. Preparation and reporting never launch, send Toypad +packets, capture the desktop, or press keys. `execute_run.py` launches only with +explicit `--launch`; it refuses another game and locks execution within the +chosen workspace. Use one workspace for all concurrent operators. + +Python3.11+ and Linux reflink support are required for large inputs. Unsupported +reflinks fall back only for individual files ≤1MiB, with an aggregate8MiB copy +budget. Copies have independent inodes. Failed preparation is preserved for +inspection and cannot launch. There are no included game assets or machine paths. + +## Inputs + +Supply these independently recorded inputs: + +- A baseline directory containing `legodimensions.exe` and + `baseline-manifest.json` with `build_fingerprint` (16 lowercase hex digits) + and `executable_sha256`. Record the source/dependency provenance here too. + `frozen_dependencies.rexruntime.dll.sha256` pins the exact compatible runtime; + a matching imported symbol set cannot replace that ABI identity. +- A staged installation with `legodimensions.toml`, the four primary binaries, + support files, and `native-staging-manifest.json`. Its `binaries` maps names + to `size`/`sha256`; `imports` lists each primary PE's actual imported DLL names; + `graphics` maps `dxgi.dll`, `d3d12.dll`, and `d3d12core.dll` to their source paths. +- A legitimate checkpoint directory with flat `savegame_1` files and + `save-metadata.json`: each filename maps to its `bytes` and `sha256`. + `GAME01`, `V2GAME01`, and `OPTS01` are required. Supply its original profile XUID. +- A tag directory with your persistent180-byte `batman.bin`, `gandalf.bin`, + `wyldstyle.bin`, and `batmobile.bin`. Prepared LOAD packets refer to private + backing files; this tool does not transmit them. The starter layout is Batman + pad2/index4, Gandalf2/0, Wyldstyle3/6, and Batmobile1/1. Only place figures the + scene recipe calls for; preparing the car does not load it. +- Proton, its data prefix, Steam's client directory, and a distinct Toypad port. + +The tool freezes private profile/install/DLC supporting files, checkpoint, +update, mod-update, mods, cache, metadata and tag paths. Game disc data remains a +shared asset reference with game-relative writes disabled. The Proton prefix is +explicitly selected and may be shared across sequential runs. Large supporting +files use per-file COW snapshots and are not rehashed: this is not an atomic +whole-machine save state. Close source writers before preparing twins. + +```sh +python3 tools/gpu-aot/regression/prepare_run.py prepare slots3-a \ + --workspace "$WORKSPACE" --runs "$WORKSPACE/runs" \ + --baseline "$BASELINE" --install "$STAGING" \ + --checkpoint "$CHECKPOINT" --xuid "$XUID" --tags "$TAGS" \ + --proton "$PROTON" --proton-prefix "$PREFIX" --steam-client "$STEAM" \ + --toypad-port 19207 --buffer-windows --cadence-only --command-slots 3 --timer-wait spin +python3 tools/gpu-aot/regression/prepare_run.py report "$WORKSPACE/runs/slots3-a" \ + --workspace "$WORKSPACE" +python3 tools/gpu-aot/regression/execute_run.py "$WORKSPACE/runs/slots3-a" \ + --workspace "$WORKSPACE" # verify and print only +python3 tools/gpu-aot/regression/execute_run.py "$WORKSPACE/runs/slots3-a" \ + --workspace "$WORKSPACE" --launch # explicit separate action +``` + +Preparation rejects customized mod/modcli configurations. Cadence mode removes +`LEGO_NATIVE_TIMING` completely, enables frame CSV metrics, and omits LongProbe, +draw triggers and snapshots. A value of `"0"` still enables presence-based +switches, so it is not equivalent to absence. Normal performance mode retains +detailed timing; the two modes have different workloads. + +`--timer-wait spin` preserves the default by omitting `REX_TIMER_WAIT_BLOCKING`. +`--timer-wait blocking` explicitly sets it to `1`; this restart-only SDK adapter +changes timer wake scheduling and remains an experiment. Matching imports and +lower background CPU usage do not prove equivalent deadline precision or FPS. +For a timer comparison keep command slots fixed and pass +`--vary timer-wait` to `compare_samples.py`. Its default `--vary command-slots` +requires timer mode to match. Varying both settings rejects comparability. + +For a PM4 reference use `--renderer pm4`, optionally +`--render-target-path-d3d12 rtv`, and provide `--pm4-plugin`, +`--pm4-plugin-sha256`, `--pm4-runtime-sha256`, and +`--pm4-compatibility-proof`. The proof is JSON with those two SHA256 fields and +a nonempty `evidence` string recording why that exact ABI pair is compatible. +The plugin's runtime imports are checked against actual exports, but symbol +closure alone does not establish internal ABI compatibility. PM4 cannot use +native diagnostic flags. Actual runtime route logs and visible rendering are +still required; a launch descriptor does not prove either. + +## Fixed cadence intervals + +Use normal controls to load the checkpoint and required figures, without moving +the camera or player. Publish any additive packs separately with the production +exporter/parser; build fingerprint gates remain strict. When packs are used, +record `hotload/publications.json` as an ordered list of +`{"generation":1,"source":"/path/to/frozen.pack","sha256":"..."}` entries. +The sampler verifies the source SHA/header build identity and every generation's +actual acceptance log. It never publishes packs itself. + +Capture the intended scene before sampling. Create a scene-confirmation JSON: + +```json +{ + "run_root": "/absolute/private/run", + "build_fingerprint": "0123456789abcdef", + "renderer": "native", + "scene_id": "powered-portal-default-vorton", + "camera_id": "untouched-load-camera", + "stationary": true, + "camera_unchanged": true, + "inputs_released": true, + "workers_held": true, + "capture_path": "/path/to/before.png", + "capture_sha256": "64 lowercase hex digits", + "figures": [{"name":"batman","pad":2,"index":4,"uid":"recorded UID"}] +} +``` + +Include every actually loaded figure with its prepared manifest UID/placement. +Scene identity and input ownership are explicit operator evidence; the scripts +do not infer them from pixels. Hold compiler/scanner/profiler workers throughout +warmup and sampling. The sampler records endpoint process observations as a +supplement, not continuous absence proof. It requires the actual native route +and initialized slot count in the game log. + +```sh +python3 tools/gpu-aot/regression/sample_cadence.py "$RUN" \ + --workspace "$WORKSPACE" --output "$WORKSPACE/samples/slots3-a" \ + --scene-confirmation "$SCENE_JSON" +``` + +Each attempt uses the same fixed settling gate: ten seconds with no new texture +adoption, buffer relocation, texture refresh, missing shader or pack load; frame +IDs must progress and the last60 rows' half-window medians must differ by at +most25%. Warmup is bounded at120seconds, then one fixed30-second interval is +recorded. No screenshots or input belong inside that interval. Streaming, +observed workers, or renderer failure reject the window while preserving its +CSV/log/outcome. Interrupted attempts also retain rejection evidence. Outputs +cannot be overwritten; give retries distinct names. +The sampler reads at most a4MiB CSV tail and retains at most10,000 rows. +Context logs are capped at16MiB and interval evidence at8MiB; excessive logging +fails closed instead of growing memory without bound. Execution evidence uses +atomic JSON publication and bounded reads. + +Capture the scene again afterward and pause normally. Prepare separate3/12/3 +runs from the same inputs, keeping every outcome. Feed all outcomes, including +rejected ones, to the comparison: + +```sh +python3 tools/gpu-aot/regression/compare_samples.py \ + "$WORKSPACE/samples/slots3-a" "$WORKSPACE/samples/slots12-b" \ + "$WORKSPACE/samples/slots3-a2" --output "$WORKSPACE/comparison.json" +``` + +Comparison checks binary/runtime/DLL hashes, initial checkpoint, normalized +configuration, flags, packs, criteria and confirmed scene/figures. It validates +metrics against frozen CSVs, aggregates all accepted windows, and never chooses +the fastest window or changes the source default. NPC/particle phases and +ordinary desktop activity remain limitations. CSV flushing shifts the recorded +duration relative to wall endpoints. Disabled stage-time/hash/conversion columns +measure no timing/work quantity. Native present intervals and PM4's XE_SWAP +request intervals have different boundaries and draw-counter scopes. + +## Asset-free verification + +```sh +python3 tools/gpu-aot/regression/test_regression.py +``` + +Tests exercise actual independent copies, private writable paths, real packet +construction, bounded fallback, corruption/alias rejection, synthetic static +PE imports, environment sanitization, cadence descriptors and fake-clock fixed +sampling/comparison. Synthetic PE files are never executed. These checks do not +validate game pixels, actual driver timing or campaign completion. diff --git a/tools/gpu-aot/regression/compare_samples.py b/tools/gpu-aot/regression/compare_samples.py new file mode 100644 index 0000000..94f18f3 --- /dev/null +++ b/tools/gpu-aot/regression/compare_samples.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Compare every supplied cadence outcome; never select the fastest window.""" +import sys +sys.dont_write_bytecode = True +import argparse +import json +from pathlib import Path + +from prepare_run import require, write_json +from sample_cadence import complete_rows, stats + + +def comparison_identity(sample, varying='command-slots'): + scene = sample['scene'] + flags = {key: value for key, value in sample['flags'].items() + if key.startswith(('LEGO_NATIVE_', 'LEGO_DUMP_', 'LEGO_GPU_', 'REX_TIMER_'))} + for key in ('LEGO_NATIVE_FRAME_METRICS', 'LEGO_DUMP_MISSING_SHADERS', + 'LEGO_NATIVE_SHADER_PACK', 'LEGO_NATIVE_SHADER_PACK_TRIGGER'): + flags.pop(key, None) # Private paths differ; sampler verifies their ownership. + flags.pop('LEGO_NATIVE_COMMAND_SLOTS' if varying == 'command-slots' else 'REX_TIMER_WAIT_BLOCKING', None) + return {'binaries': sample['binaries'], 'checkpoint': sample['checkpoint'], 'criteria': sample['criteria'], + 'normalized_config': sample['normalized_config'], 'shader_packs': sample['shader_packs'], + 'supporting_state': sample['supporting_state'], + 'scene_id': scene['scene_id'], 'camera_id': scene['camera_id'], + 'renderer': scene['renderer'], 'figures': scene['figures'], 'flags': flags} + + +def compare(paths, varying='command-slots'): + require(varying in ('command-slots', 'timer-wait'), 'Unknown experimental variable') + samples = [json.loads((path/'sample.json').read_text()) for path in paths] + for path, sample in zip(paths, samples): + if 'metrics' in sample: + _, rows = complete_rows(path/'frames.csv') + require(stats(rows) == sample['metrics'], 'Frozen CSV disagrees with sample metrics') + outcomes = [{'source': str(path.resolve()), 'status': sample['status'], 'slots': sample['slots'], + 'timer_wait': sample['timer_wait'], + 'metrics': sample.get('metrics'), 'rejection_reasons': sample.get('rejection_reasons', [])} + for path, sample in zip(paths, samples)] + accepted = [sample for sample in samples if sample['status'] == 'accepted'] + identity = comparison_identity(accepted[0], varying) if accepted else None + comparable = (all(comparison_identity(sample, varying) == identity for sample in accepted) + if accepted else None) + means = {} + if comparable: + variable = 'slots' if varying == 'command-slots' else 'timer_wait' + for value in sorted({sample[variable] for sample in accepted}): + group = [sample for sample in accepted if sample[variable] == value] + frames = sum(sample['metrics']['frames'] for sample in group) + seconds = sum(sample['metrics']['recorded_seconds'] for sample in group) + means[str(value)] = {'windows': len(group), 'frames': frames, + 'recorded_seconds': seconds, 'fps': frames/seconds} + baseline, candidate = ('3', '12') if varying == 'command-slots' else ('spin', 'blocking') + gain = (100*(means[candidate]['fps']/means[baseline]['fps']-1) + if candidate in means and baseline in means else None) + return {'outcomes': outcomes, 'comparable_identity': comparable, 'accepted_aggregate': means, + 'experimental_variable': varying, 'candidate_vs_baseline_fps_percent': gain, + 'source_default_changed': False, + 'decision': 'Retain source defaults; measurements never automatically promote an experimental setting.', + 'limits': ['All supplied windows are retained, including rejected and unfavorable outcomes.', + 'Operator scene assertions and NPC/particle phases limit causal attribution.', + 'This compares native present cadence; PM4 XE_SWAP metrics have a different boundary.', + 'Repeated A/B/A evidence and visual correctness are needed before a policy change.']} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('samples', nargs='+', type=Path) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--vary', choices=('command-slots', 'timer-wait'), default='command-slots') + args = parser.parse_args() + require(not args.output.exists(), 'Refuse overwrite of a previous comparison') + args.output.parent.mkdir(parents=True, exist_ok=True) + report = compare(args.samples, args.vary) + write_json(args.output, report) + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/tools/gpu-aot/regression/execute_run.py b/tools/gpu-aot/regression/execute_run.py new file mode 100644 index 0000000..ad8dc8c --- /dev/null +++ b/tools/gpu-aot/regression/execute_run.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Execute one verified private renderer run; refuse concurrent game processes.""" +import sys +sys.dont_write_bytecode = True +import argparse +import fcntl +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import time + +HERE = Path(__file__).resolve().parent +INTRINSIC_PREFIXES = ('LEGO_NATIVE_', 'LEGO_GPU_', 'LEGO_DUMP_', 'REX_') +INTRINSIC_KEYS = ('XENIA_TOYPAD_PORT', 'REXGLUE_TOYPAD_PORT', 'PROTON_LOG', + 'VKD3D_SHADER_CACHE_PATH', 'DXVK_STATE_CACHE_PATH') + + +def active_games(): + found = [] + for process in Path('/proc').iterdir(): + if not process.name.isdecimal(): + continue + try: + arguments = (process / 'cmdline').read_bytes().split(b'\0') + except (FileNotFoundError, PermissionError, ProcessLookupError): + continue + if any(arg.decode(errors='replace').replace('\\', '/').rsplit('/', 1)[-1].lower() + == 'legodimensions.exe' for arg in arguments): + found.append(int(process.name)) + return sorted(found) + + +def environment(plan): + result = dict(os.environ) + prefixes = tuple(plan['remove_inherited_prefixes']) + INTRINSIC_PREFIXES + keys = tuple(plan['remove_inherited_keys']) + INTRINSIC_KEYS + for key in list(result): + if key.startswith(prefixes) or key in keys: + result.pop(key) + for key, value in plan['environment_overrides'].items(): + if not isinstance(value, str) or '\0' in value: + raise ValueError('Invalid planned environment value') + result[key] = value + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('run', type=Path) + parser.add_argument('--launch', action='store_true') + parser.add_argument('--workspace', type=Path, required=True) + args = parser.parse_args() + spec = importlib.util.spec_from_file_location('regression', HERE / 'prepare_run.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + run = args.run.resolve(strict=True) + verification = module.verify(run, args.workspace) + for name in ('logs', 'cache', 'metadata', 'hotload', 'captures', + 'shader-capture', 'texture-capture', 'toypad-tags', 'no-mod-game'): + directory = run / name + if directory.is_symlink() or not directory.is_dir() or directory.resolve().parent != run: + raise ValueError('Writable run directory is not a private directory: ' + name) + plan = verification['launch_plan'] + if Path(plan['cwd']).resolve() != run: + raise ValueError('Launch cwd differs from the verified private run') + if (not isinstance(plan['argv'], list) or len(plan['argv']) != 6 + or not all(isinstance(value, str) for value in plan['argv'])): + raise ValueError('Invalid planned launch command') + if plan['argv'] != ['nice', '-n', '10', str(Path(plan['argv'][3]).resolve()), + 'runinprefix', str(run / 'legodimensions.exe')]: + raise ValueError('Launch command differs from the expected private executable') + env = environment(plan) + active = active_games() + report = {'unix': time.time(), 'run': str(run), 'active_game_pids': active, + 'route': env.get('LEGO_NATIVE_PM4_REFERENCE'), 'argv': plan['argv'], + 'cwd': plan['cwd'], 'prepared_identity_verified': True, + 'launch_requested': args.launch, 'environment_overrides': plan['environment_overrides']} + if not args.launch: + print(json.dumps(report)) + return + lock_root = args.workspace.resolve(strict=True) + lock_path = lock_root / '.renderer-regression-execution.lock' + if lock_path.is_symlink(): + raise ValueError('Execution lock may not be a symlink') + lock = os.fdopen(os.open(lock_path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600), 'a') + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise RuntimeError('Another regression runner owns the game execution lock') + active = active_games() + if active: + raise RuntimeError('Existing game process must close before an isolated run: ' + str(active)) + evidence = run / 'logs/execution.json' + if evidence.exists(): + raise RuntimeError('Run already has execution evidence; prepare a fresh run') + module.write_json(evidence, report, exclusive=True) + with (run / 'logs/launcher.log').open('x') as output: + child = subprocess.Popen(plan['argv'], cwd=plan['cwd'], env=env, + stdout=output, stderr=subprocess.STDOUT, start_new_session=True) + report['launcher_pid'] = child.pid + report['started_unix'] = time.time() + module.write_json(evidence, report) + print(json.dumps({'run': str(run), 'launcher_pid': child.pid, 'route': report['route']}), flush=True) + report['exit_code'] = child.wait() + report['finished_unix'] = time.time() + module.write_json(evidence, report) + raise SystemExit(report['exit_code']) + + +if __name__ == '__main__': + main() diff --git a/tools/gpu-aot/regression/prepare_run.py b/tools/gpu-aot/regression/prepare_run.py new file mode 100644 index 0000000..80151c4 --- /dev/null +++ b/tools/gpu-aot/regression/prepare_run.py @@ -0,0 +1,735 @@ +#!/usr/bin/env python3 +"""Prepare isolated legitimate checkpoints and an exact launch descriptor. + +Never launches a process, sends a Toypad packet, or provides desktop input. +""" +import argparse +import errno +import fcntl +import hashlib +import json +import mmap +import os +from pathlib import Path +import re +import shutil +import struct +import tempfile +import time +import tomllib + +HERE = Path(__file__).resolve().parent +WORKSPACE = Path.cwd() +FICLONE = 0x40049409 +MAX_TREE_ENTRIES = 20000 +TITLE = '5752084B' +def save_path(xuid): + require(re.fullmatch(r'[0-9A-Fa-f]{16}', xuid), 'XUID must be sixteen hexadecimal digits') + return Path(xuid.upper()) / TITLE / '00000001/savegame_1' + +BINARY_NAMES = ('legodimensions.exe', 'rexruntime.dll', 'dxcompiler.dll', 'dxil.dll') +SUPPORT_NAMES = ('FiraSans-Regular.ttf', 'achievement_unlocked.wav', 'gamecontrollerdb.txt') +SLOTS = {'batman': (2, 4), 'gandalf': (2, 0), 'wyldstyle': (3, 6), 'batmobile': (1, 1)} +SANITIZED_PREFIXES = ('LEGO_NATIVE_', 'LEGO_GPU_', 'LEGO_DUMP_', 'REX_') +RECIPE = { + 'name': 'operator-confirmed-checkpoint', + 'steps': ['Start the exact verified descriptor after every existing game closes.', + 'Load the recorded private persistent figures using normal game protocol.', + 'Load the legitimate saved game; confirm the scene and camera with external evidence.'], + 'comparison_gate': 'Compare only operator-confirmed same scene, camera, figures, build and flags.', + 'automatic_input': False, +} + + + +def require(value, message): + if not value: + raise ValueError(message) + + +def digest(path): + before = path.stat() + sha = hashlib.sha256() + with path.open('rb') as source: + for chunk in iter(lambda: source.read(1024 * 1024), b''): + sha.update(chunk) + after = path.stat() + require((before.st_size, before.st_mtime_ns, before.st_ino) == + (after.st_size, after.st_mtime_ns, after.st_ino), f'Input changed while hashing: {path}') + return {'bytes': after.st_size, 'sha256': sha.hexdigest()} + + +def windows(path): + return 'Z:' + str(path.absolute()).replace('/', '\\') + + +def configured_host(value, install): + require(isinstance(value, str) and value, 'Missing configured asset directory') + if value.startswith(('Z:\\', 'z:\\')): + path = Path(value[2:].replace('\\', '/')) + else: + require(not re.match(r'^[a-zA-Z]:', value), 'Only local Z: asset references are supported') + path = Path(value) + if not path.is_absolute(): + path = install / path + return path.resolve(strict=True) + + +def pe_tables(path, maximum_bytes=64*1024**2): + """Bounded static PE32+ import/export inspection; never loads a DLL.""" + require(0 < path.stat().st_size <= maximum_bytes, 'PE dependency exceeds inspection bound') + with path.open('rb') as source, mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ) as data: + def unpack(fmt, offset): + require(0 <= offset <= len(data) - struct.calcsize(fmt), 'PE field outside file') + return struct.unpack_from(fmt, data, offset) + require(data[:2] == b'MZ', 'Dependency is not a PE file') + pe, = unpack('= 240 and + unpack('= 2, 'Missing PE import/export directories') + sections = [] + for index in range(count): + start = optional+optional_size+40*index + virtual_size, va, raw_size, raw = unpack('> 63 else name(thunk+2)) + else: + raise ValueError('Unterminated import lookup table') + imports[dll] = symbols + require(terminated, 'Unterminated import descriptors') + exports = set() + export_rva, export_size = unpack('= 0, 'No complete frame metrics yet') + size = payload_start+last_newline+1 + raw = header+raw[:last_newline+1] + rows = deque(maxlen=10000) + for row in csv.DictReader(io.StringIO(raw.decode())): + require(None not in row and all(v is not None for v in row.values()), 'Malformed complete CSV row') + values = {k: float(v) for k, v in row.items()} + require(all(math.isfinite(v) for v in values.values()), 'Non-finite metric') + require(values['frame'].is_integer() and values['frame'] >= 0 and values['interval_ms'] > 0, + 'Invalid frame/interval metric') + rows.append(values) + rows = list(rows) + require(rows, 'No complete frame metrics yet') + require(all(a['frame'] < b['frame'] for a, b in zip(rows, rows[1:])), 'Non-monotonic frame IDs') + return size, list(rows) + + +def workers(): + """Endpoint observations supplement, rather than prove, the operator quiet hold.""" + result = [] + for process in Path('/proc').iterdir(): + if not process.name.isdecimal(): + continue + try: + args = (process/'cmdline').read_bytes().split(b'\0') + command = args[0].decode(errors='replace').replace('\\', '/').rsplit('/', 1)[-1].lower() + if command in ('clang', 'clang++', 'clang-cl', 'ninja', 'llvm-mc', 'dotnet', + 'gradient_compiler.exe', 'lego_shader_prelink.exe', 'lego_shader_indexer.exe'): + result.append({'pid': int(process.name), 'command': command}) + except OSError: + continue + return result + + +def endpoint(run): + size, rows = complete_rows(run/'logs/frames.csv') + return {'unix': time.time(), 'monotonic': time.monotonic(), 'frame': int(rows[-1]['frame']), + 'csv_bytes': size, 'log_bytes': (run/'logs/game.log').stat().st_size, + 'observed_workers': workers()}, rows + + +def interval_log(run, first, last): + require(last['log_bytes'] >= first['log_bytes'], 'Game log rotated/truncated during scope') + delta = last['log_bytes']-first['log_bytes'] + require(delta <= MAX_INTERVAL_LOG_BYTES, 'Game interval log exceeds bounded evidence size') + with (run/'logs/game.log').open('rb') as source: + source.seek(first['log_bytes']) + data = source.read(delta) + require(len(data) == delta, 'Game interval log truncated during evidence read') + return data.decode(errors='replace') + + +def stats(rows): + require(rows, 'Empty selected interval') + times = sorted(r['interval_ms'] for r in rows) + mean = lambda k: statistics.fmean(r[k] for r in rows) + result = {'frames': len(rows), 'recorded_seconds': sum(times)/1000, + 'fps': 1000/mean('interval_ms'), 'p50_ms': times[(len(times)-1)//2], + 'p95_ms': times[int((len(times)-1)*.95)], 'p99_ms': times[int((len(times)-1)*.99)], + 'maximum_ms': times[-1], 'draw_calls_mean': mean('draw_calls')} + for key in ('frame_slot_wait_calls', 'callbacks_enqueued', 'callbacks_executed', 'callbacks_pending'): + if key in rows[0]: + result[key+'_mean'] = mean(key) + result['detailed_timing_enabled'] = any(r['detailed_timing_enabled'] != 0 for r in rows) + return result + + +def context(run, scene_file, workspace): + run = owned(run, workspace).resolve(strict=True) + manifest = read_json(run/'regression-manifest.json') + require(manifest['run_root'] == str(run) and manifest['workspace'] == str(workspace.resolve()), + 'Prepared run/workspace identity differs') + require(manifest.get('cadence_only_probe') and manifest['renderer_requested'] == 'native', + 'Cadence sampling requires a native cadence-only descriptor') + execution = read_json(run/'logs/execution.json', 128*1024) + require(execution.get('exit_code') is None and execution.get('launch_requested') and + execution.get('route') == '0', 'No live native execution evidence') + env = execution['environment_overrides'] + require(env == manifest['launch_plan']['environment_overrides'], 'Launch environment differs from preparation') + require('LEGO_NATIVE_TIMING' not in env and not any(k in env for k in HEAVY_FLAGS), + 'Cadence scope contains detailed/heavy instrumentation') + mode = manifest['timer_wait'] + require((mode == 'spin' and 'REX_TIMER_WAIT_BLOCKING' not in env) or + (mode == 'blocking' and env.get('REX_TIMER_WAIT_BLOCKING') == '1'), 'Timer mode disagrees') + require(env['LEGO_NATIVE_COMMAND_SLOTS'] == str(manifest['command_slots']) and + env['LEGO_NATIVE_FRAME_METRICS'] == windows(run/'logs/frames.csv'), 'Slot/CSV target differs') + log_path = run/'logs/game.log' + require(not log_path.is_symlink() and log_path.stat().st_size <= MAX_CONTEXT_LOG_BYTES, + 'Context game log exceeds bounded evidence size or aliases another path') + with log_path.open('rb') as source: log_bytes = source.read(MAX_CONTEXT_LOG_BYTES+1) + require(len(log_bytes) <= MAX_CONTEXT_LOG_BYTES, 'Context game log grew beyond its read bound') + log = log_bytes.decode(errors='replace') + require('GPU route: detached native D3D12 renderer' in log and + f'Native GPU init: creating {manifest["command_slots"]} command slots' in log, + 'Actual runtime route/slot initialization evidence is missing') + scene = read_json(scene_file, 128*1024) + require(scene.get('run_root') == str(run) and + scene.get('build_fingerprint') == manifest['frozen_build']['build_fingerprint'] and + scene.get('renderer') == 'native', 'Scene evidence does not identify this native run') + for key in ('scene_id', 'camera_id'): + require(isinstance(scene.get(key), str) and bool(scene[key].strip()), 'Missing scene/camera identity') + for key in ('stationary', 'camera_unchanged', 'inputs_released', 'workers_held'): + require(scene.get(key) is True, 'Operator must explicitly confirm '+key) + require(isinstance(scene.get('figures'), list) and scene['figures'], 'Missing loaded-figure confirmation') + expected = {f['name']: {k: f[k] for k in ('name', 'pad', 'index', 'uid')} for f in manifest['figures']} + require(len({f['name'] for f in scene['figures']}) == len(scene['figures']) and + all(f == expected.get(f['name']) for f in scene['figures']), 'Confirmed figures differ from prepared tags') + capture = Path(scene['capture_path']).resolve(strict=True) + require_sha(scene['capture_sha256']) + require(digest(capture)['sha256'] == scene['capture_sha256'], 'Scene capture no longer matches its evidence') + pins = {Path(r['path']).name: r for r in manifest['copies'] if Path(r['path']).parent == run} + binaries = {name: digest(run/name) for name in pins if name.lower().endswith(('.dll', '.exe'))} + require(all(identity['sha256'] == pins[name]['sha256'] for name, identity in binaries.items()), + 'Live executable/DLL dependencies differ from preparation') + require(binaries['legodimensions.exe']['sha256'] == manifest['frozen_build']['executable_sha256'], + 'Live binary differs from frozen EXE') + require(binaries['rexruntime.dll']['sha256'] == pins['rexruntime.dll']['sha256'], 'Live runtime differs from preparation') + require(digest(run/'legodimensions.toml')['sha256'] == manifest['config_sha256'], 'Live config changed') + config = tomllib.loads((run/'legodimensions.toml').read_text()) + config = {key: value.replace(windows(run), '') if isinstance(value, str) else value + for key, value in config.items()} + supporting = {} + for record in manifest['copies']: + relative = Path(record['path']).relative_to(run) + if relative.parts[0] not in ('content', 'update', 'update-mods', 'mods', 'toypad-tags'): + continue + supporting[str(relative)] = ({'bytes': record['bytes'], 'sha256': record['sha256']} + if 'sha256' in record else + {'bytes': record['bytes'], 'source': record['source'], + 'source_mtime_ns': record['source_mtime_ns'], + 'content_sha256_verified': False}) + pack_file = run/'hotload/publications.json' + publications = read_json(pack_file, 1024*1024) if pack_file.exists() else [] + accepted = [int(v) for v in re.findall(r'loaded additive precompiled pack, generation=(\d+)', log)] + require([entry['generation'] for entry in publications] == accepted, + 'Pack publication provenance differs from actual accepted generations') + packs = [] + for entry in publications: + source = Path(entry['source']) + require_sha(entry['sha256']) + require(digest(source)['sha256'] == entry['sha256'], 'Published pack source changed') + with source.open('rb') as stream: header = stream.read(160) + require(header[:8] == b'LEGODX1\0' and header[32:48].decode() == manifest['frozen_build']['build_fingerprint'], + 'Published pack belongs to a different frozen build') + packs.append({'generation': entry['generation'], 'sha256': entry['sha256']}) + return {'run': str(run), 'scene': scene, 'scene_confirmation': {'source': str(scene_file.resolve()), **digest(scene_file)}, + 'binaries': binaries, 'slots': manifest['command_slots'], 'criteria': dict(CRITERIA), + 'timer_wait': mode, + 'flags': env, 'normalized_config': config, 'shader_packs': packs, + 'supporting_state': supporting, + 'checkpoint': manifest['checkpoint_files'], + 'status': 'warming', 'warmup': [], + 'limits': ['Operator evidence confirms scene/input/quiet ownership; this tool cannot inspect gameplay state.', + 'Ordinary desktop activity remains; endpoint worker observations are not continuous profiling.', + 'CSV buffering shifts recorded duration relative to wall endpoints.', + 'Disabled timing, conversion and hash columns do not measure zero work.']} + + +def sample(run, output, scene_file, workspace, *, take_endpoint=endpoint, + sleep=time.sleep, clock=time.monotonic): + require(not output.exists(), 'Refuse overwriting any prior sample outcome') + report = context(run, scene_file, workspace) + owned(output, workspace).mkdir(parents=True) + write_json(output/'sample.json', report) + deadline = clock()+CRITERIA['maximum_warmup_seconds'] + try: + while True: + first, _ = take_endpoint(run) + sleep(CRITERIA['resource_quiet_seconds']) + last, rows = take_endpoint(run) + text = interval_log(run, first, last) + tail = rows[-CRITERIA['cadence_rows']:] + require(len(tail) == CRITERIA['cadence_rows'], 'Too few cadence rows for warm criterion') + times = [r['interval_ms'] for r in tail] + medians = [statistics.median(times[:30]), statistics.median(times[30:])] + ratio = max(medians)/min(medians) + check = {'start': first, 'end': last, 'resource_events': len(RESOURCE_EVENTS.findall(text)), + 'cadence_median_halves_ratio': ratio} + report['warmup'].append(check) + write_json(output/'sample.json', report) + if FAILURE_EVENTS.search(text): + raise ValueError('Render/runtime failure during warmup') + if (not check['resource_events'] and last['frame'] > first['frame'] and + not first['observed_workers'] and not last['observed_workers'] and + ratio <= CRITERIA['median_halves_ratio_max']): + break + require(clock() < deadline, 'Fixed settling criteria not achieved before deadline') + first, _ = take_endpoint(run) + report['start'] = first + report['status'] = 'sampling' + write_json(output/'sample.json', report) + sleep(CRITERIA['interval_seconds']) + last, rows = take_endpoint(run) + selected = [r for r in rows if first['frame'] < r['frame'] <= last['frame']] + require([int(r['frame']) for r in selected] == list(range(first['frame']+1, last['frame']+1)), + 'Selected frames are not contiguous') + require(selected and all(r['detailed_timing_enabled'] == 0 for r in selected), 'Detailed timing was enabled') + text = interval_log(run, first, last) + (output/'game-interval.log').write_text(text) + with (output/'frames.csv').open('x', newline='') as target: + writer = csv.DictWriter(target, fieldnames=list(selected[0])) + writer.writeheader() + writer.writerows(selected) + reasons = [] + if RESOURCE_EVENTS.search(text): reasons.append('Resource/pack/shader streaming during interval') + if FAILURE_EVENTS.search(text): reasons.append('Render/runtime failure during interval') + if first['observed_workers'] or last['observed_workers']: reasons.append('Compiler/scanner observed at endpoint') + report.update(end=last, wall_seconds=last['monotonic']-first['monotonic'], metrics=stats(selected), + rejection_reasons=reasons, status='rejected' if reasons else 'accepted') + except BaseException as error: + report.update(status='rejected', rejection_reasons=[type(error).__name__+': '+str(error)]) + write_json(output/'sample.json', report) + raise + write_json(output/'sample.json', report) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('run', type=Path) + parser.add_argument('--workspace', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--scene-confirmation', type=Path, required=True) + args = parser.parse_args() + print(json.dumps(sample(args.run, args.output, args.scene_confirmation, args.workspace), indent=2)) + + +if __name__ == '__main__': + main() diff --git a/tools/gpu-aot/regression/test_regression.py b/tools/gpu-aot/regression/test_regression.py new file mode 100644 index 0000000..71a500f --- /dev/null +++ b/tools/gpu-aot/regression/test_regression.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python3 +"""Asset-free filesystem and static-PE tests of the actual preparation helper. + +Synthetic PE files exercise inspection only; they are never executed and do +not simulate a game or renderer. Real COW isolation is exercised when supported. +""" +import argparse +import csv +import errno +import importlib.util +import json +import os +from pathlib import Path +import shutil +import socket +import struct +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.dont_write_bytecode = True +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import sample_cadence as sampler +import compare_samples as comparer +spec = importlib.util.spec_from_file_location('native_regression', HERE / 'prepare_run.py') +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +runner_spec = importlib.util.spec_from_file_location('regression_runner', HERE / 'execute_run.py') +runner = importlib.util.module_from_spec(runner_spec) +runner_spec.loader.exec_module(runner) + + +def synthetic_pe(imports, exports=()): + image = bytearray(65536) + image[:2] = b'MZ' + struct.pack_into(' Date: Wed, 7 Oct 2026 06:18:54 +0200 Subject: [PATCH 14/40] Measure native frame cadence without per-draw clocks Count draws and completion work in cadence-only mode, distinguish disabled timing fields, and split optional constant costs into disjoint phases. Add a restart-only3/12 command-slot experiment while preserving three swap-chain images and exact resource retirement; default remains3. Validation:10 production-body fixtures passed against isolated staged sources. Frozen293 real Vorton comparison found12 slots16.432FPS versus3 slots16.389FPS, so no default promotion. --- .../src/gpu_native/descriptor_retirement.h | 10 +- rexlego/src/gpu_native/device.cpp | 96 +++++++---- rexlego/src/gpu_native/draw.cpp | 55 ++++-- rexlego/src/gpu_native/draw.h | 4 + tools/gpu-aot/CMakeLists.txt | 3 +- tools/gpu-aot/test_command_slots.py | 159 ++++++++++++++++++ tools/gpu-aot/test_constant_timing.py | 89 ++++++++++ tools/gpu-aot/test_draw_upload.py | 2 +- tools/gpu-aot/test_frame_cadence.py | 100 +++++++++++ 9 files changed, 469 insertions(+), 49 deletions(-) create mode 100644 tools/gpu-aot/test_command_slots.py create mode 100644 tools/gpu-aot/test_constant_timing.py create mode 100644 tools/gpu-aot/test_frame_cadence.py diff --git a/rexlego/src/gpu_native/descriptor_retirement.h b/rexlego/src/gpu_native/descriptor_retirement.h index c197ce7..f171771 100644 --- a/rexlego/src/gpu_native/descriptor_retirement.h +++ b/rexlego/src/gpu_native/descriptor_retirement.h @@ -1,7 +1,7 @@ #pragma once -#include #include #include +#include #include namespace legodimensions::gpu_native { @@ -9,7 +9,11 @@ namespace legodimensions::gpu_native { // still being recorded. Each ticket must finish before its slot is reusable. class DescriptorRetirement { public: - explicit DescriptorRetirement(std::size_t count = 0) : pending_(count, 0) {} + explicit DescriptorRetirement(std::size_t count = 0, uint32_t slots = 3) + : pending_(count, 0) { + if (!slots || slots > 32) throw std::invalid_argument("descriptor retirement slots outside 1..32"); + tickets_.resize(slots); + } bool Retire(uint32_t index, uint32_t live_frames) { if (pending_[index]) return false; for (uint32_t frame = 0; frame < tickets_.size(); ++frame) { @@ -26,6 +30,6 @@ class DescriptorRetirement { } private: std::vector pending_; - std::array, 3> tickets_; + std::vector> tickets_; }; } // namespace legodimensions::gpu_native diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp index 7c10b29..7f1a8ff 100644 --- a/rexlego/src/gpu_native/device.cpp +++ b/rexlego/src/gpu_native/device.cpp @@ -130,8 +130,19 @@ struct CachedNativeSampler { std::unique_ptr sampler; }; +u32 NativeCommandSlotCount() { + const char* value = std::getenv("LEGO_NATIVE_COMMAND_SLOTS"); + if (!value || std::strcmp(value, "3") == 0) return 3; + if (std::strcmp(value, "12") == 0) return 12; + REXLOG_WARN("Native GPU: LEGO_NATIVE_COMMAND_SLOTS must be 3 or 12; using 3"); + return 3; +} + struct State { - static constexpr u32 kFramesInFlight = 3; + // Guest kickoffs consume slots independently of swap-chain presentation. + // Extra slots are an opt-in experiment; every slot keeps its own fence. + static constexpr u32 kMaxCommandSlots = 12; + const u32 command_slot_count = NativeCommandSlotCount(); std::unique_ptr render_interface; std::unique_ptr device; @@ -157,23 +168,23 @@ struct State { std::array, 2> depth_alias_shaders; std::array, 2> depth_alias_pipelines; std::vector> framebuffers; - std::array, kFramesInFlight> + std::array, kMaxCommandSlots> command_lists; - std::array, kFramesInFlight> + std::array, kMaxCommandSlots> frame_fences; - std::array, kFramesInFlight> + std::array, kMaxCommandSlots> acquire_semaphores; std::vector> render_semaphores; - std::array frame_submitted{}; - std::array slot_submission{}; + std::array frame_submitted{}; + std::array slot_submission{}; u64 last_submission = 0, completed_submission = 0; CompletionQueue completion_callbacks; u64 callbacks_enqueued = 0, callbacks_executed = 0; - std::array>, kFramesInFlight> draw_uploads; + std::array>, kMaxCommandSlots> draw_uploads; u32 draw_upload_page = 0; u64 draw_upload_offset = 0; - std::array>, kFramesInFlight> + std::array>, kMaxCommandSlots> retired_resources; u32 frame_slot = 0; u32 present_number = 0; @@ -184,7 +195,7 @@ struct State { u32 snapshot_time_window = 0; u32 snapshot_selected_frame = ~0u; std::chrono::steady_clock::time_point snapshot_trigger_check; - std::array, kFramesInFlight> snapshots; + std::array, kMaxCommandSlots> snapshots; bool command_list_open = false; u32 long_wait_log_count = 0; u32 timing_log_count = 0; @@ -205,7 +216,7 @@ struct State { bool probe_budget_reported = false; std::chrono::steady_clock::time_point last_present_time{}; std::vector texture_slots; - DescriptorRetirement retired_descriptors{65536}; + DescriptorRetirement retired_descriptors{65536, command_slot_count}; Backend backend = Backend::kD3D12; }; @@ -334,7 +345,8 @@ bool CreatePresentPipeline(State& state) { } bool CreateFrameRing(State& state) { - REXLOG_INFO("Native GPU init: creating frame ring"); + REXLOG_INFO("Native GPU init: creating {} command slots (swap-chain images unchanged)", + state.command_slot_count); const u32 texture_count = state.swap_chain->getTextureCount(); if (!texture_count) return false; // Resize has already discarded the old swap-chain framebuffers. Publish a @@ -358,7 +370,7 @@ bool CreateFrameRing(State& state) { } } - for (u32 i = 0; i < State::kFramesInFlight; ++i) { + for (u32 i = 0; i < state.command_slot_count; ++i) { command_lists[i] = state.queue->createCommandList(); frame_fences[i] = state.device->createCommandFence(); acquire_semaphores[i] = state.device->createCommandSemaphore(); @@ -371,7 +383,7 @@ bool CreateFrameRing(State& state) { state.command_lists = std::move(command_lists); state.frame_fences = std::move(frame_fences); state.acquire_semaphores = std::move(acquire_semaphores); - for (u32 i = 0; i < State::kFramesInFlight; ++i) { + for (u32 i = 0; i < state.command_slot_count; ++i) { state.frame_submitted[i] = false; state.retired_resources[i].clear(); state.retired_descriptors.CompleteFrame(i, @@ -496,7 +508,7 @@ void InitializeNullTextures(State& state, plume::RenderCommandList* commands) { void RefreshCompletedSubmissionsLocked(State& state) { if (state.backend != Backend::kD3D12) return; - for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) { + for (u32 slot = 0; slot < state.command_slot_count; ++slot) { if (!state.frame_submitted[slot]) continue; auto* fence = static_cast(state.frame_fences[slot].get()); const u64 completed = fence->d3d->GetCompletedValue(); @@ -544,7 +556,7 @@ bool WaitForSubmissionLocked(State& state, u32 slot) { bool WaitForSubmittedFramesLocked(State& state) { // DXGI's frame-latency object is a present throttle, not a GPU completion // fence. Rebuilding/releasing the ring requires all submitted GPU users. - for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) { + for (u32 slot = 0; slot < state.command_slot_count; ++slot) { if (!WaitForSubmissionLocked(state, slot)) return false; } RefreshCompletedSubmissionsLocked(state); @@ -573,14 +585,16 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) { const u32 slot = state.frame_slot; if (state.frame_submitted[slot]) { - const auto wait_start = std::chrono::steady_clock::now(); + const bool timing_enabled = NativeTextureTimingEnabled(); + const auto wait_start = timing_enabled ? std::chrono::steady_clock::now() + : std::chrono::steady_clock::time_point{}; if (!WaitForSubmissionLocked(state, slot)) return nullptr; RefreshCompletedSubmissionsLocked(state); - const auto wait_ms = std::chrono::duration( + const auto wait_ms = timing_enabled ? std::chrono::duration( std::chrono::steady_clock::now() - wait_start) - .count(); - if (NativeTextureTimingEnabled()) { - ++state.frame_slot_wait_calls; + .count() : 0.0; + ++state.frame_slot_wait_calls; + if (timing_enabled) { state.frame_slot_wait_ms += wait_ms; } if (wait_ms > 20.0 && state.long_wait_log_count++ < 20) { @@ -806,7 +820,7 @@ void HostDevice::UnregisterTexture(u32 descriptor_index) { } if (!g_state->texture_slots[descriptor_index]) return; u32 live_frames = 0; - for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) { + for (u32 slot = 0; slot < g_state->command_slot_count; ++slot) { if (g_state->frame_submitted[slot] || (g_state->command_list_open && slot == g_state->frame_slot)) live_frames |= 1u << slot; @@ -872,7 +886,7 @@ bool HostDevice::Synchronize(SyncReason reason) { const auto wait_start = std::chrono::steady_clock::now(); bool waited = false; - for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) { + for (u32 slot = 0; slot < state.command_slot_count; ++slot) { if (!state.frame_submitted[slot]) { continue; } @@ -967,7 +981,7 @@ bool HostDevice::SubmitRecordedWork() { state.command_list_open = false; state.queue->executeCommandLists(commands, state.frame_fences[slot].get()); MarkSubmissionLocked(state, slot); - state.frame_slot = (slot + 1) % State::kFramesInFlight; + state.frame_slot = (slot + 1) % state.command_slot_count; return true; } @@ -983,7 +997,7 @@ void HostDevice::RetireResource(std::shared_ptr resource) { // The current slot may be unused after SubmitRecordedWork advanced it. // Retiring only there lets BeginFrameCommands destroy resources before an // older submitted list has finished. Keep a reference at EVERY live fence. - for (u32 slot = 0; slot < State::kFramesInFlight; ++slot) { + for (u32 slot = 0; slot < g_state->command_slot_count; ++slot) { if (g_state->frame_submitted[slot] || (g_state->command_list_open && slot == g_state->frame_slot)) g_state->retired_resources[slot].push_back(resource); @@ -1158,7 +1172,13 @@ bool HostDevice::TransferDepthAlias(plume::RenderTexture* source, u32 descriptor bool HostDevice::PresentTexture(plume::RenderTexture* texture, u32 descriptor_index) { auto recording = LockRecording(); - const auto present_start = std::chrono::steady_clock::now(); + const bool timing_enabled = NativeTextureTimingEnabled(); + static const bool metrics_enabled = [] { + const char* path = std::getenv("LEGO_NATIVE_FRAME_METRICS"); + return path && *path; + }(); + const auto present_start = timing_enabled ? std::chrono::steady_clock::now() + : std::chrono::steady_clock::time_point{}; if (!texture || descriptor_index == ~u32{0}) { return false; } @@ -1186,7 +1206,6 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, const u32 slot = state.frame_slot; u32 image_index = 0; - const bool timing_enabled = NativeTextureTimingEnabled(); const auto acquire_start = timing_enabled ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; const bool acquired = state.swap_chain->acquireTexture( @@ -1200,7 +1219,7 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, state.command_list_open = false; state.queue->executeCommandLists(commands, state.frame_fences[slot].get()); MarkSubmissionLocked(state, slot); - state.frame_slot = (slot + 1) % State::kFramesInFlight; + state.frame_slot = (slot + 1) % state.command_slot_count; return false; } @@ -1266,7 +1285,7 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, ReportLogoCapture(); ++state.present_number; g_probe_frame.store(state.present_number); - if (NativeTextureTimingEnabled()) { + if (timing_enabled || metrics_enabled) { const auto now = std::chrono::steady_clock::now(); const auto uploads = ConsumeTextureUploadTiming(); const auto draws = ConsumeDrawTiming(); @@ -1285,7 +1304,9 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, stream << ",callbacks_enqueued,callbacks_executed,callbacks_pending" << ",bindings_ms,begin_ms,pipeline_ms,issue_ms,tail_ms,buffer_converted_bytes" << ",frame_slot_wait_calls,frame_slot_wait_ms,acquire_cpu_ms,present_submit_cpu_ms,swap_present_cpu_ms" - << ",buffer_watch_hits,buffer_watch_audits,buffer_watch_mismatches\n"; + << ",buffer_watch_hits,buffer_watch_audits,buffer_watch_mismatches" + << ",constants_shadow_ms,constants_upload_ms,constants_binding_ms" + << ",constants_texture_binding_ms,constants_restore_ms,detailed_timing_enabled\n"; } return stream; }(); @@ -1293,7 +1314,7 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, frame_metrics << state.present_number << ',' << interval_ms << ',' << uploads.cpu_ms << ',' << draws.calls << ',' << draws.stages_ms[3] << ',' << draws.stages_ms[4] << ',' << buffers.hash_ms << ',' << uploads.hashed_bytes << ',' << buffers.hashed_bytes << ',' - << std::chrono::duration(now - present_start).count() << ',' + << (timing_enabled ? std::chrono::duration(now - present_start).count() : 0.0) << ',' << state.sync_calls << ',' << state.sync_ms << ',' << state.cpu_resource_wait_skips; for (u32 i = 0; i < state.sync_reason_calls.size(); ++i) frame_metrics << ',' << state.sync_reason_calls[i] << ',' << state.sync_reason_ms[i]; @@ -1306,7 +1327,9 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, << ',' << state.acquire_cpu_ms << ',' << state.present_submit_cpu_ms << ',' << state.swap_present_cpu_ms << ',' << buffers.watch_hits << ',' << buffers.watch_audits - << ',' << buffers.watch_mismatches << '\n'; + << ',' << buffers.watch_mismatches; + for (double elapsed : draws.constants_ms) frame_metrics << ',' << elapsed; + frame_metrics << ',' << timing_enabled << '\n'; if (state.present_number % 120 == 0) frame_metrics.flush(); } if (LongProbeEnabled()) LongProbeEvent("frame", interval_ms > 2000.0, @@ -1315,10 +1338,13 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, "converted_bytes=", uploads.converted_bytes, "hashed_bytes=", uploads.hashed_bytes, "draw_calls=", draws.calls, "bindings_ms=", draws.stages_ms[0], "pipeline_ms=", draws.stages_ms[2], "constants_ms=", draws.stages_ms[3], + "constants_shadow_ms=", draws.constants_ms[0], "constants_upload_ms=", draws.constants_ms[1], + "constants_binding_ms=", draws.constants_ms[2], "constants_texture_binding_ms=", draws.constants_ms[3], + "constants_restore_ms=", draws.constants_ms[4], "vertices_ms=", draws.stages_ms[4], "issue_ms=", draws.stages_ms[5], "buffer_hash_ms=", buffers.hash_ms, "buffer_hashed_bytes=", buffers.hashed_bytes, "presented=", presented); - if (interval_ms > 40.0 && (state.timing_log_count++ < 16 || + if (timing_enabled && interval_ms > 40.0 && (state.timing_log_count++ < 16 || state.timing_log_count % 30 == 0)) { REXLOG_INFO("Native timing: frame={} interval={:.2f}ms texture_cpu={:.2f}ms " "calls={} source_hits={} converted={} bytes source={:.2f}ms hash={:.2f}ms hashed={}", @@ -1329,6 +1355,10 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, "pipeline={:.2f} constants={:.2f} vertices={:.2f} issue={:.2f} tail={:.2f}ms", draws.calls, draws.stages_ms[0], draws.stages_ms[1], draws.stages_ms[2], draws.stages_ms[3], draws.stages_ms[4], draws.stages_ms[5], draws.stages_ms[6]); + REXLOG_INFO("Native constants timing: shadow/shared={:.2f} upload={:.2f} " + "descriptor/root={:.2f} textures={:.2f} framebuffer={:.2f}ms", + draws.constants_ms[0], draws.constants_ms[1], draws.constants_ms[2], + draws.constants_ms[3], draws.constants_ms[4]); REXLOG_INFO("Native buffer timing: calls={} hash={:.2f}ms hashed={} converted={} watch_hits={} watch_audits={} watch_mismatches={}", buffers.calls, buffers.hash_ms, buffers.hashed_bytes, buffers.converted_bytes, buffers.watch_hits, buffers.watch_audits, buffers.watch_mismatches); @@ -1354,7 +1384,7 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, } state.snapshot_number = 0; state.snapshot_draw_number = 0; - state.frame_slot = (slot + 1) % State::kFramesInFlight; + state.frame_slot = (slot + 1) % state.command_slot_count; if (!presented) { LongProbeEvent("present_failed", true, "frame=", state.present_number); REXLOG_ERROR("Native GPU: swap-chain present failed"); diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp index 1aea608..c656d58 100644 --- a/rexlego/src/gpu_native/draw.cpp +++ b/rexlego/src/gpu_native/draw.cpp @@ -142,8 +142,10 @@ struct DrawTimer { std::chrono::steady_clock::time_point last; std::array elapsed{}; DrawTimer() { + // Keep workload counts available to the per-present cadence probe even + // when detailed per-draw clocks are disabled. + ++g_draw_timing.calls; if (enabled) { - ++g_draw_timing.calls; last = std::chrono::steady_clock::now(); } } @@ -171,6 +173,25 @@ struct DrawTimer { } }; +enum class ConstantCost : size_t { kShadow, kUpload, kBindings, kTextures, kRestore }; +struct ConstantTimer { + bool enabled = NativeTextureTimingEnabled(); + ConstantCost stage; + std::chrono::steady_clock::time_point last; + explicit ConstantTimer(ConstantCost initial) : stage(initial) { + if (enabled) last = std::chrono::steady_clock::now(); + } + void Next(ConstantCost next) { + if (!enabled) return; + const auto now = std::chrono::steady_clock::now(); + g_draw_timing.constants_ms[static_cast(stage)] += + std::chrono::duration(now - last).count(); + last = now; + stage = next; + } + ~ConstantTimer() { Next(stage); } +}; + bool MapTopology(u32 primitive_type, plume::RenderPrimitiveTopology& result) { using rex::graphics::xenos::PrimitiveType; switch (static_cast(primitive_type)) { @@ -206,6 +227,7 @@ void CopyBigEndianDwords(void* destination, const void* source, u32 size) { bool BindConstants( plume::RenderCommandList* commands, const D3DDevice* device, const DrawBindings& bindings, const VertexDeclarationView& declaration, bool capture_constants) { + ConstantTimer constant_timing(ConstantCost::kUpload); constexpr u32 kConstantsSize = 0x1000; constexpr u32 kSharedOffset = kConstantsSize * 2; DrawUploadSlice upload; @@ -220,6 +242,7 @@ bool BindConstants( upload = {separate_upload.get(), separate_upload->map(), 0}; } else upload = HostDevice::AllocateDrawUpload(kSharedOffset + sizeof(SharedConstants)); if (!upload) return false; + constant_timing.Next(ConstantCost::kShadow); // Build in cacheable CPU memory. Persistently mapped UPLOAD storage is // write-combined and must never be read for diagnostics or partial updates. alignas(16) std::array data; @@ -287,6 +310,7 @@ bool BindConstants( ps[index*4], ps[index*4+1], ps[index*4+2], ps[index*4+3]); } } + constant_timing.Next(ConstantCost::kTextures); for (u32 i = 0; i < kNativeTextureSlots; ++i) { if (!(texture_mask & (1u << i))) continue; auto texture = ResolveTextureResource(bindings.textures[i]); @@ -357,6 +381,7 @@ bool BindConstants( shared.texture_2d[i] = texture.descriptor_index; } } + constant_timing.Next(ConstantCost::kShadow); std::memcpy(mapped + kSharedOffset, &shared, sizeof(shared)); if (capture_constants && HostDevice::LongProbeSnapshotActive()) { const auto name=SaveLongProbeConstants({mapped,kSharedOffset+sizeof(shared)}); @@ -365,7 +390,9 @@ bool BindConstants( "PS=", BoundShaderHash(ShaderStage::kPixel), "texture_mask=", texture_mask, "VS_offset=0 PS_offset=4096 shared_offset=8192 shared_bytes=624"); } + constant_timing.Next(ConstantCost::kUpload); std::memcpy(upload.mapped, data.data(), data.size()); + constant_timing.Next(ConstantCost::kBindings); commands->setGraphicsRootDescriptor(upload.buffer->at(upload.offset), 0); commands->setGraphicsRootDescriptor(upload.buffer->at(upload.offset + kConstantsSize), 1); commands->setGraphicsRootDescriptor(upload.buffer->at(upload.offset + kSharedOffset), 2); @@ -807,22 +834,28 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, } timing.Next(); - commands->setGraphicsPipelineLayout(HostDevice::PipelineLayout()); - if (enable_stencil && (key.depth_control & 1u)) - static_cast(commands)->d3d->OMSetStencilRef( - reinterpret_cast(device)[10499]); - commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 0); - commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 1); - commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 2); - commands->setGraphicsDescriptorSet(HostDevice::SamplerDescriptorSet(), 3); + { + ConstantTimer constant_timing(ConstantCost::kBindings); + commands->setGraphicsPipelineLayout(HostDevice::PipelineLayout()); + if (enable_stencil && (key.depth_control & 1u)) + static_cast(commands)->d3d->OMSetStencilRef( + reinterpret_cast(device)[10499]); + commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 0); + commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 1); + commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 2); + commands->setGraphicsDescriptorSet(HostDevice::SamplerDescriptorSet(), 3); + } auto constants = BindConstants(commands, device, bindings, declaration, count<=6); if (!constants) { return false; } // A raw depth texture view may need a conversion draw while binding textures. // Restore the guest framebuffer after that helper's temporary target. - commands->barriers(plume::RenderBarrierStage::GRAPHICS, texture_barriers.data(), barrier_count); - commands->setFramebuffer(framebuffer); + { + ConstantTimer constant_timing(ConstantCost::kRestore); + commands->barriers(plume::RenderBarrierStage::GRAPHICS, texture_barriers.data(), barrier_count); + commands->setFramebuffer(framebuffer); + } timing.Next(); commands->setPipeline(pipeline); // 83FB82F0 writes normalized R,G,B,A blend constants at +10464..10476. diff --git a/rexlego/src/gpu_native/draw.h b/rexlego/src/gpu_native/draw.h index 5b406ab..4b9ff95 100644 --- a/rexlego/src/gpu_native/draw.h +++ b/rexlego/src/gpu_native/draw.h @@ -10,6 +10,10 @@ struct DrawTiming { uint64_t calls = 0; // Bindings, begin commands, pipeline, constants/textures, vertices, issue, tail. std::array stages_ms{}; + // Subranges of constants/textures: CPU shadow/shared construction, arena + // allocation and mapped copy, descriptor/root binding, texture binding, + // framebuffer restoration. These do not overlap with one another. + std::array constants_ms{}; }; DrawTiming ConsumeDrawTiming(); diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index fd2dd03..29fce96 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -56,7 +56,8 @@ endif() if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") find_package(Python3 COMPONENTS Interpreter QUIET) if(Python3_Interpreter_FOUND) - foreach(check IN ITEMS frame_ring_transaction submission_waits pipeline_failures + foreach(check IN ITEMS command_slots constant_timing frame_cadence + frame_ring_transaction submission_waits pipeline_failures completion_callbacks cpu_resource_wait draw_upload query_retirement) add_test(NAME ${check} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_${check}.py diff --git a/tools/gpu-aot/test_command_slots.py b/tools/gpu-aot/test_command_slots.py new file mode 100644 index 0000000..e9eb822 --- /dev/null +++ b/tools/gpu-aot/test_command_slots.py @@ -0,0 +1,159 @@ +"""Exercise production submission, ring reuse and retirement with 3/12 slots.""" +import argparse +import ast +from pathlib import Path +import subprocess + +from test_frame_ring_transaction import function + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + parser.add_argument("--negative-control", choices=("slot-count", "retirement")) + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + source = (root / "rexlego/src/gpu_native/device.cpp").read_text() + # Share the existing fake-device boundary, while executing new production + # bodies unchanged. These fakes do not measure GPU or presentation speed. + tree = ast.parse((Path(__file__).with_name("test_frame_ring_transaction.py")).read_text()) + harness = next(ast.literal_eval(node.value) for node in ast.walk(tree) + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "harness" + for target in node.targets)) + harness = harness.replace("#include ", "#include \n#include \n#include \n#include \n#include ") + harness = harness.replace('#include "gpu_native/descriptor_retirement.h"', '#include "gpu_native/descriptor_retirement.h"\n#include "gpu_native/completion_queue.h"\nusing legodimensions::gpu_native::CompletionQueue;') + harness = harness.replace("int begins=0;void begin(){++begins;}", + "int begins=0,ends=0;void begin(){++begins;}void end(){++ends;}") + harness = harness.replace(" int waits=0;", " int waits=0,submits=0;\n void executeCommandLists(RenderCommandList*,RenderCommandFence* base){++submits;++static_cast(base)->fenceValue;} ") + start = harness.index("struct State {") + state = harness[start:] + state = state.replace(",3>", ",12>") + state = state.replace("u64 completed_submission=0;", "u64 completed_submission=0,last_submission=0;") + state = state.replace("u64 completed_submission=0,last_submission=0;", "u64 completed_submission=0,last_submission=0;CompletionQueue completion_callbacks;u64 callbacks_enqueued=0,callbacks_executed=0;") + state = state.replace(" State(){for(u32 i=0;i<3;++i){", " State(u32 slots=3):command_slot_count(slots),retired_descriptors(8,slots){for(u32 i=0;i());", " if(i<3)framebuffers.push_back(std::make_unique());") + state = state.replace(" render_semaphores.push_back(std::make_unique());", " if(i<3)render_semaphores.push_back(std::make_unique());") + harness = harness[:start] + state + harness += r''' +std::recursive_mutex recording_mutex; +std::mutex g_mutex; +std::unique_ptr g_state; +constexpr u32 kFirstTextureSlot=0; +struct HostDevice { + static auto LockRecording(){return std::unique_lock(recording_mutex);} + static bool SubmitRecordedWork(); + static void RetireResource(std::shared_ptr); + static void UnregisterTexture(u32); + static bool EnqueueCompletionCallback(std::function); + static void PollCompletionCallbacks(); +}; +void Check(bool ok,const char* message){if(!ok)throw std::runtime_error(message);} +void Environment(const char* value){ +#ifdef _WIN32 + _putenv_s("LEGO_NATIVE_COMMAND_SLOTS",value?value:""); +#else + if(value)setenv("LEGO_NATIVE_COMMAND_SLOTS",value,1);else unsetenv("LEGO_NATIVE_COMMAND_SLOTS"); +#endif +} +''' + bodies = "\n".join(function(source, signature) for signature in ( + "u32 NativeCommandSlotCount(", "bool CreateFrameRing(", + "void RefreshCompletedSubmissionsLocked(", "void MarkSubmissionLocked(", + "bool WaitForSubmissionLocked(", "bool WaitForSubmittedFramesLocked(", + "bool RebuildSwapChain(", "plume::RenderCommandList* BeginFrameCommandsLocked(", + "bool HostDevice::SubmitRecordedWork(", "void HostDevice::RetireResource(", + "void HostDevice::UnregisterTexture(", + "bool HostDevice::EnqueueCompletionCallback(", "void HostDevice::PollCompletionCallbacks(", + )) + if args.negative_control == "slot-count": + bodies = bodies.replace("state.command_slot_count", "State::kFramesInFlight") + tests = r''' +int main(){try{ + Environment(nullptr);Check(NativeCommandSlotCount()==3,"default slots changed"); + Environment("12");Check(NativeCommandSlotCount()==12,"12-slot opt-in rejected"); + for(const char* value:{"3","0","1","32","13","-12","12junk","12 ","012","999999999999999999999999",""}){ + Environment(value);Check(NativeCommandSlotCount()==3,"invalid/default selector changed"); + } + Environment(nullptr); + // Slot11 retirement is checked independently: old three-ticket traversal + // must fail this oracle without executing an out-of-bounds access. + { + DescriptorRetirement retirement(8,12); + Check(!retirement.Retire(3,1u<<11),"descriptor retained through slot11"); + bool freed=false;for(u32 i=0;i<11;++i)retirement.CompleteFrame(i,[&](u32){freed=true;}); + Check(!freed,"descriptor released before slot11"); + retirement.CompleteFrame(11,[&](u32 i){Check(i==3,"wrong descriptor");freed=true;}); + Check(freed,"descriptor never released after slot11"); + } + for(u32 slots:{3u,12u}){ + g_state=std::make_unique(slots);auto& state=*g_state; + state.framebuffers.clear();state.render_semaphores.clear();state.swap_chain->resize_pending=false; + allocation={0,0};Check(CreateFrameRing(state),"creation failed"); + Check(allocation.attempts==int(6+3*slots),"all selected command slots allocated"); + Check(state.framebuffers.size()==3&&state.render_semaphores.size()==3,"swap-chain image count changed"); + for(u32 i=0;iwaits==0,"unused slot waited"); + state.draw_upload_page=7;state.draw_upload_offset=1024; + Check(HostDevice::SubmitRecordedWork(),"submission failed"); + Check(commands->ends==1&&state.last_submission==i+1&&state.frame_submitted[i],"submission order changed"); + } + Check(state.frame_slot==0&&state.queue->submits==int(slots),"ring did not wrap exactly"); + const auto last=state.last_submission;Check(HostDevice::SubmitRecordedWork()&&state.last_submission==last,"empty submit created work"); + auto resource=std::make_shared(17);std::weak_ptr retained=resource; + HostDevice::RetireResource(std::move(resource));HostDevice::UnregisterTexture(3); + int callbacks=0;Check(HostDevice::EnqueueCompletionCallback([&]{++callbacks;}),"callback enqueue failed"); + HostDevice::PollCompletionCallbacks();Check(callbacks==0,"callback ran before final submission"); + for(u32 i=0;iwaits==int(i+1)&&state.completed_submission==i+1,"fence completion order changed"); + Check(!state.frame_submitted[i]&&state.draw_upload_page==0&&state.draw_upload_offset==0,"reuse before fence/reset"); + HostDevice::PollCompletionCallbacks();Check(callbacks==(i+1==slots?1:0),"callback fence ordering changed"); + state.command_list_open=false;state.frame_slot=(i+1)%slots; + } + Check(retained.expired()&&!state.texture_slots[3],"resource/descriptor retained after all fences"); + g_state.reset();Check(Tracked::live==0,"ring resources leaked"); + // Every selected-slot allocation failure leaves prior ring publication intact. + for(int failure=1;failure<=int(6+3*slots);++failure){ + State state(slots);auto* old=state.command_lists[slots-1].get(); + allocation={0,failure};Check(!CreateFrameRing(state),"allocation failure accepted"); + Check(state.command_lists[slots-1].get()==old&&state.framebuffers.size()==3,"partial ring published"); + allocation={0,0};Check(CreateFrameRing(state),"allocation failure was not retryable"); + } + Check(Tracked::live==0,"failure/retry resources leaked"); + } + for(u32 invalid:{0u,33u,UINT32_MAX}){bool rejected=false;try{DescriptorRetirement d(8,invalid);}catch(const std::invalid_argument&){rejected=true;}Check(rejected,"invalid retirement slot count accepted");} + DescriptorRetirement maxslots(8,32);Check(!maxslots.Retire(3,1u<<31),"maximum mask bit lost"); + bool freed=false;maxslots.CompleteFrame(31,[&](u32){freed=true;});Check(freed,"maximum ticket not completed"); + std::cout<<"PASS: production 3/12-slot submission/reuse, exact fence retirement and allocation retries.\n"; + return 0; +}catch(const std::exception& error){std::cerr< +#include +#include +#include +#include +#include "gpu_native/draw.h" +using namespace legodimensions::gpu_native; +using u32=uint32_t; +bool enabled=false; +bool NativeTextureTimingEnabled(){return enabled;} +DrawTiming g_draw_timing; +#define REXLOG_INFO(...) ((void)0) +struct FakeClock { + using time_point=std::chrono::steady_clock::time_point; + static inline int milliseconds=0,calls=0; + static time_point now(){++calls;return time_point{std::chrono::milliseconds(milliseconds)};} +}; +''' + timer + r''' +int main(){ + // A cadence-only probe still counts attempted draws without any draw clocks. + {DrawTimer t;t.Next();t.Next();} + assert(g_draw_timing.calls==1&&FakeClock::calls==0); + for(double v:g_draw_timing.stages_ms)assert(v==0); + enabled=true; + { + DrawTimer t; + for(int i=1;i<=6;++i){FakeClock::milliseconds=i;t.Next();} + FakeClock::milliseconds=7; + } + assert(g_draw_timing.calls==2); + for(double v:g_draw_timing.stages_ms)assert(v==1); + auto draw_frame=std::exchange(g_draw_timing,{}); + assert(draw_frame.calls==2&&g_draw_timing.calls==0); + enabled=false;FakeClock::calls=0;FakeClock::milliseconds=0; + // Disabled instrumentation must not sample the clock or change totals. + {ConstantTimer t(ConstantCost::kUpload);t.Next(ConstantCost::kShadow);} + assert(FakeClock::calls==0); + for(double v:g_draw_timing.constants_ms)assert(v==0); + enabled=true; + { + ConstantTimer t(ConstantCost::kUpload); + FakeClock::milliseconds=3;t.Next(ConstantCost::kShadow); + FakeClock::milliseconds=7;t.Next(ConstantCost::kTextures); + FakeClock::milliseconds=10;t.Next(ConstantCost::kShadow); + FakeClock::milliseconds=12;t.Next(ConstantCost::kUpload); + FakeClock::milliseconds=17;t.Next(ConstantCost::kBindings); + FakeClock::milliseconds=23; + } + assert((g_draw_timing.constants_ms==std::array{6,8,6,3,0})); + // Outer descriptor and framebuffer scopes accumulate into the same frame. + {ConstantTimer t(ConstantCost::kRestore);FakeClock::milliseconds=24;} + {ConstantTimer t(ConstantCost::kBindings);FakeClock::milliseconds=29;} + assert((g_draw_timing.constants_ms==std::array{6,8,11,3,1})); + double total=0;for(double v:g_draw_timing.constants_ms)total+=v; + assert(total==29); // Disjoint ranges account for the entire controlled interval. + // Early exit accounts for the active phase without inventing later work. + {ConstantTimer t(ConstantCost::kUpload);FakeClock::milliseconds=36;} + assert((g_draw_timing.constants_ms==std::array{6,15,11,3,1})); + auto completed=std::exchange(g_draw_timing,{}); + for(double v:g_draw_timing.constants_ms)assert(v==0); + assert(completed.constants_ms[1]==15); + enabled=false;const int calls=FakeClock::calls; + for(int i=0;i<100;++i){ConstantTimer t(ConstantCost::kBindings);t.Next(ConstantCost::kRestore);} + assert(FakeClock::calls==calls); + for(double v:g_draw_timing.constants_ms)assert(v==0); +} +''') +exe = a.output / 'constant-timing.exe' +subprocess.run([a.compiler, '-std=c++20', '-O2', '-I'+str(root/'rexlego/src'), + str(cpp), '-o', str(exe)], check=True, timeout=45) +subprocess.run([str(exe.resolve())], check=True, timeout=10) +print('PASS: production constant timing counts disjoint phases, early exits, reset and disabled clock') diff --git a/tools/gpu-aot/test_draw_upload.py b/tools/gpu-aot/test_draw_upload.py index c9a9c4c..c6744c1 100644 --- a/tools/gpu-aot/test_draw_upload.py +++ b/tools/gpu-aot/test_draw_upload.py @@ -107,7 +107,7 @@ def function(signature): timing_enabled=false; g_state->command_list_open=false;g_state->frame_submitted[0]=true; auto untimed=HostDevice::AllocateDrawUpload(8816);assert(untimed); - assert(g_state->queue->waits==2 && g_state->frame_slot_wait_calls==1); + assert(g_state->queue->waits==2 && g_state->frame_slot_wait_calls==2); // Cheap count survives timing-off. std::memset(reuse.mapped,0xDD,8816);assert(static_cast(other.mapped)[0]==0xCC); auto large=HostDevice::AllocateDrawUpload(5*1024*1024);assert(large&&large.offset==0); assert(g_state->device->allocations==4); diff --git a/tools/gpu-aot/test_frame_cadence.py b/tools/gpu-aot/test_frame_cadence.py new file mode 100644 index 0000000..2b83201 --- /dev/null +++ b/tools/gpu-aot/test_frame_cadence.py @@ -0,0 +1,100 @@ +"""Verify actual frame CSV publication without per-draw detailed timing.""" +import argparse +import csv +from pathlib import Path +import subprocess + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + parser.add_argument("--negative-control", action="store_true") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + source = (root / "rexlego/src/gpu_native/device.cpp").read_text() + start = source.index(" if (timing_enabled || metrics_enabled) {") + end = source.index(" state.snapshot_number = 0;", start) + block = source[start:end] + if args.negative_control: + block = block.replace("if (timing_enabled || metrics_enabled)", "if (timing_enabled)") + cpp = r''' +#include +#include +#include +#include +#include +#include +#include +#include +using u32=uint32_t;using u64=uint64_t; +int detailed_logs=0,draw_consumptions=0,upload_consumptions=0,buffer_consumptions=0; +#define REXLOG_INFO(...) (++detailed_logs) +bool LongProbeEnabled(){return false;} +templatevoid LongProbeEvent(T...){throw std::runtime_error("unexpected trace");} +struct Upload {double cpu_ms=0,source_ms=0,hash_ms=0;u64 calls=0,source_hits=0,converted_bytes=0,hashed_bytes=0;}; +struct Draw {u64 calls=542;std::array stages_ms{};std::array constants_ms{};}; +struct Buffer {double hash_ms=0;u64 calls=0,hashed_bytes=0,converted_bytes=0,watch_hits=0,watch_audits=0,watch_mismatches=0;}; +Upload ConsumeTextureUploadTiming(){++upload_consumptions;return {};} +Draw ConsumeDrawTiming(){++draw_consumptions;return {};} +Buffer ConsumeBufferUploadTiming(){++buffer_consumptions;return {};} +struct State { + u32 present_number=120,timing_log_count=0; + std::chrono::steady_clock::time_point last_present_time=std::chrono::steady_clock::now()-std::chrono::milliseconds(70); + u64 sync_calls=3,cpu_resource_wait_skips=4,frame_slot_wait_calls=8,callbacks_enqueued=42,callbacks_executed=42; + double sync_ms=0,frame_slot_wait_ms=0,acquire_cpu_ms=0,present_submit_cpu_ms=0,swap_present_cpu_ms=0; + std::array sync_reason_calls{};std::array sync_reason_ms{}; + std::vector completion_callbacks; +}; +void Publish(State& state,bool timing_enabled,bool metrics_enabled){ + const auto present_start=std::chrono::steady_clock::time_point{}; + const bool presented=true; +'''+block+r''' +} +int main(int argc,char** argv){try{ +#ifdef _WIN32 + _putenv_s("LEGO_NATIVE_FRAME_METRICS",argv[1]); +#else + setenv("LEGO_NATIVE_FRAME_METRICS",argv[1],1); +#endif + State disabled;Publish(disabled,false,false); + if(draw_consumptions)throw std::runtime_error("disabled metrics consumed counters"); + State cadence;Publish(cadence,false,true); + if(draw_consumptions!=1||upload_consumptions!=1||buffer_consumptions!=1) + throw std::runtime_error("cadence-only CSV not published"); + if(cadence.sync_calls||cadence.frame_slot_wait_calls||cadence.callbacks_enqueued||cadence.callbacks_executed) + throw std::runtime_error("cadence-only counters did not reset"); + if(cadence.last_present_time.time_since_epoch().count()==0||detailed_logs) + throw std::runtime_error("cadence-only detailed logs emitted"); + State detailed;detailed.present_number=240;Publish(detailed,true,false); + if(draw_consumptions!=2||detailed_logs==0)throw std::runtime_error("detailed timing disabled"); + return 0; +}catch(const std::exception& error){std::cerr<= 60 and int(row['draw_calls']) == 542 + assert int(row['frame_slot_wait_calls']) == 8 and int(row['callbacks_enqueued']) == 42 + for key in ('constants_ms', 'vertices_ms', 'buffer_hash_ms', 'present_cpu_ms', 'frame_slot_wait_ms'): + assert float(row[key]) == 0, key + print("PASS: actual frame CSV cadence without detailed timing; counters/reset and explicit timing field.") + + +if __name__ == '__main__': + main() From 8cce273ce702facb15ac92a81a9e4d74196a2fb9 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:20:15 +0200 Subject: [PATCH 15/40] Use the audited TU23 alpha-reference state Read normalized ALPHAREF from offset10500 and initialize its actual integer setter. Offset10620 holds a separate float state and must not drive fragment alpha comparisons. Validation: actual mappedTU23 words, untouched generated leaf setters, gameplay alpha helper, both renderer reads and968 comparison cases passed. Restoring the old offset fails the exact renderer-reference oracle. --- rexlego/src/gpu_native/alpha_test.h | 4 +- rexlego/src/gpu_native/draw.cpp | 6 +- rexlego/src/gpu_native/hooks_device.cpp | 11 +- tools/gpu-aot/test_alpha_test.py | 151 ++++++++++++++++++++---- 4 files changed, 141 insertions(+), 31 deletions(-) diff --git a/rexlego/src/gpu_native/alpha_test.h b/rexlego/src/gpu_native/alpha_test.h index bb5a42b..0e071aa 100644 --- a/rexlego/src/gpu_native/alpha_test.h +++ b/rexlego/src/gpu_native/alpha_test.h @@ -10,7 +10,9 @@ struct NativeAlphaState { inline NativeAlphaState DecodeNativeAlphaState(std::uint32_t color_control, float reference) { // TU23 83FB7E00 / 83FB82C0: enable bit 3, raw Xenos compare bits 0..2. - // 83FB92B8 stores FLOAT BITS unchanged; no integer /255 or /256 mapping. + // Reference is the normalized float shadow at +10500. TU23 83FB8260 + // converts the integer ALPHAREF to float and multiplies by 1/255 there. + // The separate float state written by 83FB92B8 at +10620 is not ALPHAREF. return {(color_control & 8u) != 0, color_control & 7u, reference}; } } // namespace legodimensions::gpu_native diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp index c656d58..1dd9a39 100644 --- a/rexlego/src/gpu_native/draw.cpp +++ b/rexlego/src/gpu_native/draw.cpp @@ -277,7 +277,7 @@ bool BindConstants( } const auto alpha = DecodeNativeAlphaState( *reinterpret_cast(bytes + 10556), - *reinterpret_cast(bytes + 10620)); + *reinterpret_cast(bytes + 10500)); shared.alpha_function = alpha.function; shared.alpha_threshold = alpha.reference; std::fill(std::begin(shared.texture_3d), std::end(shared.texture_3d), 1u); @@ -594,7 +594,7 @@ u32 TraceMeshDraw(D3DDevice* device, u32 primitive, bool indexed, u32(device->vertex_declaration), bindings.index_buffer, bindings.render_targets[0], bindings.depth_stencil, read(10548), read(10496), read(10568), read(10460), read(10552), bindings.textures[5]); REXLOG_INFO("Native mesh: id={} alpha_control={:08X} alpha_ref={}", id, - read(10556), float(*reinterpret_cast(bytes + 10620))); + read(10556), float(*reinterpret_cast(bytes + 10500))); REXLOG_INFO("Native mesh: id={} frame={} brick_pair={}", id, g_probe_frame.load(), brick_trace); for (u32 i = 0; i < bindings.vertex_streams.size(); ++i) { @@ -794,7 +794,7 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, const auto* state_bytes = reinterpret_cast(device); const auto alpha = DecodeNativeAlphaState( *reinterpret_cast(state_bytes + 10556), - *reinterpret_cast(state_bytes + 10620)); + *reinterpret_cast(state_bytes + 10500)); if (alpha.enabled) key.spec_constants |= 2u; // Polygon offset. TU23 D3DRS_SLOPESCALEDEPTHBIAS (83FB88A8) stores the // scale, already in 1/16 subpixel units, at +10832 (front) / +10840 (back); diff --git a/rexlego/src/gpu_native/hooks_device.cpp b/rexlego/src/gpu_native/hooks_device.cpp index 38b77e5..b1acb73 100644 --- a/rexlego/src/gpu_native/hooks_device.cpp +++ b/rexlego/src/gpu_native/hooks_device.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "gpu_native/renderer_route.h" @@ -37,24 +38,24 @@ void CopyStateDispatch(D3DDevice* device, u8* membase) { } bool InitializeRenderDefaults(u32 guest_address, u8* membase) { - // Only statically audited leaf alpha/blend/write-mask setters. Use the actual + // Only statically audited leaf alpha/blend/write-mask/float setters. Use the actual // TU23 descriptor defaults, not guessed reset values or full queue init. - constexpr u32 indices[] = {15,16,18,19,20,21,22,23,24,26,53,54,55,56,59}; + constexpr u32 indices[] = {15,16,18,19,20,21,22,23,24,25,26,53,54,55,56,59}; constexpr u32 setters[] = {0x83FB7E38,0x83FB81C8,0x83FB7F58,0x83FB7FE8, - 0x83FB7EC8,0x83FB80E8,0x83FB8158,0x83FB8078,0x83FB7E00,0x83FB82C0, + 0x83FB7EC8,0x83FB80E8,0x83FB8158,0x83FB8078,0x83FB7E00,0x83FB8260,0x83FB82C0, 0x83FB8A68,0x83FB8AA8,0x83FB8AE8,0x83FB8B28,0x83FB92B8}; const auto* table = reinterpret_cast(membase + kRenderStateTable); auto* kernel_state = REX_KERNEL_STATE(); auto* dispatcher = kernel_state ? kernel_state->function_dispatcher() : nullptr; if (!dispatcher) return false; - for (u32 i = 0; i < 15; ++i) { + for (u32 i = 0; i < std::size(indices); ++i) { const u32 address = table[indices[i] * 3 + 1]; if (address != setters[i] || !dispatcher->GetFunction(address)) { REXLOG_ERROR("Native GPU: unexpected render default setter {:08X}", address); return false; } } - for (u32 i = 0; i < 15; ++i) { + for (u32 i = 0; i < std::size(indices); ++i) { const u32 address = table[indices[i] * 3 + 1]; const u32 value = table[indices[i] * 3 + 2]; rex::ppc::GuestToHostFunction(dispatcher->GetFunction(address), guest_address, value); diff --git a/tools/gpu-aot/test_alpha_test.py b/tools/gpu-aot/test_alpha_test.py index 8c43d0d..e04d95e 100644 --- a/tools/gpu-aot/test_alpha_test.py +++ b/tools/gpu-aot/test_alpha_test.py @@ -3,6 +3,7 @@ No game or window is started. C++ tests share the exact emitted HLSL helper. """ import argparse +import hashlib import json import os from pathlib import Path @@ -18,6 +19,10 @@ p.add_argument('compiler', type=Path) p.add_argument('common', type=Path) p.add_argument('output', type=Path) +p.add_argument('--image', type=Path, help='TU23 decoded image, based at 0x82000000') +p.add_argument('--image-proof', type=Path, help='Selected mapped TU23 words and source provenance JSON') +p.add_argument('--host-compiler', default='clang++') +p.add_argument('--state-only', action='store_true', help='Skip the separate DXIL compiler checks') a = p.parse_args() root = Path(__file__).resolve().parents[2] a.output.mkdir(parents=True, exist_ok=True) @@ -31,16 +36,55 @@ def body(source, sig): end += 1 return source[start:end] -image = (root / 'xexdump/dump/default.bin').read_bytes() -indices = (15,16,18,19,20,21,22,23,24,26,53,54,55,56,59) -table = [struct.unpack_from('>III', image, 0x847F9B18 - 0x82000000 + 12*k) for k in indices] +indices = (15,16,18,19,20,21,22,23,24,25,26,53,54,55,56,59) +proof_record = None +if a.image_proof: + assert not a.image, '--image and --image-proof are mutually exclusive' + proof = json.loads(a.image_proof.read_text()) + assert proof['status'] == 'verified_actual_TU23_mapped_image' + assert proof['image']['base'] == '0x82000000' and proof['game_data_read_only'] + scale = proof['alpha_scale'] + assert scale['guest_address'] == '0x82005D78' + alpha_integer_scale = int(scale['float32_bits'], 16) + assert bytes.fromhex(scale['raw_big_endian_bytes']) == struct.pack('>I', alpha_integer_scale) + descriptors = {d['index']: d for d in proof['render_state_descriptors']} + table = [] + for index in indices: + d = descriptors[index] + assert int(d['guest_address'], 16) == 0x847F9B18 + 12*index + table.append(tuple(int(d[key], 16) for key in ('getter', 'setter', 'default_bits'))) + # Retain and check the proof's actual source identity, rather than inventing + # a synthetic image whose constants happen to satisfy the test. + for record in proof['source_provenance']: + source = root / record['path'] + assert source.stat().st_size == record['bytes'] + digest = hashlib.sha256() + with source.open('rb') as stream: + while chunk := stream.read(1024*1024): + digest.update(chunk) + assert digest.hexdigest() == record['sha256'], source + setter_words = {int(w['guest_address'],16): int(w['word'],16) + for w in proof['selected_guest_PPC_words']['SETTER']} + assert setter_words[0x83FB8278] == 0xC00B5D78 # lfs f0,0x5D78(r11) + assert setter_words[0x83FB8280] == 0xD0032904 # stfs f0,10500(r3) + proof_record = {'path':str(a.image_proof.resolve()), + 'sha256':hashlib.sha256(a.image_proof.read_bytes()).hexdigest(), + 'source_provenance':proof['source_provenance']} +else: + image = (a.image or root / 'xexdump/dump/default.bin').read_bytes() + alpha_integer_scale = struct.unpack_from('>I', image, 0x82005D78 - 0x82000000)[0] + table = [struct.unpack_from('>III', image, 0x847F9B18 - 0x82000000 + 12*k) for k in indices] +assert alpha_integer_scale == 0x3B808081 # Actual float 1/255 used by the integer setter. assert [t[1] for t in table] == [0x83FB7E38,0x83FB81C8,0x83FB7F58,0x83FB7FE8, - 0x83FB7EC8,0x83FB80E8,0x83FB8158,0x83FB8078,0x83FB7E00,0x83FB82C0, + 0x83FB7EC8,0x83FB80E8,0x83FB8158,0x83FB8078,0x83FB7E00,0x83FB8260,0x83FB82C0, 0x83FB8A68,0x83FB8AA8,0x83FB8AE8,0x83FB8B28,0x83FB92B8] -assert [t[2] for t in table] == [0,0,1,0,0,1,0,0,0,7,15,15,15,15,0x3F800000] +assert [t[2] for t in table] == [0,0,1,0,0,1,0,0,0,0,7,15,15,15,15,0x3F800000] functions = [] +game_alpha_helper = None for path in (root / 'rexlego/generated/default').glob('*.cpp'): s = path.read_text() + if 'DEFINE_REX_FUNC(sub_82BCF720)' in s: + game_alpha_helper = body(s, 'DEFINE_REX_FUNC(sub_82BCF720)') for _, setter, _ in table: sig = f'DEFINE_REX_FUNC(sub_{setter:08X})' if sig in s: @@ -54,34 +98,42 @@ def body(source, sig): assert not re.match(r'bctr|bctrl', asm) assert not re.search(r'13928\(r\d+\)|13932\(r\d+\)', f) functions.append(f) -assert len(functions) == 15 +assert len(functions) == 16 and game_alpha_helper initializer = body((root / 'rexlego/src/gpu_native/hooks_device.cpp').read_text(), 'bool InitializeRenderDefaults(') -shared = body((root / 'rexlego/src/gpu_native/draw.cpp').read_text(), 'struct SharedConstants') + ';' +draw_source = (root / 'rexlego/src/gpu_native/draw.cpp').read_text() +shared = body(draw_source, 'struct SharedConstants') + ';' +renderer_alpha_reads = re.findall(r'const auto alpha = (DecodeNativeAlphaState\(\s*\*reinterpret_cast\((?:bytes|state_bytes) \+ \d+\),\s*\*reinterpret_cast\((?:bytes|state_bytes) \+ \d+\)\));', draw_source) +assert len(renderer_alpha_reads) == 2 h = a.output / 'alpha-test.cpp' shared_bytes = 752 if 'fetch_lod_bias[32]' in shared else 624 -h.write_text(r''' +h.write_text((r''' #include +#include #include #include #include #include #include #include +#include #include #include "gpu_native/alpha_test.h" using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using uint=unsigned; #include "alpha_compare.hlsli" union PPCRegister {u64 u64;int64_t s64;u32 u32;int32_t s32;u8 u8;float f32;}; -struct FReg {double f64=0;};struct Fpscr {void disableFlushMode(){}}; +union FReg {double f64=0;uint64_t u64;int64_t s64;};struct Fpscr {void disableFlushMode(){}}; struct XER {u8 ca=0;}; struct CR {bool eq=false,gt=false,lt=false;templatevoid compare(T a,T b,XER){eq=a==b;gt=a>b;lt=a ram{}; -u32 Load(u32 p){return (u32(ram[p])<<24)|(u32(ram[p+1])<<16)|(u32(ram[p+2])<<8)|ram[p+3];} +u32 Load(u32 p){ + if(p==0x82005D78)return ALPHA_INTEGER_SCALE; + if(p==0x84950478)return 12288; + assert(p+3>(24-8*i));} u64 Load64(u32 p){return (u64(Load(p))<<32)|Load(p+4);} void Store64(u32 p,u64 v){Store(p,u32(v>>32));Store(p+4,u32(v));} @@ -93,8 +145,13 @@ def body(source, sig): #define DEFINE_REX_FUNC(name) void name(PPCContext& ctx,u8* base) #define REXLOG_ERROR(...) ((void)0) #define REXLOG_INFO(...) ((void)0) -'''+ '\n'.join(functions) + r''' +'''+ '\n'.join(functions) + '\n' + game_alpha_helper + r''' struct be_u32 {u32 raw;operator u32()const{return __builtin_bswap32(raw);}}; +struct be_f32 {u32 raw;operator float()const{return std::bit_cast(__builtin_bswap32(raw));}}; +using legodimensions::gpu_native::DecodeNativeAlphaState; +'''+ '\n'.join('auto RendererAlpha'+str(i)+r'''(u32 device){ + const auto* bytes=ram.data()+device;const auto* state_bytes=bytes; + return '''+expression+';\n}' for i,expression in enumerate(renderer_alpha_reads)) + r''' constexpr u32 kRenderStateTable=4096; std::vector calls; struct Dispatcher {u32 missing=0;u32 GetFunction(u32 p){return p==missing?0:p;}} dispatcher; @@ -113,7 +170,8 @@ def body(source, sig): constexpr u32 device=12288; '''+''.join(f'Store(kRenderStateTable+12*{k}+4,{t[1]}u);Store(kRenderStateTable+12*{k}+8,{t[2]}u);\n' for k, t in zip(indices, table))+r''' - assert(InitializeRenderDefaults(device,ram.data()));assert(calls.size()==15); + assert(InitializeRenderDefaults(device,ram.data()));assert(calls.size()==16); + assert(Load(device+10500)==0); assert(Load(device+10556)==7&&Load(device+10620)==0x3F800000); assert(Load(device+10568)==0); // Cull/fill untouched. for(u32 at: {10552u,10584u,10588u,10592u})assert(Load(device+at)==0x00010001); @@ -133,24 +191,72 @@ def body(source, sig): assert(std::bit_cast(decoded.reference)==std::bit_cast(reference)); assert(!DecodeNativeAlphaState(f,reference).enabled); } - // Execute original setters for every compare and non-integer reference. + // Execute both original references and actual renderer reads. The unrelated + // +10620 float stays 1, reproducing the native missing-icon state exactly. for(u32 f=0;f<8;++f){ rex::ppc::GuestToHostFunction(0x83FB82C0,device,f); rex::ppc::GuestToHostFunction(0x83FB7E00,device,1); - rex::ppc::GuestToHostFunction(0x83FB92B8,device,std::bit_cast(0.37f)); - auto d=DecodeNativeAlphaState(Load(device+10556),std::bit_cast(Load(device+10620))); - assert(d.enabled&&d.function==f&&d.reference==0.37f); + for(u32 reference:{0u,1u,127u,254u,255u}){ + rex::ppc::GuestToHostFunction(0x83FB8260,device,reference); + auto d=RendererAlpha0(device);auto pipeline=RendererAlpha1(device); + assert(d.enabled&&d.function==f&&d.reference==float(reference)*(1.0f/255.0f)); + assert(pipeline.enabled==d.enabled&&pipeline.function==d.function&&pipeline.reference==d.reference); + assert(Load(device+10620)==0x3F800000); + } } + // Actual gameplay helper clamps integer references and selects comparison. + const u32 functions_by_mode[]={0,4,1,3,2,6,4,5,4}; + for(u32 mode=0;mode<9;++mode)for(u32 value:{0u,1u,254u,255u,1024u}){ + PPCContext c{};c.r1.u32=60000;c.r4.u32=mode;c.r5.u32=value; + sub_82BCF720(c,ram.data()); + auto d=RendererAlpha0(device); + assert(d.enabled==(mode!=1)); + if(mode!=1)assert(d.function==functions_by_mode[mode]); + if(mode>=2)assert(d.reference==float(mode==8?0u:std::min(value,255u))*(1.0f/255.0f)); + } + // Real icon state: GREATER zero keeps opaque UNORM254 alpha and clips zero. + PPCContext icon{};icon.r1.u32=60000;icon.r4.u32=8;icon.r5.u32=0; + sub_82BCF720(icon,ram.data());auto d=RendererAlpha0(device); + assert(d.enabled&&d.function==4&&d.reference==0); + assert(LegoAlphaPass(254.0f/255.0f,d.reference,d.function)); + assert(!LegoAlphaPass(0,d.reference,d.function)); static_assert(sizeof(SharedConstants)==EXPECTED_SHARED_BYTES); static_assert(offsetof(SharedConstants,alpha_function)==600); static_assert(offsetof(SharedConstants,alpha_threshold)==556); static_assert(offsetof(SharedConstants,color_output_scale)==608); } -'''.replace('EXPECTED_SHARED_BYTES', str(shared_bytes))) +''').replace('EXPECTED_SHARED_BYTES', str(shared_bytes)).replace('ALPHA_INTEGER_SCALE',str(alpha_integer_scale)+'u')) exe = a.output / 'alpha-test.exe' -subprocess.run(['clang++', '-std=c++20', '-DNOMINMAX', '-I'+str(root/'rexlego/src'), - '-I'+str(root/'tools/gpu-aot'), str(h), '-o', str(exe)], check=True) -subprocess.run([str(exe.resolve())], check=True) +subprocess.run([a.host_compiler, '-std=c++20', '-DNOMINMAX', '-I'+str(root/'rexlego/src'), + '-I'+str(root/'tools/gpu-aot'), str(h), '-o', str(exe)], check=True,timeout=45) +subprocess.run([str(exe.resolve())], check=True,timeout=10) +if a.state_only: + # Reintroduce exactly the old production read in an isolated fixture. + # It must compile successfully and then fail a renderer reference assertion. + negative = a.output/'alpha-reference-old-offset.cpp' + negative.write_text(h.read_text().replace('bytes + 10500', 'bytes + 10620') + .replace('state_bytes + 10500', 'state_bytes + 10620')) + negative_exe = a.output/'alpha-reference-old-offset.exe' + subprocess.run([a.host_compiler,'-std=c++20','-DNOMINMAX','-I'+str(root/'rexlego/src'), + '-I'+str(root/'tools/gpu-aot'),str(negative),'-o',str(negative_exe)], + check=True,timeout=45) + def disable_core_dumps(): + import resource + resource.setrlimit(resource.RLIMIT_CORE,(0,0)) + rejected = subprocess.run([str(negative_exe.resolve())],capture_output=True,text=True, + timeout=10,preexec_fn=disable_core_dumps if os.name=='posix' else None) + assert rejected.returncode != 0 and 'd.reference==' in rejected.stderr, rejected.stderr + (a.output/'verification.json').write_text(json.dumps({'passed':True, + 'image_proof':proof_record,'alpha_integer_scale':hex(alpha_integer_scale), + 'old_reference_negative_returncode':rejected.returncode, + 'old_reference_negative_assertion':rejected.stderr.strip(), + 'checks':['actual TU23 integer setter and table defaults', + 'actual gameplay helper clamps and compares','both actual renderer reference reads', + 'unrelated float state unchanged','opaque icon alpha passes; transparent alpha fails', + '968 comparisons incl NaN/inf','original wrong10620 read rejected'], + 'game_launched':False},indent=2)+'\n') + print('PASS: TU23 integer alpha reference, gameplay helper, renderer reads and icon state') + raise SystemExit(0) inputs = a.output / 'input' inputs.mkdir(exist_ok=True) @@ -175,7 +281,8 @@ def body(source, sig): 'passed': True, 'actual_tu23_setters': [f'{t[1]:08X}' for t in table], 'table_defaults': [f'{t[2]:08X}' for t in table], 'shared_bytes': shared_bytes, 'checks': ['original leaf setters and native initializer', 'table validation before mutation', - '968 comparisons incl NaN/inf', 'raw float reference', 'no cull/fill writes', + '968 comparisons incl NaN/inf', 'integer alpha reference normalized by actual setter', + 'actual gameplay helper and both renderer reads', 'no cull/fill writes', '5 actual microcode/HLSL/DXIL fixtures', 'depth-only gate', 'compare before EDRAM scale'], 'game_visual_validation': False}, indent=2)) print('PASS: TU23 defaults/setters, eight compares, and five HLSL/DXIL fixtures; no game launched.') From debe406a85fe428bac00d5a71e0656303a78fad8 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:22:29 +0200 Subject: [PATCH 16/40] Add opt-in native culling and bounded draw diagnostics Track guest face culling and winding in pipeline identity, preserve both-face draw/query execution with empty scissor coverage, and select surviving-face stencil and polygon-offset state. Keep culling opt-in. Provide a one-frame CPU draw recorder capped at64 draws and1MiB, with checked guest-memory reads and arithmetic bounds. Disabled diagnostics perform no capture. Validation: actual production raster mapping and untouchedTU23 setter passed GNU fixture; recorder/reader failure and bound fixtures passed clang. Frozen293 Vorton already exercises this raster candidate; campaign-wide equivalence remains unproved. --- rexlego/src/gpu_native/cull_state.h | 33 +++ rexlego/src/gpu_native/draw.cpp | 293 +++++++++++++++++++++-- rexlego/src/gpu_native/draw_diagnostic.h | 130 ++++++++++ tools/gpu-aot/CMakeLists.txt | 2 +- tools/gpu-aot/test_cull_state.py | 226 +++++++++++++++++ tools/gpu-aot/test_draw_diagnostic.py | 244 +++++++++++++++++++ 6 files changed, 911 insertions(+), 17 deletions(-) create mode 100644 rexlego/src/gpu_native/cull_state.h create mode 100644 rexlego/src/gpu_native/draw_diagnostic.h create mode 100644 tools/gpu-aot/test_cull_state.py create mode 100644 tools/gpu-aot/test_draw_diagnostic.py diff --git a/rexlego/src/gpu_native/cull_state.h b/rexlego/src/gpu_native/cull_state.h new file mode 100644 index 0000000..f940106 --- /dev/null +++ b/rexlego/src/gpu_native/cull_state.h @@ -0,0 +1,33 @@ +#pragma once +#include + +namespace legodimensions::gpu_native { +// The low three bits are TU23 PA_SU_SC_MODE_CNTL, not PC D3DCULL values. +// Keep the candidate bit in pipeline identity so legacy state stays distinct. +constexpr std::uint32_t kNativeCullCandidate = 1u << 8; +struct NativeCullState { + std::uint32_t faces = 0; // 0 none, 1 front, 2 back, 3 both. + bool clockwise = false; + bool discard = false; +}; +constexpr NativeCullState DecodeNativeCullState(std::uint32_t control, + bool polygonal) { + const std::uint32_t faces = polygonal ? control & 3u : 0; + return {faces, (control & 4u) != 0, faces == 3}; +} +constexpr std::uint32_t NativeStencilWordOffset(std::uint32_t control, + std::uint32_t depth_control, bool polygonal) { + // D3D12 has a single dynamic reference/mask. Like the SDK, use the back + // values only when two-sided stencil is enabled and only back faces survive. + return (control & kNativeCullCandidate) && polygonal && (depth_control & 128u) && + (control & 3u) == 1u ? 10492u : 10496u; +} +constexpr std::uint32_t CullPolygonOffsetMode(std::uint32_t mode, + std::uint32_t control, bool polygonal) { + if ((control & kNativeCullCandidate) && polygonal) { + if (control & 1u) mode &= ~(1u << 11); + if (control & 2u) mode &= ~(1u << 12); + } + return mode; +} +} // namespace legodimensions::gpu_native diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp index 1dd9a39..03ead98 100644 --- a/rexlego/src/gpu_native/draw.cpp +++ b/rexlego/src/gpu_native/draw.cpp @@ -5,9 +5,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -28,9 +30,11 @@ #include "gpu_native/index_draw.h" #include "gpu_native/alpha_test.h" #include "gpu_native/blend_state.h" +#include "gpu_native/cull_state.h" #include "gpu_native/queries.h" #include "gpu_native/d3d.h" #include "gpu_native/draw.h" +#include "gpu_native/draw_diagnostic.h" #include "gpu_native/depth_state.h" #include "gpu_native/polygon_offset.h" #include "gpu_native/stencil_state.h" @@ -97,6 +101,7 @@ struct PipelineKey { u32 depth_format = 0; u32 depth_control = 0; u32 clip_disable = 0; + u32 raster_control = 0; u32 stencil_masks = 0; u32 stencil_face = 0; u32 color_write_mask = 0; @@ -120,6 +125,7 @@ struct PipelineKeyHash { for (const auto format : key.render_target_formats) mix(format); mix(key.depth_format); mix(key.depth_control); mix(key.color_write_mask); mix(key.clip_disable); mix(key.stencil_masks); mix(key.stencil_face); + mix(key.raster_control); for (const auto blend : key.blend_controls) mix(blend); mix(key.topology); mix(key.spec_constants); mix(u32(key.depth_bias)); mix(key.slope_scaled_depth_bias); @@ -224,6 +230,27 @@ void CopyBigEndianDwords(void* destination, const void* source, u32 size) { } } +void ApplyCullPipeline(plume::RenderGraphicsPipelineDesc& desc, + u32 control, bool polygonal) { + desc.cullMode = plume::RenderCullMode::NONE; + if (!(control & kNativeCullCandidate)) return; + const auto raster = DecodeNativeCullState(control, polygonal); + // Winding also controls SV_IsFrontFace when stencil is disabled. + desc.frontFace = raster.clockwise ? plume::RenderFrontFace::CLOCKWISE + : plume::RenderFrontFace::COUNTER_CLOCKWISE; + if (raster.faces == 1) desc.cullMode = plume::RenderCullMode::FRONT; + else if (raster.faces == 2) desc.cullMode = plume::RenderCullMode::BACK; +} + +plume::RenderRect CullDrawScissor(plume::RenderRect scissor, + u32 control, bool polygonal) { + // D3D12 has no BOTH cull mode. Empty coverage preserves the draw and its + // vertex processing/query scope, while producing no pixel/depth/stencil work. + if ((control & kNativeCullCandidate) && + DecodeNativeCullState(control, polygonal).discard) return {0, 0, 0, 0}; + return scissor; +} + bool BindConstants( plume::RenderCommandList* commands, const D3DDevice* device, const DrawBindings& bindings, const VertexDeclarationView& declaration, bool capture_constants) { @@ -465,7 +492,9 @@ plume::RenderPipeline* GetPipeline( : plume::RenderFrontFace::COUNTER_CLOCKWISE; } desc.primitiveTopology = topology; - desc.cullMode = plume::RenderCullMode::NONE; + ApplyCullPipeline(desc, key.raster_control, + topology == plume::RenderPrimitiveTopology::TRIANGLE_LIST || + topology == plume::RenderPrimitiveTopology::TRIANGLE_STRIP); desc.fillMode = plume::RenderFillMode::SOLID; desc.renderTargetCount = 0; for (u32 i = 0; i < kNativeRenderTargets; ++i) { @@ -527,6 +556,229 @@ plume::RenderPipeline* GetPipeline( return result; } +// Inspect only committed, readable guest spans. Diagnostic reads must not +// introduce a fault for a bad declaration/index or walk an unbounded buffer. +bool ReadDrawDiagnosticBytes(u32 address, void* output, u32 bytes) { + if (!address || !output || !bytes || bytes > 64 * 12 || + u64(address) + bytes > 0x100000000ull) return false; + auto* memory = REX_KERNEL_MEMORY(); + if (!memory) return false; + const u64 end = u64(address) + bytes; + for (u64 at = address; at < end;) { + auto* heap = memory->LookupHeap(u32(at)); + if (!heap) return false; + const u32 size = heap->page_size(), base = heap->heap_base(); + if (!size || (size & (size - 1)) || at < base) return false; + const u32 page = base + ((u32(at) - base) & ~(size - 1)); + rex::memory::HeapAllocationInfo info{}; + if (!heap->QueryRegionInfo(page, &info) || + !(info.state & rex::memory::kMemoryAllocationCommit) || + !(info.protect & rex::memory::kMemoryProtectRead)) return false; + const u64 region_end = std::min(u64(page) + info.region_size, + u64(info.allocation_base) + info.allocation_size); + if (region_end <= at) return false; + at = std::min(end, region_end); + } + std::memcpy(output, memory->TranslateVirtual(address), bytes); + return true; +} + +std::optional ReadDrawDiagnosticDeviceWord(const D3DDevice* device, u32 offset) { + auto* memory = REX_KERNEL_MEMORY(); + if (!device || !memory) return std::nullopt; + const uintptr_t pointer = reinterpret_cast(device); + const uintptr_t base = reinterpret_cast(memory->virtual_membase()); + if (pointer < base || pointer - base > UINT32_MAX) return std::nullopt; + const u64 address = u64(pointer - base) + offset; + be_u32 word{}; + if (address > UINT32_MAX || !ReadDrawDiagnosticBytes(u32(address), &word, sizeof(word))) + return std::nullopt; + return u32(word); +} + +struct CpuDrawDiagnostic { + DrawDiagnosticSession& session; + std::ostringstream record; + DrawBindings bindings; + bool indexed = false, submitted = false; + u32 start = 0, count = 0; + i32 base_vertex = 0; + + CpuDrawDiagnostic(DrawDiagnosticSession& selected, u32 id, + D3DDevice* device, u32 primitive, bool indexed_draw, u32 first, + u32 vertices, i32 base) + : session(selected), bindings(SnapshotDrawBindings()), + indexed(indexed_draw), start(first), count(vertices), base_vertex(base) { + record << "draw=" << id << " attempted_ordinal=" << session.attempted_ordinal() + << " frame=" << g_probe_frame.load() + << " primitive=" << primitive << " indexed=" << indexed + << " start=" << start << " count=" << count << " base=" << base_vertex + << std::hex << " VS=" << BoundShaderHash(ShaderStage::kVertex) + << " PS=" << BoundShaderHash(ShaderStage::kPixel) + << " IB=" << bindings.index_buffer << " RT=" << bindings.render_targets[0] + << " DS=" << bindings.depth_stencil; + if (!device) { record << " device=null\n"; return; } + const auto guest_declaration = ReadDrawDiagnosticDeviceWord(device, + offsetof(D3DDevice, vertex_declaration)); + if (!guest_declaration) { record << " device_fields=unreadable\n"; return; } + bindings.vertex_declaration = *guest_declaration; + record << " decl=" << bindings.vertex_declaration; + constexpr std::pair fields[] = {{"raster", 10568}, {"depth", 10548}, + {"stencil", 10496}, {"alpha", 10556}, {"alpha_ref_bits", 10500}, + {"legacy_float10620_bits", 10620}, + {"blend", 10552}, {"color_mask", 12292}, {"clip_control", 10564}, + {"viewport_control", 10572}, {"viewport_x", 13024}, {"viewport_y", 13028}, + {"viewport_width", 13032}, {"viewport_height", 13036}, + {"viewport_min_z_bits", 13040}, {"viewport_max_z_bits", 13044}, + {"scissor_left", 13052}, {"scissor_top", 13056}, + {"scissor_right", 13060}, {"scissor_bottom", 13064}}; + for (const auto& [name, offset] : fields) { + record << ' ' << name << '='; + if (const auto word = ReadDrawDiagnosticDeviceWord(device, offset)) record << *word; + else record << "unreadable"; + } + record << '\n'; + // Raw guest constants used by the captured UI/world transforms and alpha. + // Read individual words through the same protected guest reader; never + // inspect write-combined host constant uploads. Fixed 48+12-word budget. + constexpr u32 vertex_registers[] = {0, 1, 2, 3, 23, 24, 48, 49, 50, 51, 73, 74}; + const auto constants = [&](const char* name, u32 bank, u32 index) { + record << name << std::dec << index << "=" << std::hex; + for (u32 component = 0; component < 4; ++component) { + if (const auto word = ReadDrawDiagnosticDeviceWord(device, + bank + index * 16 + component * 4)) + record << std::setfill('0') << std::setw(8) << *word; + else record << "unreadable"; + record << ','; + } + record << '\n'; + }; + for (const u32 index : vertex_registers) constants("VS_C", 0x780, index); + for (const u32 index : {4u, 32u, 45u}) constants("PS_C", 0x1780, index); + std::array header{}; + if (!ReadDrawDiagnosticBytes(bindings.vertex_declaration, header.data(), sizeof(header))) { + record << "raw_decl=unreadable\n"; + return; + } + const u32 elements = header[6]; + record << "raw_decl_header=" << LongProbeHex(header) << std::dec + << " raw_count=" << elements << '\n'; + std::array declaration{}; + const u64 address = u64(bindings.vertex_declaration) + 52; + if (elements > 64 || address > UINT32_MAX || + (elements && !ReadDrawDiagnosticBytes(u32(address), declaration.data(), elements * 12))) { + record << "raw_decl=invalid_or_unreadable\n"; + return; + } + record << "raw_decl_bytes=" << std::hex << std::setfill('0'); + for (u32 i = 0; i < elements * 12; ++i) + record << std::setw(2) << u32(declaration[i]); + record << '\n'; + } + + void Layout(const VertexDeclarationView& declaration) { + record << "host_supported=" << declaration.supported + << " swapped_position=" << declaration.swapped_positions + << " swapped_normal=" << declaration.swapped_normals + << " swapped_texcoord=" << declaration.swapped_texcoords + << " sint_texcoord=" << declaration.sint_texcoords + << " reversed_elements=" << declaration.reversed_byte_elements << '\n'; + std::array vertices{start, i64(start) + 1, i64(start) + 2, i64(start) + 3}; + u32 vertex_count = std::min(count, 4u); + if (indexed) { + std::array header{}; + const auto index = ReadDrawDiagnosticBytes(bindings.index_buffer, header.data(), header.size()) + ? InspectBufferResource(bindings.index_buffer, BufferKind::kIndex) : BufferResourceView{}; + const u32 width = index.guest_format == 1 ? 2 : 4; + record << "index_source=" << index.mirror_address << std::dec + << " length=" << index.length << " guest_format=" << index.guest_format + << " first_indices="; + vertex_count = 0; + for (u32 i = 0; i < std::min(count, 6u); ++i) { + const u64 offset = (u64(start) + i) * width; + const u64 address = u64(index.mirror_address) + offset; + std::array raw{}; + if (!index.mirror_address || offset > index.length || width > index.length - offset || + address > UINT32_MAX || !ReadDrawDiagnosticBytes(u32(address), raw.data(), width)) { + record << "unreadable,"; + continue; + } + const u32 value = width == 2 ? u32(raw[0]) : + (u32(raw[0]) << 16) | u32(raw[1]); + record << value << ','; + // Preserve actual corner order, including the fourth quad corner. + // All attribute reads still share the twelve-sample draw budget. + if (i < 4) vertices[vertex_count++] = i64(value) + base_vertex; + } + record << '\n' << std::hex; + } + std::array buffers{}; + std::array inspected{}; + u32 samples_left = 12; + for (u32 i = 0; declaration.elements && i < std::min(declaration.element_count, 75u); ++i) { + const auto& element = declaration.elements[i]; + record << "host_attr=" << element.semanticName << std::dec << element.semanticIndex + << " slot=" << element.slotIndex << " offset=" << element.alignedByteOffset + << " format=" << u32(element.format); + if (element.slotIndex >= kNativeVertexStreams) { record << '\n'; continue; } + const auto& binding = bindings.vertex_streams[element.slotIndex]; + record << " stride=" << binding.stride << " stream_offset=" << binding.offset + << std::hex << " VB=" << binding.buffer << '\n'; + if (!binding.buffer || (std::strcmp(element.semanticName, "POSITION") && + std::strcmp(element.semanticName, "COLOR") && + std::strcmp(element.semanticName, "NORMAL"))) continue; + if (!inspected[element.slotIndex]) { + std::array header{}; + if (ReadDrawDiagnosticBytes(binding.buffer, header.data(), header.size())) + buffers[element.slotIndex] = InspectBufferResource(binding.buffer, BufferKind::kVertex); + inspected[element.slotIndex] = true; + } + const auto& buffer = buffers[element.slotIndex]; + const u32 width = std::min(plume::RenderFormatSize(element.format), 16u); + for (u32 v = 0; v < vertex_count && samples_left; ++v, --samples_left) { + const auto offset = DrawDiagnosticVertexOffset(vertices[v], binding.stride, + binding.offset, element.alignedByteOffset, width, buffer.length); + const u64 address = offset ? u64(buffer.mirror_address) + *offset : UINT64_MAX; + std::array sample{}; + record << "sample_attr=" << element.semanticName << std::dec << element.semanticIndex + << " vertex=" << vertices[v] << std::hex << " source=" << address; + if (!buffer.mirror_address || !offset || address > UINT32_MAX || + !ReadDrawDiagnosticBytes(u32(address), sample.data(), width)) { + record << " bytes=unreadable\n"; + continue; + } + record << " bytes=" << std::setfill('0'); + for (u32 b = 0; b < width; ++b) record << std::setw(2) << u32(sample[b]); + record << '\n'; + } + } + } + + ~CpuDrawDiagnostic() { + record << "submitted=" << submitted << '\n'; + session.Write(record.str()); + } +}; + +std::unique_ptr BeginCpuDrawDiagnostic(D3DDevice* device, + u32 primitive, bool indexed, u32 start, u32 count, i32 base_vertex) { + static DrawDiagnosticSession session([] { + char* value = nullptr; size_t length = 0; + _dupenv_s(&value, &length, "LEGO_NATIVE_DRAW_TRACE_TRIGGER"); + std::filesystem::path result(value ? value : ""); + std::free(value); + return result; + }()); + static const bool enabled = !session.output().empty(); + if (!enabled) return {}; + const u32 id = session.Begin(g_probe_frame.load()); + if (!id) return {}; + if (id == 1) REXLOG_INFO("Native CPU draw trace: capturing one frame, up to 64 draws, output={}", + session.output().string()); + return std::make_unique(session, id, device, primitive, + indexed, start, count, base_vertex); +} + u32 TraceMeshDraw(D3DDevice* device, u32 primitive, bool indexed, u32 start, u32 count, i32 base_vertex) { static const bool enabled = std::getenv("LEGO_NATIVE_MESH_TRACE") != nullptr; @@ -622,6 +874,7 @@ u32 TraceMeshDraw(D3DDevice* device, u32 primitive, bool indexed, bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, u32 start, u32 count, i32 base_vertex) { DrawTimer timing; + auto cpu_trace = BeginCpuDrawDiagnostic(device, primitive_type, indexed, start, count, base_vertex); const u32 mesh_trace = TraceMeshDraw(device, primitive_type, indexed, start, count, base_vertex); // Report early exits as well as successful submissions for each sample. struct MeshTraceResult { @@ -655,6 +908,8 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, } return false; } + const bool polygonal = topology == plume::RenderPrimitiveTopology::TRIANGLE_LIST || + topology == plume::RenderPrimitiveTopology::TRIANGLE_STRIP; DrawBindings bindings = SnapshotDrawBindings(); if (g_logo_capture_api) { static std::set> traced; @@ -670,6 +925,7 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, // the public SetVertexDeclaration hook. bindings.vertex_declaration = device->vertex_declaration; const auto declaration = ResolveVertexDeclaration(bindings.vertex_declaration); + if (cpu_trace) cpu_trace->Layout(declaration); std::array colors; for (u32 i = 0; i < colors.size(); ++i) colors[i] = ResolveTextureResource(bindings.render_targets[i]); @@ -739,6 +995,11 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, key.pixel_shader_hash = BoundShaderHash(ShaderStage::kPixel); key.declaration_hash = declaration.content_hash; key.declaration = bindings.vertex_declaration; + const auto* state_bytes = reinterpret_cast(device); + const u32 guest_raster = *reinterpret_cast(state_bytes + 10568); + static const bool cull_candidate = std::getenv("LEGO_NATIVE_CULL") != nullptr; + if (cull_candidate) + key.raster_control = kNativeCullCandidate | (guest_raster & (polygonal ? 7u : 4u)); for (u32 i = 0; i < colors.size(); ++i) key.render_target_formats[i] = static_cast(colors[i].format); key.depth_format = static_cast(depth.format); @@ -758,17 +1019,18 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, key.depth_control = *reinterpret_cast( reinterpret_cast(device) + 10548); if ((key.depth_control & 1u) && depth.format == plume::RenderFormat::D32_FLOAT_S8_UINT) { - // D3D12 has one mask pair. With native culling disabled, follow Xenia's - // front-face mask choice. Per-face mask differences remain unsupported. + // Select back masks when the candidate culls only front faces; with + // both faces surviving, per-face mask differences remain unsupported. + const u32 word_offset = NativeStencilWordOffset(key.raster_control, + key.depth_control, polygonal); key.stencil_masks = *reinterpret_cast( - reinterpret_cast(device) + 10496) & 0x00FFFF00u; - key.stencil_face = (*reinterpret_cast( - reinterpret_cast(device) + 10568) >> 2) & 1u; + state_bytes + word_offset) & 0x00FFFF00u; + key.stencil_face = (guest_raster >> 2) & 1u; static std::unordered_set logged_stencil; const u64 identity = (u64(key.depth_control) << 32) | key.stencil_masks; if (logged_stencil.size() < 32 && logged_stencil.insert(identity).second) REXLOG_INFO("Native GPU: stencil candidate control={:08X} masks={:06X} ref={} VS={:016X} PS={:016X}", - key.depth_control, key.stencil_masks, reinterpret_cast(device)[10499], + key.depth_control, key.stencil_masks, state_bytes[word_offset + 3], key.vertex_shader_hash, key.pixel_shader_hash); } } @@ -791,7 +1053,6 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, key.depth_control, bindings.depth_stencil); key.topology = static_cast(topology); key.spec_constants = declaration.has_r11g11b10_normal ? 1u : 0u; - const auto* state_bytes = reinterpret_cast(device); const auto alpha = DecodeNativeAlphaState( *reinterpret_cast(state_bytes + 10556), *reinterpret_cast(state_bytes + 10500)); @@ -801,18 +1062,16 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, // D3DRS_DEPTHBIAS (83FB8970) stores the offset at +10836 / +10844; both set // PA_SU_SC_MODE_CNTL (+10568) poly_offset_front/back_enable (bits 11/12). // Convert the stored offset for the host rasterizer. Face selection follows - // SDK's GetPreferredFacePolygonOffset/GetD3D10IntegerPolygonOffset: native - // never culls, so front wins, then back; non-triangles use para_enable. + // SDK's GetPreferredFacePolygonOffset/GetD3D10IntegerPolygonOffset: ignore + // culled faces in the candidate; non-triangles use para_enable. // Unorm24 and float24 guests both come to ~offset * 2^24 host D32 units. // LEGO_NATIVE_NO_DEPTH_BIAS=1 restores the old behaviour for A/B checks. static const bool depth_bias_disabled = std::getenv("LEGO_NATIVE_NO_DEPTH_BIAS") != nullptr; if (!depth_bias_disabled && depth.format != plume::RenderFormat::UNKNOWN) { - const u32 mode = *reinterpret_cast(state_bytes + 10568); + const u32 mode = CullPolygonOffsetMode(guest_raster, key.raster_control, polygonal); const auto poly = [&](u32 offset) { return float(*reinterpret_cast(state_bytes + offset)); }; - const bool polygonal = topology == plume::RenderPrimitiveTopology::TRIANGLE_LIST || - topology == plume::RenderPrimitiveTopology::TRIANGLE_STRIP; const auto bias = DecodePolygonOffset(mode, polygonal, poly(10832), poly(10836), poly(10840), poly(10844)); key.depth_bias = bias.depth_bias; @@ -839,7 +1098,7 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, commands->setGraphicsPipelineLayout(HostDevice::PipelineLayout()); if (enable_stencil && (key.depth_control & 1u)) static_cast(commands)->d3d->OMSetStencilRef( - reinterpret_cast(device)[10499]); + state_bytes[NativeStencilWordOffset(key.raster_control, key.depth_control, polygonal) + 3]); commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 0); commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 1); commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 2); @@ -878,11 +1137,12 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, screen_space ? 1.0f : static_cast(device->viewport.max_z))); const i32 scissor_right = static_cast(device->scissor.right); const i32 scissor_bottom = static_cast(device->scissor.bottom); - commands->setScissors(plume::RenderRect( + commands->setScissors(CullDrawScissor(plume::RenderRect( static_cast(device->scissor.left), static_cast(device->scissor.top), scissor_right > 0 ? scissor_right : static_cast(color.width), - scissor_bottom > 0 ? scissor_bottom : static_cast(color.height))); + scissor_bottom > 0 ? scissor_bottom : static_cast(color.height)), + key.raster_control, polygonal)); std::array views; std::array slots; @@ -1181,6 +1441,7 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, REXLOG_INFO("Native GPU: issued draw primitive={} count={} indexed={} decl=0x{:08X}", primitive_type, count, indexed, bindings.vertex_declaration); mesh_result.submitted = true; + if (cpu_trace) cpu_trace->submitted = true; for (u32 i = 0; i < kNativeRenderTargets; ++i) if ((key.color_write_mask >> (i * 4)) & 15u) MarkSurfaceWritten(bindings.render_targets[i]); if (DecodeDepthState(key.depth_control, depth.texture != nullptr).write) diff --git a/rexlego/src/gpu_native/draw_diagnostic.h b/rexlego/src/gpu_native/draw_diagnostic.h new file mode 100644 index 0000000..3c90c97 --- /dev/null +++ b/rexlego/src/gpu_native/draw_diagnostic.h @@ -0,0 +1,130 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace legodimensions::gpu_native { + +// One trigger selects the first following frame containing a draw. The +// recording lock serializes this CPU-only diagnostic with normal draw calls. +class DrawDiagnosticSession { + public: + static constexpr uint32_t kMaxDraws = 64; + static constexpr size_t kMaxBytes = 1024 * 1024; + static constexpr size_t kMaxRecordBytes = kMaxBytes / kMaxDraws; + + explicit DrawDiagnosticSession(std::filesystem::path trigger = {}) + : trigger_(std::move(trigger)) { + if (!trigger_.empty()) output_ = trigger_.string() + ".draws.tsv"; + } + + static std::optional ParseTrigger(std::string_view value) { + if (!value.empty() && value.back() == '\n') value.remove_suffix(1); + if (!value.empty() && value.back() == '\r') value.remove_suffix(1); + if (value.empty()) return 0; + if (!value.starts_with("skip=")) return std::nullopt; + value.remove_prefix(5); + uint32_t skip = 0; + const auto parsed = std::from_chars(value.data(), value.data() + value.size(), skip); + if (parsed.ec != std::errc{} || parsed.ptr != value.data() + value.size() || skip > 4096) + return std::nullopt; + return skip; + } + + uint32_t Begin(uint32_t frame) { + if (trigger_.empty() || done_) return 0; + if (!armed_) { + if (checked_ && checked_frame_ == frame) return 0; + checked_ = true; + checked_frame_ = frame; + std::error_code error; + if (!std::filesystem::is_regular_file(trigger_, error) || error) return 0; + char text[32]{}; + std::ifstream request(trigger_, std::ios::binary); + if (!request) return 0; + request.read(text, sizeof(text)); + const auto skip = !request.bad() && request.gcount() < std::streamsize(sizeof(text)) + ? ParseTrigger(std::string_view(text, size_t(request.gcount()))) : std::nullopt; + request.close(); // Windows cannot remove an open trigger file. + if (!std::filesystem::remove(trigger_, error) || error || !skip) return 0; + skip_ = *skip; + armed_ = true; + armed_frame_ = frame; + return 0; // A trigger noticed mid-frame must not select a partial frame. + } + if (!active_) { + if (frame == armed_frame_) return 0; + active_ = true; + selected_frame_ = frame; + file_.open(output_, std::ios::binary | std::ios::trunc); + if (!file_) { done_ = true; return 0; } + } + if (frame != selected_frame_ || draws_ >= kMaxDraws) { + done_ = true; + file_.flush(); + return 0; + } + if (++attempts_ <= skip_) return 0; + return ++draws_; + } + + void Write(std::string_view record) { + if (!file_ || record.empty() || records_ >= draws_) return; + ++records_; + // Bound even malformed/unexpected records, rather than relying on the + // expected number of declaration elements or their printed representation. + if (record.size() > kMaxRecordBytes || + record.size() > kMaxBytes - written_) { + constexpr std::string_view message = "record_truncated\n"; + if (message.size() <= kMaxBytes - written_) { + file_.write(message.data(), message.size()); + written_ += message.size(); + } + } else { + file_.write(record.data(), record.size()); + written_ += record.size(); + } + file_.flush(); + if (!file_) done_ = true; + } + + const std::filesystem::path& output() const { return output_; } + size_t written() const { return written_; } + uint32_t attempted_ordinal() const { return attempts_; } + + private: + std::filesystem::path trigger_, output_; + std::ofstream file_; + bool checked_ = false, armed_ = false, active_ = false, done_ = false; + uint32_t checked_frame_ = 0, armed_frame_ = 0, selected_frame_ = 0; + uint32_t draws_ = 0, records_ = 0; + uint32_t skip_ = 0, attempts_ = 0; + size_t written_ = 0; +}; + +// Reject negative/rebased vertices and arithmetic overflow before touching +// guest memory. The requested sample must fit the actual source buffer. +inline std::optional DrawDiagnosticVertexOffset( + int64_t vertex, uint32_t stride, uint32_t stream_offset, + uint32_t element_offset, uint32_t bytes, uint32_t buffer_length) { + if (vertex < 0 || !stride || !bytes || + uint64_t(vertex) > std::numeric_limits::max() / stride) + return std::nullopt; + const uint64_t relative = uint64_t(vertex) * stride; + const uint64_t prefix = uint64_t(stream_offset) + element_offset; + if (relative > std::numeric_limits::max() - prefix) + return std::nullopt; + const uint64_t offset = relative + prefix; + if (offset > buffer_length || bytes > buffer_length - offset) + return std::nullopt; + return uint32_t(offset); +} + +} // namespace legodimensions::gpu_native diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index 29fce96..6421503 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -56,7 +56,7 @@ endif() if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") find_package(Python3 COMPONENTS Interpreter QUIET) if(Python3_Interpreter_FOUND) - foreach(check IN ITEMS command_slots constant_timing frame_cadence + foreach(check IN ITEMS command_slots constant_timing frame_cadence cull_state draw_diagnostic frame_ring_transaction submission_waits pipeline_failures completion_callbacks cpu_resource_wait draw_upload query_retirement) add_test(NAME ${check} diff --git a/tools/gpu-aot/test_cull_state.py b/tools/gpu-aot/test_cull_state.py new file mode 100644 index 0000000..665b680 --- /dev/null +++ b/tools/gpu-aot/test_cull_state.py @@ -0,0 +1,226 @@ +"""Exercise native culling bodies, host winding, cache identity and guest state.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + + +def body(source, signature): + start = source.index(signature) + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('output', type=Path) +p.add_argument('--compiler', default='clang++') +a = p.parse_args() +root = Path(__file__).resolve().parents[2] +a.output.mkdir(parents=True, exist_ok=True) +draw = (root/'rexlego/src/gpu_native/draw.cpp').read_text() +functions = body(draw, 'void ApplyCullPipeline(') + '\n' + body(draw, 'plume::RenderRect CullDrawScissor(') +key = body(draw, 'struct PipelineKey {') + ';\n' + body(draw, 'struct PipelineKeyHash {') + ';\n' +# Both-cull must still reach the ordinary submission and query scope; this is +# not a CPU draw rejection. Run the actual non-indexed submission body below. +nonindex_draw = draw.index(' commands->drawInstanced(count, 1, use_windows ? 0 : start, 0);') +nonindex_begin = draw.rfind(' QueryDrawScope query_scope(commands);', 0, nonindex_draw) +assert nonindex_begin >= 0 +submission = draw[nonindex_begin:] +submission = submission[:submission.index('\n }')] +assert 'CullDrawScissor(' in draw[draw.index('bool DispatchDraw('):] +assert draw.index('commands->setScissors(CullDrawScissor(') < draw.index(submission) +assert 'DecodeNativeCullState(' not in body(draw, 'bool DispatchDraw(') + +plume_types = root/'thirdparty/plume/plume_render_interface_types.h' +plume_d3d = root/'thirdparty/plume/plume_d3d12.cpp' +host_oracle = plume_types.exists() and plume_d3d.exists() +if host_oracle: + types = plume_types.read_text() + enums = body(types, 'enum class RenderCullMode {') + ';\n' + body(types, 'enum class RenderFrontFace {') + ';\n' + rect = body(types, 'struct RenderRect {') + ';\n' + d3d = plume_d3d.read_text() + mapping = body(d3d, 'switch (desc.cullMode) {') + '\n' + face_line = 'psoDesc.RasterizerState.FrontCounterClockwise = desc.frontFace == RenderFrontFace::COUNTER_CLOCKWISE;' + assert face_line in d3d + mapping += face_line +else: + # Standard asset-free checkout may not have initialized this submodule. + # Production culling and key bodies still execute with the narrow fake ABI. + enums = 'enum class RenderCullMode {UNKNOWN,NONE,FRONT,BACK}; enum class RenderFrontFace {UNKNOWN,CLOCKWISE,COUNTER_CLOCKWISE};' + rect = '''struct RenderRect {int32_t left,top,right,bottom; + bool operator==(const RenderRect&) const = default; + bool isEmpty() const {return left>=right||top>=bottom;}};''' + mapping = 'psoDesc.RasterizerState.CullMode = int(desc.cullMode); psoDesc.RasterizerState.FrontCounterClockwise = desc.frontFace == RenderFrontFace::COUNTER_CLOCKWISE;' + +registers_path = root/'rexglue-sdk/include/rex/graphics/registers.h' +guest_oracle = registers_path.exists() +registers = '' +if guest_oracle: + registers = '''namespace xenos {enum class PolygonModeEnable:uint32_t {}; enum class PolygonType:uint32_t {};} + using Register=uint32_t; constexpr Register XE_GPU_REG_PA_SU_SC_MODE_CNTL=0;\n''' + registers += body(registers_path.read_text(), 'union alignas(uint32_t) PA_SU_SC_MODE_CNTL {') + ';\n' + +# When actual game output is present, also execute the untouched TU23 setter. +# Finding its definition is independent of codegen partition numbering. +setter = '' +setter_path = None +signature = 'DEFINE_REX_FUNC(sub_83FB7DA0)' +for path in sorted((root/'rexlego/generated/default').glob('*.cpp')): + with path.open() as source: + if any(line.startswith(signature + ' {') for line in source): + if setter_path is not None: + raise RuntimeError('Duplicate actual guest cull setter definition') + setter_path = path + setter = body(path.read_text(), signature) + +cpp = a.output/'cull-state.cpp' +cpp.write_text(r''' +#include +#include +#include +#include +#include +#include "gpu_native/cull_state.h" +#include "gpu_native/polygon_offset.h" +using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using i32=int32_t; +using namespace legodimensions::gpu_native; +constexpr u32 kNativeRenderTargets=4,kNativeVertexStreams=16; +namespace plume { +''' + enums + '\n' + rect + r''' +struct RenderGraphicsPipelineDesc { + RenderCullMode cullMode=RenderCullMode::UNKNOWN; + RenderFrontFace frontFace=RenderFrontFace::CLOCKWISE; + bool stencilEnabled=false; +}; +} +''' + functions + '\n' + key + registers + r''' +enum {D3D12_CULL_MODE_NONE=1,D3D12_CULL_MODE_FRONT=2,D3D12_CULL_MODE_BACK=3}; +struct PSO {struct {int CullMode=0;bool FrontCounterClockwise=false;} RasterizerState;}; +void MapToActualHost(const plume::RenderGraphicsPipelineDesc& desc,PSO& psoDesc) { + using plume::RenderCullMode;using plume::RenderFrontFace; +''' + mapping + r''' +} +union PPCRegister {uint64_t u64;uint32_t u32;}; +struct PPCContext {PPCRegister r3{},r4{},r11{};}; +std::array ram{}; +u32 Load(u32 p){return u32(ram[p])<<24|u32(ram[p+1])<<16|u32(ram[p+2])<<8|ram[p+3];} +void Store(u32 p,u32 v){for(u32 i=0;i<4;++i)ram[p+i]=u8(v>>(24-8*i));} +u64 Load64(u32 p){return (u64(Load(p))<<32)|Load(p+4);} +void Store64(u32 p,u64 v){Store(p,u32(v>>32));Store(p+4,u32(v));} +#define REX_LOAD_U32(p) Load(p) +#define REX_STORE_U32(p,v) Store(p,v) +#define REX_LOAD_U64(p) Load64(p) +#define REX_STORE_U64(p,v) Store64(p,v) +#define REX_FUNC_PROLOGUE() ((void)0) +#define DEFINE_REX_FUNC(n) void n(PPCContext& ctx,u8* base) +#ifndef __clang__ +// The generated TU23 leaf uses Clang's spelling; std::rotl is the same +// unsigned rotation for fixtures compiled with other GNU-style drivers. +#define __builtin_rotateleft64(value,amount) std::rotl(u64(value),int(amount)) +#endif +''' + setter + r''' +struct Recorder { + plume::RenderRect scissor{1,2,100,200}; + u32 expected_start=2; + int begun=0,ended=0,draws=0,vertices=0,covered=0; + void drawInstanced(u32 count,u32 instances,u32 start,u32 first) { + assert(instances==1&&start==expected_start&&first==0);++draws;vertices+=count; + covered+=scissor.isEmpty()?0:count; + } +}; +struct QueryDrawScope {Recorder* c;explicit QueryDrawScope(Recorder* c):c(c){++c->begun;} + ~QueryDrawScope(){++c->ended;}}; +void SubmitActualNonIndexed(Recorder* commands,u32 count,u32 start,bool use_windows) { +''' + submission + r''' +} +int main(){ + const plume::RenderRect original{1,2,100,200}; + std::unordered_map pipelines; + PipelineKey legacy;pipelines[legacy]=99; + for(u32 guest=0;guest<8;++guest) { +''' + (r''' + PA_SU_SC_MODE_CNTL reg;reg.value=guest; + assert(reg.cull_front==(guest&1)&®.cull_back==((guest>>1)&1)); + assert(reg.face==((guest>>2)&1)); +''' if guest_oracle else '') + (r''' + for(u32 initial:{0u,0xFFFFF8u,0xA3512800u,0xFFFFFFFFu}) { + constexpr u32 dev=4096;Store(dev+10568,initial);Store64(dev+16,0xA010000000000001ULL); + PPCContext ctx;ctx.r3.u32=dev;ctx.r4.u64=0xABCD000000000000ULL|guest; + sub_83FB7DA0(ctx,ram.data()); + assert(Load(dev+10568)==((initial&~7u)|guest)); + assert(Load64(dev+16)==0xA010000000000041ULL); + } +''' if setter else '') + r''' + for(bool polygonal:{false,true})for(bool stencil:{false,true}) { + plume::RenderGraphicsPipelineDesc desc;desc.stencilEnabled=stencil; + desc.frontFace=plume::RenderFrontFace::CLOCKWISE; + ApplyCullPipeline(desc,0,polygonal); + assert(desc.cullMode==plume::RenderCullMode::NONE); + assert(desc.frontFace==plume::RenderFrontFace::CLOCKWISE); + assert(CullDrawScissor(original,0,polygonal)==original); + const u32 control=kNativeCullCandidate|(guest&(polygonal?7u:4u)); + ApplyCullPipeline(desc,control,polygonal); + const u32 faces=polygonal?guest&3u:0u; + assert(desc.frontFace==((guest&4)?plume::RenderFrontFace::CLOCKWISE:plume::RenderFrontFace::COUNTER_CLOCKWISE)); + assert(desc.cullMode==(faces==1?plume::RenderCullMode::FRONT:faces==2?plume::RenderCullMode::BACK:plume::RenderCullMode::NONE)); + PSO host;MapToActualHost(desc,host); + assert(host.RasterizerState.CullMode==(faces==1?2:faces==2?3:1)); + assert(host.RasterizerState.FrontCounterClockwise==!(guest&4)); + for(bool use_windows:{false,true}) { + Recorder recorder;recorder.scissor=CullDrawScissor(original,control,polygonal); + recorder.expected_start=use_windows?0:2; + assert(recorder.scissor.isEmpty()==(faces==3)); + if(faces!=3)assert(recorder.scissor==original); + SubmitActualNonIndexed(&recorder,6,2,use_windows); + assert(recorder.begun==1&&recorder.ended==1&&recorder.draws==1&&recorder.vertices==6); + assert(recorder.covered==(faces==3?0:6)); + } + assert(NativeStencilWordOffset(control,129,polygonal)==(faces==1?10492u:10496u)); + assert(NativeStencilWordOffset(control,1,polygonal)==10496); + assert(NativeStencilWordOffset(0,129,polygonal)==10496); + } + PipelineKey candidate;candidate.raster_control=kNativeCullCandidate|guest; + assert(!(candidate==legacy));assert(PipelineKeyHash{}(candidate)!=PipelineKeyHash{}(legacy)); + pipelines[candidate]=guest; + } + assert(pipelines.size()==9); + for(u32 guest=0;guest<8;++guest){PipelineKey key;key.raster_control=kNativeCullCandidate|guest;assert(pipelines.at(key)==int(guest));} + // Only surviving faces contribute polygon offset. Para offsets are unaffected. + const u32 biasmode=(1u<<11)|(1u<<12)|(1u<<13); + for(u32 faces=0;faces<4;++faces) { + auto bias=DecodePolygonOffset(CullPolygonOffsetMode(biasmode,kNativeCullCandidate|faces,true),true,16,1,32,2); + assert(bias.slope_bits==std::bit_cast(faces==3?0.f:faces==1?2.f:1.f)); + assert(bias.depth_bias==(faces==3?0:faces==1?33554430:16777215)); + auto legacybias=DecodePolygonOffset(CullPolygonOffsetMode(biasmode,faces,true),true,16,1,32,2); + assert(legacybias.depth_bias==16777215); + auto pointbias=DecodePolygonOffset(CullPolygonOffsetMode(biasmode,kNativeCullCandidate|faces,false),false,16,1,32,2); + assert(pointbias.depth_bias==16777215); + } + // Distinct reference/mask bytes survive selection of the actual guest word. + Store(10492,0x00AABB11);Store(10496,0x00CCDD22); + auto back=NativeStencilWordOffset(kNativeCullCandidate|1,129,true); + auto front=NativeStencilWordOffset(kNativeCullCandidate|2,129,true); + assert((Load(back)&0xFFFF00)==0xAABB00&&ram[back+3]==0x11); + assert((Load(front)&0xFFFF00)==0xCCDD00&&ram[front+3]==0x22); +} +''') +exe = a.output/'cull-state.exe' +subprocess.run([a.compiler, '-std=c++20', '-O2', '-UNDEBUG', '-I', str(root/'rexlego/src'), + str(cpp), '-o', str(exe)], check=True, timeout=45) +subprocess.run([str(exe)], check=True, timeout=10) +proof = {'passed': True, 'candidate': 'LEGO_NATIVE_CULL environment presence', + 'raster_cases': 32, 'actual_production_bodies': 5, + 'actual_plume_d3d12_mapping': host_oracle, 'actual_sdk_register_layout': guest_oracle, + 'actual_tu23_setter_cases': 32 if setter else 0, + 'setter_source': str(setter_path) if setter_path else None, + 'setter_sha256': hashlib.sha256(setter.encode()).hexdigest() if setter else None, + 'both_cull': 'Empty scissor; actual nonindexed submission/query body still runs', + 'nonindexed_window_modes': 'Both disabled original start and enabled rebased start execute actual submission', + 'scope': 'CPU state/winding/identity/submission contract; no GPU or visual repair claim'} +(a.output/'verification.json').write_text(json.dumps(proof, indent=2)+'\n') +print('PASS: cull/winding mapping, points/lines, both-face query submission, stencil/bias and nine distinct PSOs') diff --git a/tools/gpu-aot/test_draw_diagnostic.py b/tools/gpu-aot/test_draw_diagnostic.py new file mode 100644 index 0000000..de2d6bd --- /dev/null +++ b/tools/gpu-aot/test_draw_diagnostic.py @@ -0,0 +1,244 @@ +"""Exercise the real bounded draw diagnostic reader, recorder and trigger.""" +import argparse +from pathlib import Path +import subprocess +import tempfile + + +def function(source, signature): + start = source.index(signature) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + root = Path(__file__).resolve().parents[2] + source = (root / "rexlego/src/gpu_native/draw.cpp").read_text() + reader = function(source, "bool ReadDrawDiagnosticBytes(") + device_reader = function(source, "std::optional ReadDrawDiagnosticDeviceWord(") + recorder = function(source, "struct CpuDrawDiagnostic {") + ";" + # Check that the production integration reaches the recorder before draw + # rejection, publishes the real layout, and sets success only at submission. + dispatch = function(source, "bool DispatchDraw(") + assert dispatch.index("BeginCpuDrawDiagnostic(") < dispatch.index("ShouldSkipBoundPixelShader()") + assert "if (cpu_trace) cpu_trace->Layout(declaration);" in dispatch + assert dispatch.index("cpu_trace->submitted = true") > dispatch.index("drawIndexedInstanced(") + harness = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "gpu_native/draw_diagnostic.h" +using namespace legodimensions::gpu_native; +using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using i32=int32_t;using i64=int64_t; +template struct BigEndian { + std::array data{}; + BigEndian& operator=(T value) { + for(size_t i=0;i>(i*8));return *this; + } + operator T() const {T value=0;for(auto b:data)value=T((value<<8)|b);return value;} +}; +using be_u16=BigEndian;using be_u32=BigEndian; +namespace rex::memory { +constexpr u32 kMemoryAllocationCommit=1,kMemoryProtectRead=1; +struct HeapAllocationInfo {u32 allocation_base=0,allocation_size=0,region_size=0,state=0,protect=0;}; +} +struct Heap { + u32 size=4096,base=0x10000,readable_pages=1; + u32 page_size()const{return size;}u32 heap_base()const{return base;} + bool QueryRegionInfo(u32 page,rex::memory::HeapAllocationInfo* info) { + assert(page>=base&&(page-base)%size==0); + if(page>=uint64_t(base)+2*size)return false; + *info={base,2*size,size,1,u32((page-base)/size bytes=std::vector(8192);u32 translations=0; + Heap* LookupHeap(u32 address) { + return address>=heap.base&&uint64_t(address)T TranslateVirtual(u32 address) { + ++translations;assert(address>=heap.base);return reinterpret_cast(bytes.data()+address-heap.base); + } + u8* virtual_membase(){return reinterpret_cast(reinterpret_cast(bytes.data())-heap.base);} + void Configure(u32 size,u32 base) {heap={size,base,1};bytes.assign(2*size,0);translations=0;} + void Word(u32 offset,u32 value) {for(u32 i=0;i<4;++i)bytes[offset+i]=u8(value>>(24-i*8));} +} fixture_memory; +#define REX_KERNEL_MEMORY() (&::fixture_memory) +''' + reader + r''' +constexpr u32 kNativeVertexStreams=16; +enum class BufferKind{kVertex,kIndex}; +struct BufferResourceView {void* buffer=nullptr;u32 length=0,guest_format=0,mirror_address=0;}; +struct Stream {u32 buffer=0,offset=0,stride=0;}; +struct DrawBindings {u32 vertex_declaration=0,index_buffer=0,depth_stencil=0; + std::array render_targets{};std::array vertex_streams{};}; +struct D3DBuffer {u8 bytes[32];}; +struct D3DDevice {u8 prefix[0x2FD0];be_u32 vertex_declaration;u8 suffix[0x6080-0x2FD4];}; +namespace plume { +enum class RenderFormat:u32 {FLOAT3=1,COLOR=2}; +u32 RenderFormatSize(RenderFormat format){return format==RenderFormat::FLOAT3?12:4;} +struct RenderInputElement {const char* semanticName;u32 semanticIndex,slotIndex,alignedByteOffset;RenderFormat format;}; +} +struct VertexDeclarationView {const plume::RenderInputElement* elements=nullptr;u32 element_count=0; + bool supported=true;u32 swapped_positions=0,swapped_normals=0,swapped_texcoords=0,sint_texcoords=0; + u64 reversed_byte_elements=0;}; +DrawBindings bindings; +DrawBindings SnapshotDrawBindings(){return bindings;} +enum class ShaderStage{kVertex,kPixel}; +u64 BoundShaderHash(ShaderStage stage){return stage==ShaderStage::kVertex?0x1234:0xabcd;} +std::atomic g_probe_frame{8}; +std::map metadata; +BufferResourceView InspectBufferResource(u32 address,BufferKind){return metadata.at(address);} +templatestd::string LongProbeHex(const Words& words) { + std::ostringstream s;s<(f),{}};} +int main(int argc,char**argv){ + assert(argc==2);const std::filesystem::path directory=argv[1]; + for(u32 size:{4096u,65536u,16777216u}) { + // A non-page-aligned heap base catches incorrect absolute/4KiB alignment. + fixture_memory.Configure(size,0x11000); + std::array sample{};const auto base=fixture_memory.heap.base; + fixture_memory.bytes[size-1]=0x71; + assert(ReadDrawDiagnosticBytes(base+size-1,sample.data(),1)&&sample[0]==0x71); + const u32 translated=fixture_memory.translations; + assert(!ReadDrawDiagnosticBytes(base+size-1,sample.data(),2)); + assert(!ReadDrawDiagnosticBytes(base+size,sample.data(),1)); + assert(fixture_memory.translations==translated); + fixture_memory.heap.readable_pages=2; + assert(ReadDrawDiagnosticBytes(base+size-1,sample.data(),2)); + } + std::array sample{}; + assert(!ReadDrawDiagnosticBytes(0,sample.data(),1)); + assert(!ReadDrawDiagnosticBytes(0xFFFFFFF8,sample.data(),16)); + assert(!ReadDrawDiagnosticBytes(fixture_memory.heap.base,sample.data(),769)); + assert(!ReadDrawDiagnosticBytes(fixture_memory.heap.base,nullptr,1)); + assert(!ReadDrawDiagnosticBytes(fixture_memory.heap.base,sample.data(),0)); + assert(DrawDiagnosticVertexOffset(2,16,4,8,4,48)==44); + assert(!DrawDiagnosticVertexOffset(-1,16,0,0,4,48)); + assert(!DrawDiagnosticVertexOffset(INT64_MAX,UINT32_MAX,0,0,4,UINT32_MAX)); + assert(!DrawDiagnosticVertexOffset(INT64_MAX,2,UINT32_MAX,UINT32_MAX,4,UINT32_MAX)); + assert(!DrawDiagnosticVertexOffset(2,16,4,8,5,48)); + assert(!DrawDiagnosticVertexOffset(0,0,0,0,4,48)); + assert(!DrawDiagnosticVertexOffset(0,16,0,0,0,48)); + using S=DrawDiagnosticSession; + assert(S::ParseTrigger("")==0&&S::ParseTrigger("skip=0")==0); + assert(S::ParseTrigger("skip=100\r\n")==100&&S::ParseTrigger("skip=4096")==4096); + for(std::string_view invalid:{"skip=","skip=-1","skip=+1","skip=4097","skip=1x", + "skip=1 "," skip=1","skip=42949672960","skip=1\n\n","100"})assert(!S::ParseTrigger(invalid)); + const auto disabled_trigger=directory/"disabled";Touch(disabled_trigger); + S disabled;assert(!disabled.Begin(1));assert(std::filesystem::exists(disabled_trigger)); + const auto trigger=directory/"draw trigger";S session(trigger); + assert(!session.Begin(7));Touch(trigger);assert(!session.Begin(7)); // no second poll/frame + assert(!session.Begin(8));assert(!std::filesystem::exists(trigger));assert(!session.Begin(8)); + for(u32 i=1;i<=64;++i){assert(session.Begin(9)==i);session.Write("draw\n");} + assert(!session.Begin(9));Touch(trigger);assert(!session.Begin(10)); + assert(std::filesystem::exists(trigger));assert(session.written()==64*5); + const auto skip_trigger=directory/"late";Touch(skip_trigger,"skip=100\n");S late(skip_trigger); + assert(!late.Begin(UINT32_MAX)); + for(u32 i=0;i<100;++i)assert(!late.Begin(0)); + assert(late.Begin(0)==1&&late.attempted_ordinal()==101);late.Write("late\n"); + assert(!late.Begin(1)); // cannot spill into another frame + const auto bad_trigger=directory/"bad";Touch(bad_trigger,std::string(40,'0'));S bad(bad_trigger); + assert(!bad.Begin(1));assert(!bad.Begin(2)); // truncated input must not arm + Touch(bad_trigger,"skip=0");assert(!bad.Begin(3));assert(bad.Begin(4)==1);bad.Write("valid\n"); + const auto budget_trigger=directory/"budget";Touch(budget_trigger);S budget(budget_trigger); + assert(!budget.Begin(1)); + const std::string block(S::kMaxRecordBytes,'x'); + for(u32 i=1;i<=64;++i){assert(budget.Begin(2)==i);budget.Write(block);budget.Write(block);} + assert(budget.written()==S::kMaxBytes); + assert(std::filesystem::file_size(budget.output())==S::kMaxBytes); + const auto large_trigger=directory/"large";Touch(large_trigger);S large(large_trigger); + assert(!large.Begin(1));assert(large.Begin(2)==1);large.Write(block+"x"); + assert(Read(large.output())=="record_truncated\n"); + // Feed actual production recorder known big-endian indices/position/color. + fixture_memory.Configure(65536,0x10000); + auto* device=fixture_memory.TranslateVirtual(0x12000); + fixture_memory.Word(0x2000+offsetof(D3DDevice,vertex_declaration),0x10080); + fixture_memory.Word(0x2000+10564,0x90000);fixture_memory.Word(0x2000+10572,0x400); + fixture_memory.Word(0x2000+10500,0x3b808081);fixture_memory.Word(0x2000+10620,0x3f800000); + fixture_memory.Word(0x2000+13032,1280);fixture_memory.Word(0x2000+13036,720); + fixture_memory.Word(0x2000+0x780+48*16,0x3f800000); + fixture_memory.Word(0x2000+0x1780+4*16+12,0x3f000000); + fixture_memory.Word(0x2000+0x1780+45*16,0x3f800000); + fixture_memory.Word(128+24,2);for(u32 i=0;i<24;++i)fixture_memory.bytes[128+52+i]=u8(i); + fixture_memory.bytes[1024]=0;fixture_memory.bytes[1025]=3; + fixture_memory.bytes[1026]=0;fixture_memory.bytes[1027]=1; + fixture_memory.bytes[1028]=0;fixture_memory.bytes[1029]=2; + bindings={};bindings.index_buffer=0x10200;bindings.vertex_streams[0]={0x10300,4,16}; + metadata[0x10200]={nullptr,6,1,0x10400};metadata[0x10300]={nullptr,128,0,0x10500}; + // base -1 + index3 => vertex2; offset4 + 2*16 =36, color offset+12. + fixture_memory.Word(1280+36,0x3f800000);fixture_memory.Word(1280+48,0xff112233); + const plume::RenderInputElement elements[]={{"POSITION",0,0,0,plume::RenderFormat::FLOAT3}, + {"COLOR",0,0,12,plume::RenderFormat::COLOR}}; + VertexDeclarationView decl;decl.elements=elements;decl.element_count=2; + const auto recorder_trigger=directory/"recorder";Touch(recorder_trigger);S trace(recorder_trigger); + assert(!trace.Begin(7));const u32 id=trace.Begin(8);assert(id); + {CpuDrawDiagnostic draw(trace,id,device,6,true,0,3,-1);draw.Layout(decl);draw.submitted=true;} + auto text=Read(trace.output()); + for(std::string_view expected:{"attempted_ordinal=1","VS=1234 PS=abcd","raw_count=2", + "raw_decl_bytes=000102030405060708090a0b0c0d0e0f1011121314151617", + "clip_control=90000 viewport_control=400", "viewport_width=500 viewport_height=2d0", + "alpha_ref_bits=3b808081 legacy_float10620_bits=3f800000", + "VS_C48=3f800000,00000000,00000000,00000000,", + "PS_C4=00000000,00000000,00000000,3f000000,", + "PS_C45=3f800000,00000000,00000000,00000000,", + "first_indices=3,1,2,","host_attr=COLOR0","vertex=2 source=10524 bytes=3f800000", + "vertex=2 source=10530 bytes=ff112233","submitted=1"})assert(text.find(expected)!=std::string::npos); + // Rejected/invalid indices must not cause fabricated vertex samples. + metadata[0x10200].mirror_address=0x20000; + {CpuDrawDiagnostic draw(trace,trace.Begin(8),device,6,true,0,3,-1);draw.Layout(decl);} + text=Read(trace.output());assert(text.find("first_indices=unreadable,unreadable,unreadable,")!=std::string::npos); + assert(text.find("submitted=0")!=std::string::npos); + // The fourth non-indexed quad corner is essential to distinguish perimeter + // from strip ordering. Capture it through the real recorder with the same + // twelve-sample cap; index/base handling above remains independently tested. + fixture_memory.Word(1280+52,0x3f000000);fixture_memory.Word(1280+56,0xbf000000); + fixture_memory.Word(1280+60,0);fixture_memory.Word(1280+64,0xffaabbcc); + {CpuDrawDiagnostic draw(trace,trace.Begin(8),device,6,false,0,4,0);draw.Layout(decl);} + text=Read(trace.output()); + assert(text.find("vertex=3 source=10534 bytes=3f000000bf00000000000000")!=std::string::npos); + assert(text.find("vertex=3 source=10540 bytes=ffaabbcc")!=std::string::npos); + // Non-null device pointers at a protected boundary, outside the guest + // range, and zero-count early rejects must never be dereferenced by capture. + auto* truncated=fixture_memory.TranslateVirtual(0x1FFF8); + const u32 translated=fixture_memory.translations; + {CpuDrawDiagnostic draw(trace,trace.Begin(8),truncated,6,false,0,0,0);} + assert(fixture_memory.translations==translated); + auto* outside=reinterpret_cast(reinterpret_cast(fixture_memory.virtual_membase())-1); + {CpuDrawDiagnostic draw(trace,trace.Begin(8),outside,6,false,0,0,0);} + assert(fixture_memory.translations==translated); + text=Read(trace.output());assert(text.find("device_fields=unreadable")!=std::string::npos); + std::cout<<"PASS: real CPU draw recorder/reader; one frame/64 attempts/1MiB; late selection; heap/protection/overflow bounds\n"; +} +''' + cpp = args.output / "draw-diagnostic.cpp" + exe = args.output / "draw-diagnostic.exe" + cpp.write_text(harness) + subprocess.run([args.compiler, "-std=c++20", "-UNDEBUG", "-I" + str(root / "rexlego/src"), + str(cpp), "-o", str(exe)], check=True, timeout=45) + cases = tempfile.mkdtemp(prefix="case-", dir=args.output.resolve()) + subprocess.run([str(exe.resolve()), cases], check=True, timeout=10) + + +if __name__ == "__main__": + main() From 4f251df22fbcfc74dace19319223dd9e6bd27594 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:26:25 +0200 Subject: [PATCH 17/40] Add fence-gated native GPU timestamp diagnostics Record opt-in per-submission queue elapsed spans for every command close path. Reuse existing fences, check actual queue frequency and raw readback, consume each pair once, and disable diagnostics gracefully on failure. Keep the sidecar separate from CPU frame cadence. Validation:10 staged-source CTest checks passed, including34 harness cases and production timestamp/fence/resize bodies; premature readback and duplicate-generation negative controls rejected. Native execution remains the next validation gate. --- rexlego/src/gpu_native/device.cpp | 43 ++++ rexlego/src/gpu_native/gpu_timestamps.h | 199 +++++++++++++++ tools/gpu-aot/CMakeLists.txt | 5 + tools/gpu-aot/regression/README.md | 8 + tools/gpu-aot/regression/compare_samples.py | 3 + tools/gpu-aot/regression/prepare_run.py | 15 ++ tools/gpu-aot/regression/test_regression.py | 37 +++ tools/gpu-aot/test_command_slots.py | 2 + tools/gpu-aot/test_completion_callbacks.py | 6 + tools/gpu-aot/test_draw_upload.py | 6 + tools/gpu-aot/test_frame_ring_transaction.py | 6 + tools/gpu-aot/test_gpu_timestamps.py | 249 +++++++++++++++++++ tools/gpu-aot/test_resize_fences.py | 6 + tools/gpu-aot/test_submission_waits.py | 6 + 14 files changed, 591 insertions(+) create mode 100644 rexlego/src/gpu_native/gpu_timestamps.h create mode 100644 tools/gpu-aot/test_gpu_timestamps.py diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp index 7f1a8ff..5eff88b 100644 --- a/rexlego/src/gpu_native/device.cpp +++ b/rexlego/src/gpu_native/device.cpp @@ -2,6 +2,7 @@ #include "gpu_native/descriptor_retirement.h" #include "gpu_native/completion_queue.h" #include "gpu_native/present_rect.h" +#include "gpu_native/gpu_timestamps.h" #include "gpu_native/buffers.h" #include "gpu_native/queries.h" @@ -179,6 +180,7 @@ struct State { std::array frame_submitted{}; std::array slot_submission{}; u64 last_submission = 0, completed_submission = 0; + std::unique_ptr gpu_timestamps; CompletionQueue completion_callbacks; u64 callbacks_enqueued = 0, callbacks_executed = 0; std::array>, kMaxCommandSlots> draw_uploads; @@ -512,6 +514,10 @@ void RefreshCompletedSubmissionsLocked(State& state) { if (!state.frame_submitted[slot]) continue; auto* fence = static_cast(state.frame_fences[slot].get()); const u64 completed = fence->d3d->GetCompletedValue(); + if (state.gpu_timestamps) + state.gpu_timestamps->Complete(slot, completed, + fence->fenceValue > 1 ? fence->fenceValue - 1 : 0, + state.present_number); // Plume signals fenceValue, then increments it. UINT64_MAX is removal, // never successful completion. Do not consume its auto-reset wait event. if (completed != UINT64_MAX && fence->fenceValue > 1 && @@ -525,6 +531,8 @@ void RefreshCompletedSubmissionsLocked(State& state) { void MarkSubmissionLocked(State& state, u32 slot) { state.frame_submitted[slot] = true; state.slot_submission[slot] = ++state.last_submission; + if (state.gpu_timestamps) + state.gpu_timestamps->Submitted(slot, state.slot_submission[slot]); } bool WaitForSubmissionLocked(State& state, u32 slot) { @@ -532,6 +540,10 @@ bool WaitForSubmissionLocked(State& state, u32 slot) { if (state.backend == Backend::kD3D12) { auto* fence = static_cast(state.frame_fences[slot].get()); if (fence->d3d->GetCompletedValue() == UINT64_MAX) { + if (state.gpu_timestamps) + state.gpu_timestamps->Complete(slot, UINT64_MAX, + fence->fenceValue > 1 ? fence->fenceValue - 1 : 0, + state.present_number); REXLOG_ERROR("Native GPU: cannot wait for submission after device removal"); return false; } @@ -540,6 +552,10 @@ bool WaitForSubmissionLocked(State& state, u32 slot) { if (state.backend == Backend::kD3D12) { auto* fence = static_cast(state.frame_fences[slot].get()); const u64 completed = fence->d3d->GetCompletedValue(); + if (state.gpu_timestamps) + state.gpu_timestamps->Complete(slot, completed, + fence->fenceValue > 1 ? fence->fenceValue - 1 : 0, + state.present_number); if (completed == UINT64_MAX || completed < fence->fenceValue - 1) { REXLOG_ERROR("Native GPU: submission GPU wait failed"); return false; @@ -610,6 +626,8 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) { [&](u32 index) { state.texture_slots[index] = false; }); auto* commands = state.command_lists[slot].get(); commands->begin(); + if (state.gpu_timestamps) + state.gpu_timestamps->Begin(commands, slot, u64(state.present_number) + 1); InitializeNullTextures(state, commands); state.command_list_open = true; return commands; @@ -688,6 +706,19 @@ bool HostDevice::Create(rex::ui::Window* window, Backend backend) { } const auto& description = state->device->getDescription(); + if (backend == Backend::kD3D12) { + const char* path = std::getenv("LEGO_NATIVE_GPU_TIMESTAMPS"); + if (path && *path) { + state->gpu_timestamps = GpuSubmissionTimestamps::Create( + state->device.get(), state->queue.get(), state->command_slot_count, + path, kNativeGpuBuildFingerprint); + if (!state->gpu_timestamps) + REXLOG_WARN("Native GPU: timestamp diagnostic unavailable (allocation, frequency or output failure)"); + else + REXLOG_INFO("Native GPU: per-submission GPU timestamps enabled, frequency={} Hz (queue elapsed, not GPU utilization or whole-frame time)", + state->gpu_timestamps->Frequency()); + } + } REXLOG_INFO("Native GPU [{}]: {} ready on '{}' ({}x{}, {} images)", kNativeGpuBuildFingerprint, NameOf(backend), description.name, state->swap_chain->getWidth(), state->swap_chain->getHeight(), @@ -705,6 +736,8 @@ void HostDevice::Shutdown() { return; } WaitForSubmittedFramesLocked(*g_state); + if (g_state->gpu_timestamps) + g_state->gpu_timestamps->Shutdown(g_state->present_number); if (g_state->completion_callbacks.size()) REXLOG_INFO("Native GPU: cancelling {} completion callbacks at title shutdown", g_state->completion_callbacks.size()); @@ -878,6 +911,8 @@ bool HostDevice::Synchronize(SyncReason reason) { if (state.command_list_open) { const u32 slot = state.frame_slot; auto* commands = state.command_lists[slot].get(); + if (state.gpu_timestamps) + state.gpu_timestamps->End(commands, slot, "synchronize", static_cast(reason)); commands->end(); state.command_list_open = false; state.queue->executeCommandLists(commands, state.frame_fences[slot].get()); @@ -977,6 +1012,8 @@ bool HostDevice::SubmitRecordedWork() { } const u32 slot = state.frame_slot; auto* commands = state.command_lists[slot].get(); + if (state.gpu_timestamps) + state.gpu_timestamps->End(commands, slot, "kickoff"); commands->end(); state.command_list_open = false; state.queue->executeCommandLists(commands, state.frame_fences[slot].get()); @@ -1215,6 +1252,8 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, if (!acquired || image_index >= state.framebuffers.size()) { // Close and submit any recorded uploads/draws so the allocator remains in // a valid state while the window is minimized or the swap chain changes. + if (state.gpu_timestamps) + state.gpu_timestamps->End(commands, slot, "acquire_failed"); commands->end(); state.command_list_open = false; state.queue->executeCommandLists(commands, state.frame_fences[slot].get()); @@ -1262,6 +1301,8 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, commands->barriers( plume::RenderBarrierStage::NONE, plume::RenderTextureBarrier(back, plume::RenderTextureLayout::PRESENT)); + if (state.gpu_timestamps) + state.gpu_timestamps->End(commands, slot, "present", GpuSubmissionTimestamps::kNoImage, image_index); commands->end(); state.command_list_open = false; @@ -1280,6 +1321,8 @@ bool HostDevice::PresentTexture(plume::RenderTexture* texture, const auto swap_start = timing_enabled ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; const bool presented = state.swap_chain->present(image_index, signals, 1); + if (state.gpu_timestamps) + state.gpu_timestamps->PresentResult(slot, presented); if (timing_enabled) state.swap_present_cpu_ms += std::chrono::duration( std::chrono::steady_clock::now() - swap_start).count(); ReportLogoCapture(); diff --git a/rexlego/src/gpu_native/gpu_timestamps.h b/rexlego/src/gpu_native/gpu_timestamps.h new file mode 100644 index 0000000..11c2d25 --- /dev/null +++ b/rexlego/src/gpu_native/gpu_timestamps.h @@ -0,0 +1,199 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace legodimensions::gpu_native { + +// Optional diagnostic, serialized by HostDevice's recording/state locks. +// Same-list bottom-of-pipe elapsed time is not GPU utilization or frame latency. +// Each slot owns its query/readback storage until its existing submission fence +// completes. This class never submits, signals, waits, or polls a fence itself. +class GpuSubmissionTimestamps { + public: + static constexpr unsigned kMaxSlots = 12; + static constexpr unsigned kNoImage = ~0u; + + static std::unique_ptr Create( + plume::RenderDevice* device, plume::RenderCommandQueue* queue, + unsigned slots, const char* path, const char* fingerprint) noexcept { + if (!path || !*path || !fingerprint || slots == 0 || slots > kMaxSlots) return nullptr; + try { + auto result = std::make_unique(); + auto* native_queue = static_cast(queue); + if (!device || !native_queue || !native_queue->d3d || + FAILED(native_queue->d3d->GetTimestampFrequency(&result->frequency_)) || + result->frequency_ == 0) return nullptr; + result->count_ = slots; + result->fingerprint_ = fingerprint; + for (unsigned slot = 0; slot < slots; ++slot) { + auto& pool = result->slots_[slot].pool; + pool = device->createQueryPool(2); + if (!pool || pool->getCount() != 2) return nullptr; + auto* native = static_cast(pool.get()); + if (!native->d3d || !native->readbackBuffer || + !static_cast(native->readbackBuffer.get())->d3d) + return nullptr; + } + result->file_.open(path); + if (!result->file_) return nullptr; + constexpr const char* header = + "build_fingerprint,source_frame,slot,recording,submission,submit_kind,sync_reason,image,present_result,completion_frame,start_ticks,end_ticks,frequency_hz,queue_elapsed_ms,status\n"; + result->file_ << header; + if (!result->file_) return nullptr; + result->bytes_ = std::strlen(header); + result->enabled_ = true; + return result; + } catch (...) { + // Diagnostics must never turn allocation/file failure into title failure. + return nullptr; + } + } + + UINT64 Frequency() const noexcept { return frequency_; } + + void Begin(plume::RenderCommandList* commands, unsigned slot, + unsigned long long source_frame) noexcept { + if (!enabled_ || slot >= count_) return; + auto& sample = slots_[slot]; + // Reuse is legal only after Complete consumed this exact slot's old pair. + if (sample.pending || sample.recording) { enabled_ = false; return; } + sample.source_frame = source_frame; + sample.number = ++recording_number_; + sample.submission = 0; + sample.kind = "unsubmitted"; + sample.sync_reason = sample.image = kNoImage; + sample.present_result = -1; + sample.recording = true; + commands->writeTimestamp(sample.pool.get(), 0); + } + + void End(plume::RenderCommandList* commands, unsigned slot, + const char* kind, unsigned sync_reason = kNoImage, + unsigned image = kNoImage) noexcept { + if (slot >= count_) return; + auto& sample = slots_[slot]; + // Finish any already-written start even if a later diagnostic failure + // disabled new recordings. Never resolve an uninitialized query pair. + if (!sample.recording) return; + commands->writeTimestamp(sample.pool.get(), 1); + sample.kind = kind; + sample.sync_reason = sync_reason; + sample.image = image; + sample.recording = false; + sample.pending = true; + } + + void Submitted(unsigned slot, unsigned long long submission) noexcept { + if (slot < count_ && slots_[slot].pending) + slots_[slot].submission = submission; + } + + void PresentResult(unsigned slot, bool presented) noexcept { + if (slot < count_ && slots_[slot].pending) + slots_[slot].present_result = presented ? 1 : 0; + } + + void Complete(unsigned slot, unsigned long long completed, + unsigned long long target, + unsigned long long completion_frame) noexcept { + if (slot >= count_) return; + auto& sample = slots_[slot]; + if (!sample.pending || !sample.submission) return; + if (completed == ~0ull) { + enabled_ = false; + Write(slot, completion_frame, nullptr, "device_removed"); + sample.pending = false; + return; + } + if (!target || completed < target) return; + auto* pool = static_cast(sample.pool.get()); + auto* buffer = static_cast(pool->readbackBuffer.get()); + const D3D12_RANGE read_range{0, 2 * sizeof(UINT64)}; + void* mapped = nullptr; + const HRESULT status = buffer->d3d->Map(0, &read_range, &mapped); + UINT64 ticks[2]{}; + if (FAILED(status) || !mapped) { + if (SUCCEEDED(status)) { + const D3D12_RANGE no_writes{0, 0}; + buffer->d3d->Unmap(0, &no_writes); + } + enabled_ = false; + Write(slot, completion_frame, nullptr, "map_failed"); + } else { + std::memcpy(ticks, mapped, sizeof(ticks)); + const D3D12_RANGE no_writes{0, 0}; + buffer->d3d->Unmap(0, &no_writes); + if (ticks[1] < ticks[0]) enabled_ = false; + Write(slot, completion_frame, ticks, + ticks[1] < ticks[0] ? "reversed_ticks" : "ok"); + } + sample.pending = false; + } + + void Shutdown(unsigned long long completion_frame) noexcept { + enabled_ = false; + for (unsigned slot = 0; slot < count_; ++slot) { + auto& sample = slots_[slot]; + if (sample.pending || sample.recording) { + Write(slot, completion_frame, nullptr, + sample.pending ? "unverified_shutdown" : "unsubmitted_shutdown"); + sample.pending = sample.recording = false; + } + } + file_.flush(); + } + + private: + struct Sample { + std::unique_ptr pool; + unsigned long long source_frame = 0, number = 0, submission = 0; + const char* kind = "unsubmitted"; + unsigned sync_reason = kNoImage, image = kNoImage; + int present_result = -1; + bool recording = false, pending = false; + }; + std::array slots_; + UINT64 frequency_ = 0; + unsigned count_ = 0; + unsigned long long recording_number_ = 0, rows_ = 0, bytes_ = 0; + const char* fingerprint_ = ""; + std::ofstream file_; + bool enabled_ = false; + + void Write(unsigned slot, unsigned long long frame, const UINT64* ticks, + const char* status) noexcept { + // Fixed stack formatting and a bounded buffered file: no per-row heap + // allocation, no writer thread, no unbounded log growth at high kick rates. + if (!file_ || rows_ >= 200000) { enabled_ = false; return; } + auto& sample = slots_[slot]; + char start[32]{}, end[32]{}, elapsed[48]{}; + if (ticks) { + std::snprintf(start, sizeof(start), "%llu", static_cast(ticks[0])); + std::snprintf(end, sizeof(end), "%llu", static_cast(ticks[1])); + if (ticks[1] >= ticks[0]) + std::snprintf(elapsed, sizeof(elapsed), "%.9f", + static_cast(ticks[1] - ticks[0]) * 1000.0 / static_cast(frequency_)); + } + char row[512]; + const int length = std::snprintf(row, sizeof(row), + "%s,%llu,%u,%llu,%llu,%s,%u,%u,%d,%llu,%s,%s,%llu,%s,%s\n", + fingerprint_, sample.source_frame, slot, sample.number, sample.submission, + sample.kind, sample.sync_reason, sample.image, sample.present_result, + frame, start, end, static_cast(frequency_), elapsed, status); + if (length <= 0 || static_cast(length) >= sizeof(row) || + bytes_ + static_cast(length) > 64ull * 1024 * 1024) { + enabled_ = false; + return; + } + file_.write(row, length); + bytes_ += static_cast(length); + if (++rows_ % 120 == 0) file_.flush(); + if (!file_) enabled_ = false; + } +}; +} // namespace legodimensions::gpu_native diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index 6421503..31183b4 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -42,6 +42,11 @@ if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") ${CMAKE_CURRENT_BINARY_DIR}/fixtures/plume_command_creation --compiler ${CMAKE_CXX_COMPILER}) set_tests_properties(plume_command_creation PROPERTIES TIMEOUT 60) + add_test(NAME gpu_timestamps + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_gpu_timestamps.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/gpu_timestamps + --compiler ${CMAKE_CXX_COMPILER}) + set_tests_properties(gpu_timestamps PROPERTIES TIMEOUT 60) if(UNIX AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/.git") add_test(NAME prepare_plume COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_prepare_plume.py diff --git a/tools/gpu-aot/regression/README.md b/tools/gpu-aot/regression/README.md index a6214c8..809da33 100644 --- a/tools/gpu-aot/regression/README.md +++ b/tools/gpu-aot/regression/README.md @@ -70,6 +70,14 @@ For a timer comparison keep command slots fixed and pass `--vary timer-wait` to `compare_samples.py`. Its default `--vary command-slots` requires timer mode to match. Varying both settings rejects comparability. +For a build supporting native GPU timestamps, add `--gpu-timestamps` to write +`logs/gpu-timestamps.csv` separately from CPU cadence metrics. Query writes and +readback add diagnostic overhead; comparisons require the same probe presence. +The spans measure elapsed queue execution within individual command submissions, +not GPU utilization, CPU command recording or DXGI presentation. A descriptor +only requests the probe; startup evidence and actual completed rows must confirm +availability on the selected device. PM4 cannot request this native probe. + For a PM4 reference use `--renderer pm4`, optionally `--render-target-path-d3d12 rtv`, and provide `--pm4-plugin`, `--pm4-plugin-sha256`, `--pm4-runtime-sha256`, and diff --git a/tools/gpu-aot/regression/compare_samples.py b/tools/gpu-aot/regression/compare_samples.py index 94f18f3..1596f09 100644 --- a/tools/gpu-aot/regression/compare_samples.py +++ b/tools/gpu-aot/regression/compare_samples.py @@ -18,6 +18,9 @@ def comparison_identity(sample, varying='command-slots'): 'LEGO_NATIVE_SHADER_PACK', 'LEGO_NATIVE_SHADER_PACK_TRIGGER'): flags.pop(key, None) # Private paths differ; sampler verifies their ownership. flags.pop('LEGO_NATIVE_COMMAND_SLOTS' if varying == 'command-slots' else 'REX_TIMER_WAIT_BLOCKING', None) + if 'LEGO_NATIVE_GPU_TIMESTAMPS' in flags: + # Preserve instrumentation presence while normalizing its private path. + flags['LEGO_NATIVE_GPU_TIMESTAMPS'] = 'enabled' return {'binaries': sample['binaries'], 'checkpoint': sample['checkpoint'], 'criteria': sample['criteria'], 'normalized_config': sample['normalized_config'], 'shader_packs': sample['shader_packs'], 'supporting_state': sample['supporting_state'], diff --git a/tools/gpu-aot/regression/prepare_run.py b/tools/gpu-aot/regression/prepare_run.py index 80151c4..4a97d3c 100644 --- a/tools/gpu-aot/regression/prepare_run.py +++ b/tools/gpu-aot/regression/prepare_run.py @@ -293,6 +293,9 @@ def prepare(args): require(timer_wait in ('spin', 'blocking'), 'Unknown timer wait mode') cadence = getattr(args, 'cadence_only', False) performance = getattr(args, 'performance', False) or cadence + gpu_timestamps = getattr(args, 'gpu_timestamps', False) + require(not gpu_timestamps or args.renderer == 'native', + 'GPU timestamp sidecar requires native route') command_slots = getattr(args, 'command_slots', None) require(command_slots is None or (cadence and command_slots in (3, 12)), 'Explicit command slots require native cadence-only mode and count3 or12') @@ -473,6 +476,8 @@ def prepare(args): if cadence: overrides.pop('LEGO_NATIVE_TIMING') overrides['LEGO_NATIVE_COMMAND_SLOTS'] = str(command_slots or 3) + if gpu_timestamps: + overrides['LEGO_NATIVE_GPU_TIMESTAMPS'] = windows(final / 'logs/gpu-timestamps.csv') if args.buffer_windows: overrides['LEGO_NATIVE_BUFFER_WINDOWS'] = '1' if args.snapshots: @@ -507,6 +512,7 @@ def prepare(args): 'renderer_requested': args.renderer, 'renderer_verified_by_launch': False, 'performance_probe': performance, 'cadence_only_probe': cadence, + 'gpu_timestamp_probe': gpu_timestamps, 'command_slots': (command_slots or 3) if cadence else None, 'pm4_plugin': plugin_provenance, 'checkpoint': str(checkpoint), 'checkpoint_files': pinned_saves, @@ -596,6 +602,7 @@ def verify(run, workspace=None): allowed.update({'LEGO_NATIVE_STATIC_TEXTURE_WATCH', 'LEGO_NATIVE_BUFFER_WATCH', 'LEGO_NATIVE_TIMING', 'LEGO_NATIVE_STENCIL', 'LEGO_NATIVE_VIEWPORT', 'LEGO_NATIVE_CULL', 'LEGO_NATIVE_FRAME_METRICS', 'LEGO_DUMP_MISSING_SHADERS', 'LEGO_NATIVE_SHADER_PACK', + 'LEGO_NATIVE_GPU_TIMESTAMPS', 'LEGO_NATIVE_SHADER_PACK_TRIGGER', 'LEGO_NATIVE_BUFFER_WINDOWS', 'LEGO_NATIVE_COMMAND_SLOTS', 'LEGO_NATIVE_TRACE_DIR', 'LEGO_NATIVE_DRAW_TRACE_TRIGGER', 'LEGO_GPU_SNAPSHOT_DIR'}) @@ -603,6 +610,7 @@ def verify(run, workspace=None): 1024 <= manifest['toypad_port'] <= 65535 and manifest['toypad_port'] != 9191, 'Unexpected environment override or private Toypad port') path_overrides = {'PROTON_LOG_DIR': run/'logs', 'LEGO_NATIVE_FRAME_METRICS': run/'logs/frames.csv', + 'LEGO_NATIVE_GPU_TIMESTAMPS': run/'logs/gpu-timestamps.csv', 'LEGO_DUMP_MISSING_SHADERS': run/'shader-capture', 'LEGO_NATIVE_SHADER_PACK': run/'hotload/next.pack', 'LEGO_NATIVE_SHADER_PACK_TRIGGER': run/'hotload/next.trigger', @@ -613,6 +621,11 @@ def verify(run, workspace=None): if key in env: require(env[key] == (str(target) if key == 'PROTON_LOG_DIR' else windows(target)), 'Diagnostic output escaped the private run: '+key) + timestamp_probe = manifest.get('gpu_timestamp_probe', False) + require(type(timestamp_probe) is bool and + ('LEGO_NATIVE_GPU_TIMESTAMPS' in env) is timestamp_probe and + (not timestamp_probe or renderer == 'native'), + 'GPU timestamp descriptor disagrees with declared probe') if manifest.get('performance_probe'): require(renderer == 'native' and not any(key in env for key in ( 'LEGO_NATIVE_TRACE_DIR', 'LEGO_NATIVE_DRAW_TRACE_TRIGGER', 'LEGO_GPU_SNAPSHOT_DIR', @@ -711,6 +724,8 @@ def main(): help='Native frame CSV with detailed timing flag absent; implies performance mode') p.add_argument('--command-slots', type=int, choices=(3, 12), help='Explicit private cadence experiment slot count (default3)') + p.add_argument('--gpu-timestamps', action='store_true', + help='Opt-in native per-submission queue timestamps in a separate private CSV') p.add_argument('--snapshots', action='store_true', help='Explicitly enable bounded native DDS readbacks; changes timing workload') p.add_argument('--renderer', choices=('native', 'pm4'), default='native') diff --git a/tools/gpu-aot/regression/test_regression.py b/tools/gpu-aot/regression/test_regression.py index 71a500f..1d1e2c5 100644 --- a/tools/gpu-aot/regression/test_regression.py +++ b/tools/gpu-aot/regression/test_regression.py @@ -187,6 +187,32 @@ def test_cadence_descriptors_preserve_frozen_bytes_and_absent_timing(self): with self.assertRaises(Exception): m.prepare(self.args(name='invalid-cadence', cadence_only=True, **changes)) + def test_gpu_timestamp_probe_is_explicit_private_and_native_only(self): + identities = [] + for enabled in (False, True): + run = Path(m.prepare(self.args(name='gpu-timestamps-'+str(enabled).lower(), + cadence_only=True, gpu_timestamps=enabled))['run_root']) + manifest = json.loads((run/'regression-manifest.json').read_text()) + env = manifest['launch_plan']['environment_overrides'] + identities.append(m.digest(run/'legodimensions.exe')) + self.assertIs(manifest['gpu_timestamp_probe'], enabled) + self.assertEqual('LEGO_NATIVE_GPU_TIMESTAMPS' in env, enabled) + m.verify(run) + if enabled: + self.assertEqual(env['LEGO_NATIVE_GPU_TIMESTAMPS'], m.windows(run/'logs/gpu-timestamps.csv')) + env['LEGO_NATIVE_GPU_TIMESTAMPS'] = m.windows(self.root/'escaped.csv') + (run/'regression-manifest.json').write_text(json.dumps(manifest)) + with self.assertRaisesRegex(Exception, 'Diagnostic output escaped'): + m.verify(run) + env['LEGO_NATIVE_GPU_TIMESTAMPS'] = m.windows(run/'logs/gpu-timestamps.csv') + manifest['gpu_timestamp_probe'] = False + (run/'regression-manifest.json').write_text(json.dumps(manifest)) + with self.assertRaisesRegex(Exception, 'GPU timestamp descriptor'): + m.verify(run) + self.assertEqual(identities[0], identities[1]) + with self.assertRaisesRegex(Exception, 'GPU timestamp sidecar requires native'): + m.prepare(self.args(name='pm4-timestamps', renderer='pm4', gpu_timestamps=True)) + def test_performance_preparation_omits_heavy_diagnostics_and_rejects_tampering(self): report = m.prepare(self.args(name='performance', performance=True, buffer_windows=True)) run = Path(report['run_root']) @@ -628,6 +654,17 @@ def test_comparison_must_vary_only_one_declared_setting(self): path.write_text(json.dumps(report)) self.assertFalse(comparer.compare([a,b],varying='timer-wait')['comparable_identity']) + def test_comparison_preserves_gpu_probe_presence_but_normalizes_private_path(self): + a, report_a = self.sample_fixture('gpu-probe-a',3) + b, report_b = self.sample_fixture('gpu-probe-b',12) + for path, report in ((a,report_a),(b,report_b)): + report['flags']['LEGO_NATIVE_GPU_TIMESTAMPS'] = m.windows(path/'gpu-timestamps.csv') + (path/'sample.json').write_text(json.dumps(report)) + self.assertTrue(comparer.compare([a,b])['comparable_identity']) + del report_b['flags']['LEGO_NATIVE_GPU_TIMESTAMPS'] + (b/'sample.json').write_text(json.dumps(report_b)) + self.assertFalse(comparer.compare([a,b])['comparable_identity']) + def test_inventory_limit_stops_iteration_before_materialization_and_refuses_aliases(self): directory=self.root/'bounded-tree';directory.mkdir() for index in range(5):(directory/str(index)).write_bytes(b'x') diff --git a/tools/gpu-aot/test_command_slots.py b/tools/gpu-aot/test_command_slots.py index e9eb822..ac0ee9c 100644 --- a/tools/gpu-aot/test_command_slots.py +++ b/tools/gpu-aot/test_command_slots.py @@ -15,6 +15,8 @@ def main(): args = parser.parse_args() root = Path(__file__).resolve().parents[2] source = (root / "rexlego/src/gpu_native/device.cpp").read_text() + # The shared boundary includes a disabled optional timestamp pointer; + # test_gpu_timestamps separately executes the enabled production helper. # Share the existing fake-device boundary, while executing new production # bodies unchanged. These fakes do not measure GPU or presentation speed. tree = ast.parse((Path(__file__).with_name("test_frame_ring_transaction.py")).read_text()) diff --git a/tools/gpu-aot/test_completion_callbacks.py b/tools/gpu-aot/test_completion_callbacks.py index d0d6c19..c4c432e 100644 --- a/tools/gpu-aot/test_completion_callbacks.py +++ b/tools/gpu-aot/test_completion_callbacks.py @@ -32,7 +32,13 @@ def body(sig): struct NativeFence {u64 completed=0;u64 GetCompletedValue(){return completed;}}; struct D3D12CommandFence {NativeFence native;NativeFence* d3d=&native;u64 fenceValue=1;}; } +// Disabled optional timestamp boundary; test_gpu_timestamps covers its real enabled implementation. +struct GpuSubmissionTimestamps { + void Begin(auto*,u32,u64){};void End(auto*,u32,const char*,u32=~0u,u32=~0u){}; + void Submitted(u32,u64){};void Complete(u32,u64,u64,u64){}; +}; struct State { + GpuSubmissionTimestamps* gpu_timestamps=nullptr;u32 present_number=0; static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight; Backend backend=Backend::kD3D12; std::array frame_submitted{}; diff --git a/tools/gpu-aot/test_draw_upload.py b/tools/gpu-aot/test_draw_upload.py index c6744c1..162aa3c 100644 --- a/tools/gpu-aot/test_draw_upload.py +++ b/tools/gpu-aot/test_draw_upload.py @@ -52,7 +52,13 @@ def function(signature): '''+page+r''' struct DescriptorRetirement { void CompleteFrame(u32,auto callback){} }; enum class Backend {kD3D12,kVulkan}; +// Disabled optional timestamp boundary; test_gpu_timestamps covers its real enabled implementation. +struct GpuSubmissionTimestamps { + void Begin(auto*,u32,u64){};void End(auto*,u32,const char*,u32=~0u,u32=~0u){}; + void Submitted(u32,u64){};void Complete(u32,u64,u64,u64){}; +}; struct State { + GpuSubmissionTimestamps* gpu_timestamps=nullptr;u32 present_number=0; static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight; std::unique_ptr device=std::make_unique(); std::unique_ptr queue=std::make_unique(); diff --git a/tools/gpu-aot/test_frame_ring_transaction.py b/tools/gpu-aot/test_frame_ring_transaction.py index 0b34696..3f46781 100644 --- a/tools/gpu-aot/test_frame_ring_transaction.py +++ b/tools/gpu-aot/test_frame_ring_transaction.py @@ -82,7 +82,13 @@ def main(): bool isEmpty(){return count==0;} }; } +// Disabled optional timestamp boundary; test_gpu_timestamps covers its real enabled implementation. +struct GpuSubmissionTimestamps { + void Begin(auto*,u32,u64){};void End(auto*,u32,const char*,u32=~0u,u32=~0u){}; + void Submitted(u32,u64){};void Complete(u32,u64,u64,u64){}; +}; struct State { + GpuSubmissionTimestamps* gpu_timestamps=nullptr;u32 present_number=0; static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight;Backend backend=Backend::kD3D12; std::unique_ptr device=std::make_unique(); std::unique_ptr queue=std::make_unique(); diff --git a/tools/gpu-aot/test_gpu_timestamps.py b/tools/gpu-aot/test_gpu_timestamps.py new file mode 100644 index 0000000..e0ab52d --- /dev/null +++ b/tools/gpu-aot/test_gpu_timestamps.py @@ -0,0 +1,249 @@ +"""Exercise real timestamp helper, Plume writes and all native close paths. + +Fake COM/fences assert that CPU readback follows verified completion. No real +GPU, frame latency or utilization claim follows from this asset-free fixture. +""" +import argparse +import json +from pathlib import Path +import subprocess + + +def function(source, signature): + start = source.index(signature) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + parser.add_argument("--helper", type=Path) + parser.add_argument("--device", type=Path) + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=True) + helper = (args.helper or root / "rexlego/src/gpu_native/gpu_timestamps.h").read_text() + helper = helper.replace("#include ", "").replace("#pragma once", "") + device = (args.device or root / "rexlego/src/gpu_native/device.cpp").read_text() + plume = (root / "thirdparty/plume/plume_d3d12.cpp").read_text() + timestamp_write = function(plume, "void D3D12CommandList::writeTimestamp(") + completion = "\n".join(function(device, signature) for signature in ( + "void RefreshCompletedSubmissionsLocked(", "void MarkSubmissionLocked(", + "bool WaitForSubmissionLocked(", "bool WaitForSubmittedFramesLocked(")) + paths = [] + for signature, kind in (("bool HostDevice::Synchronize(", "synchronize"), + ("bool HostDevice::SubmitRecordedWork(", "kickoff"), + ("bool HostDevice::PresentTexture(", "acquire_failed"), + ("bool HostDevice::PresentTexture(", "present")): + body = function(device, signature) + call = body.index('state.gpu_timestamps->End(commands, slot, "' + kind + '"') + guard = body.rfind("if (state.gpu_timestamps)", 0, call) + start = body.rfind("\n", 0, guard) + 1 + end = body.index("MarkSubmissionLocked(state, slot);", start) + len("MarkSubmissionLocked(state, slot);") + paths.append("void Close_" + kind + "(State& state,unsigned slot,unsigned image_index=1) {\n" + "auto* commands=&state.commands;bool timing_enabled=false;SyncReason reason=SyncReason::Resource;\n" + + body[start:end] + "\n}\n") + begin = function(device, "plume::RenderCommandList* BeginFrameCommandsLocked(") + start = begin.index(" commands->begin();") + end = begin.index(" return commands;", start) + len(" return commands;") + begin = "plume::RenderCommandList* Open(State& state,unsigned slot) {auto* commands=&state.commands;\n" + begin[start:end] + "\n}" + present = function(device, "bool HostDevice::PresentTexture(") + call = present.index("state.gpu_timestamps->PresentResult(slot, presented);") + guard = present.rfind("if (state.gpu_timestamps)", 0, call) + present_result = ("void ReportPresent(State& state,unsigned slot,bool presented){\n" + + present[guard:call + len("state.gpu_timestamps->PresentResult(slot, presented);")] + + "\n}\n") + harness = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using u32=uint32_t;using u64=uint64_t;using UINT64=unsigned long long;using HRESULT=long; +#define FAILED(x) ((x)<0) +#define SUCCEEDED(x) ((x)>=0) +#define REXLOG_ERROR(...) ((void)0) +constexpr int D3D12_QUERY_TYPE_TIMESTAMP=2; +struct D3D12_RANGE {size_t Begin,End;}; +enum class Failure {None,Frequency,ZeroFrequency,Heap,Buffer,Resource,Count,Map,NullMap}; +Failure failure=Failure::None; +struct Heap {bool initialized[2]{};UINT64 ticks[2]{};}; +struct Resource { + UINT64 ticks[2]{};bool ready=false;int maps=0,unmaps=0; + HRESULT Map(int index,const D3D12_RANGE* range,void** out) { + assert(ready && "timestamp premature map oracle");assert(index==0&&range->Begin==0&&range->End==16); + ++maps;if(failure==Failure::Map)return -1; + *out=failure==Failure::NullMap?nullptr:ticks;return 0; + } + void Unmap(int index,const D3D12_RANGE* range){assert(index==0&&range->Begin==0&&range->End==0);++unmaps;} +}; +struct NativeQueue {int frequency_calls=0;HRESULT GetTimestampFrequency(UINT64* value) { + ++frequency_calls;*value=failure==Failure::ZeroFrequency?0:1000000000ull; + return failure==Failure::Frequency?-1:0; +}}; +struct Fence {UINT64 completed=0;UINT64 GetCompletedValue(){return completed;}}; +struct NativeCommands { + UINT64 tick=0;int writes=0,resolves=0; + void EndQuery(Heap* heap,int type,unsigned index){assert(heap&&type==2&&index<2);heap->initialized[index]=true;heap->ticks[index]=tick;++writes;} + void ResolveQueryData(Heap* heap,int type,unsigned index,unsigned count,Resource* resource,UINT64 offset){ + assert(type==2&&count==1&&index<2&&heap->initialized[index]&&offset==index*8); + resource->ticks[index]=heap->ticks[index];resource->ready=false;++resolves;tick+=5; + } +}; +namespace plume { +struct RenderBuffer {virtual ~RenderBuffer()=default;}; +struct D3D12Buffer:RenderBuffer {Resource native;Resource* d3d=&native;}; +struct RenderQueryPool {virtual ~RenderQueryPool()=default;virtual unsigned getCount() const=0;}; +struct D3D12QueryPool:RenderQueryPool { + Heap native;Heap* d3d=&native;std::unique_ptr readbackBuffer; + unsigned getCount()const override{return failure==Failure::Count?1:2;} + D3D12QueryPool(){if(failure==Failure::Heap)d3d=nullptr;if(failure!=Failure::Buffer){ + auto b=std::make_unique();if(failure==Failure::Resource)b->d3d=nullptr;readbackBuffer=std::move(b);}} +}; +struct RenderDevice {int creates=0;std::vector pools; + std::unique_ptr createQueryPool(unsigned count){assert(count==2);++creates;auto pool=std::make_unique();pools.push_back(pool.get());return pool;} + Resource& resource(unsigned slot){return static_cast(pools.at(slot)->readbackBuffer.get())->native;} +}; +struct RenderCommandFence {virtual ~RenderCommandFence()=default;}; +struct D3D12CommandFence:RenderCommandFence {Fence native;Fence* d3d=&native;UINT64 fenceValue=1;}; +struct RenderCommandSemaphore {}; +struct RenderCommandList {virtual ~RenderCommandList()=default;virtual void writeTimestamp(const RenderQueryPool*,unsigned)=0;virtual void begin()=0;virtual void end()=0;}; +struct D3D12CommandList:RenderCommandList {NativeCommands native;NativeCommands* d3d=&native;int begins=0,ends=0; + void writeTimestamp(const RenderQueryPool*,unsigned)override;void begin()override{++begins;}void end()override{++ends;}}; +struct RenderCommandQueue {virtual ~RenderCommandQueue()=default;int executes=0,waits=0; + void executeCommandLists(const RenderCommandList*,RenderCommandFence* fence){++executes;++static_cast(fence)->fenceValue;} + void executeCommandLists(const RenderCommandList** lists,unsigned count,RenderCommandSemaphore** w,unsigned wc,RenderCommandSemaphore** s,unsigned sc,RenderCommandFence* f){assert(count==1&&lists[0]&&wc==1&&sc==1&&w[0]&&s[0]);executeCommandLists(lists[0],f);} + void waitForCommandFence(RenderCommandFence*){++waits;} +}; +struct D3D12CommandQueue:RenderCommandQueue {NativeQueue native;NativeQueue* d3d=&native;}; +''' + timestamp_write + r''' +} +''' + helper + r''' +using namespace legodimensions::gpu_native; +enum class Backend {kD3D12,kVulkan};enum class SyncReason {Resource=3}; +struct State { + Backend backend=Backend::kD3D12;unsigned command_slot_count=3,present_number=0; + std::array frame_submitted{};std::array slot_submission{}; + u64 last_submission=0,completed_submission=0;bool command_list_open=false; + std::array,12> frame_fences; + std::array,12> acquire_semaphores; + std::array,3> render_semaphores; + plume::RenderDevice device;std::unique_ptr queue=std::make_unique(); + std::unique_ptr gpu_timestamps;plume::D3D12CommandList commands; + double present_submit_cpu_ms=0; + State(){for(auto& f:frame_fences)f=std::make_unique();for(auto& s:acquire_semaphores)s=std::make_unique();for(auto& s:render_semaphores)s=std::make_unique();} + auto& fence(unsigned slot){return *static_cast(frame_fences[slot].get());} +}; +void InitializeNullTextures(State& state,plume::RenderCommandList*){state.commands.native.tick+=100;} +''' + completion + begin + "\n" + "\n".join(paths) + present_result + r''' +std::vector lines(const std::filesystem::path& path){std::ifstream f(path);std::vector r;std::string s;while(std::getline(f,s))r.push_back(s);return r;} +int main(int argc,char** argv){assert(argc==2);const std::filesystem::path directory=argv[1]; + auto make=[&](State& state,const char* name,unsigned count=3){auto path=directory/name;return GpuSubmissionTimestamps::Create(&state.device,state.queue.get(),count,path.string().c_str(),"0123456789abcdef");}; + { + State state;assert(!GpuSubmissionTimestamps::Create(&state.device,state.queue.get(),3,nullptr,"x"));assert(state.device.creates==0&&state.queue->native.frequency_calls==0); + Open(state,0);Close_kickoff(state,0);state.fence(0).native.completed=1;assert(WaitForSubmissionLocked(state,0)); + assert(state.commands.native.writes==0&&state.queue->executes==1&&state.queue->waits==1); + } + { + std::array default_schedule{}; + for(bool diagnostic:{false,true}) { + State state;if(diagnostic)state.gpu_timestamps=make(state,"same-schedule.csv"); + void(*close[])(State&,unsigned,unsigned)={Close_synchronize,Close_kickoff,Close_acquire_failed,Close_present}; + for(unsigned step=0;step<4;++step){const unsigned slot=step%3;Open(state,slot);close[step](state,slot,1); + if(diagnostic)state.device.resource(slot).ready=true; + state.fence(slot).native.completed=state.fence(slot).fenceValue-1;assert(WaitForSubmissionLocked(state,slot)); + } + const std::array schedule{state.commands.begins,state.commands.ends,state.queue->executes,state.queue->waits}; + if(!diagnostic)default_schedule=schedule;else {assert(schedule==default_schedule&&"timestamp unchanged submission/wait schedule oracle");state.gpu_timestamps->Shutdown(0);} + assert(state.completed_submission==4&&state.commands.native.writes==(diagnostic?8:0)); + } + } + for(auto mode:{Failure::Frequency,Failure::ZeroFrequency,Failure::Heap,Failure::Buffer,Failure::Resource,Failure::Count}){ + State state;failure=mode;assert(!make(state,"allocation-failure.csv"));failure=Failure::None; + } + { + State state;auto bad=directory/"absent-parent"/"file.csv";assert(!GpuSubmissionTimestamps::Create(&state.device,state.queue.get(),3,bad.string().c_str(),"x")); + } + { + State state;state.command_slot_count=12;state.gpu_timestamps=make(state,"paths.csv",12);assert(state.gpu_timestamps&&state.device.creates==12); + void(*close[])(State&,unsigned,unsigned)={Close_synchronize,Close_kickoff,Close_acquire_failed,Close_present}; + for(unsigned slot=0;slot<12;++slot){state.present_number=17;state.commands.native.tick=(1ull<<62)+slot*1000000;Open(state,slot);state.commands.native.tick+=2000000;close[slot%4](state,slot,1);assert(state.slot_submission[slot]==slot+1);} + assert(state.commands.native.writes==24&&state.commands.native.resolves==24&&state.queue->executes==12); + RefreshCompletedSubmissionsLocked(state);assert(state.completed_submission==0); + for(unsigned slot=0;slot<12;++slot){assert(state.device.resource(slot).maps==0);state.device.resource(slot).ready=true;state.fence(slot).native.completed=1;} + RefreshCompletedSubmissionsLocked(state);RefreshCompletedSubmissionsLocked(state);assert(state.completed_submission==12); + for(unsigned slot=0;slot<12;++slot)assert(state.device.resource(slot).maps==1&&state.device.resource(slot).unmaps==1&&"timestamp exactly-once map oracle"); + assert(WaitForSubmittedFramesLocked(state));assert(state.queue->waits==12); + for(unsigned slot=0;slot<12;++slot)assert(state.device.resource(slot).maps==1); + // Same-slot reuse after actual wait, with a new ticket and present failure metadata. + Open(state,0);Close_present(state,0,2);ReportPresent(state,0,false); + state.device.resource(0).ready=true;state.fence(0).native.completed=2;assert(WaitForSubmissionLocked(state,0)); + state.gpu_timestamps->Shutdown(19);auto rows=lines(directory/"paths.csv");assert(rows.size()==14&&"timestamp all-path/double-drain oracle"); + for(unsigned slot=0;slot<12;++slot){assert(rows[slot+1].find(",18,"+std::to_string(slot)+",")!=std::string::npos);assert(rows[slot+1].ends_with(",2.000105000,ok"));} + assert(rows[13].find(",13,present,4294967295,2,0,")!=std::string::npos); + } + for(auto mode:{Failure::Map,Failure::NullMap}) { + State state;state.gpu_timestamps=make(state,mode==Failure::Map?"map.csv":"null-map.csv");Open(state,0);Close_kickoff(state,0); + state.device.resource(0).ready=true;state.fence(0).native.completed=1;failure=mode;RefreshCompletedSubmissionsLocked(state);failure=Failure::None; + RefreshCompletedSubmissionsLocked(state);Open(state,1);assert(state.commands.native.writes==2);state.gpu_timestamps->Shutdown(0); + auto rows=lines(directory/(mode==Failure::Map?"map.csv":"null-map.csv"));assert(rows.size()==2&&rows[1].ends_with(",,map_failed")); + assert(state.device.resource(0).unmaps==(mode==Failure::Map?0:1)); + } + { + State state;state.gpu_timestamps=make(state,"removed.csv");Open(state,0);Close_present(state,0); + state.fence(0).native.completed=UINT64_MAX;RefreshCompletedSubmissionsLocked(state);assert(!WaitForSubmissionLocked(state,0));assert(state.device.resource(0).maps==0); + state.gpu_timestamps->Shutdown(0);auto rows=lines(directory/"removed.csv");assert(rows.size()==2&&rows[1].ends_with(",,device_removed")); + } + { + State state;state.gpu_timestamps=make(state,"short.csv");Open(state,0);Close_kickoff(state,0); + assert(!WaitForSubmissionLocked(state,0));assert(state.device.resource(0).maps==0); + state.gpu_timestamps->Complete(0,100,0,0);assert(state.device.resource(0).maps==0); + state.gpu_timestamps->Shutdown(0);auto rows=lines(directory/"short.csv");assert(rows.size()==2&&rows[1].ends_with(",,unverified_shutdown")); + } + { + State state;state.gpu_timestamps=make(state,"unsubmitted.csv");Open(state,0);state.gpu_timestamps->Shutdown(0); + assert(state.device.resource(0).maps==0&&state.queue->executes==0);auto rows=lines(directory/"unsubmitted.csv");assert(rows.size()==2&&rows[1].ends_with(",,unsubmitted_shutdown")); + } + { + State state;state.gpu_timestamps=make(state,"reversed.csv");Open(state,0);Close_kickoff(state,0);auto& r=state.device.resource(0);r.ticks[0]=2;r.ticks[1]=1;r.ready=true;state.fence(0).native.completed=1; + RefreshCompletedSubmissionsLocked(state);state.gpu_timestamps->Shutdown(0);auto rows=lines(directory/"reversed.csv");assert(rows.size()==2&&rows[1].ends_with(",,reversed_ticks")); + } + { + State state;state.gpu_timestamps=make(state,"unsafe-reuse.csv");Open(state,0);Close_kickoff(state,0); + Open(state,0);assert(state.commands.native.writes==2);state.gpu_timestamps->Shutdown(0);assert(state.device.resource(0).maps==0); + } +} +''' + cpp = args.output / "gpu-timestamps.cpp" + cpp.write_text(harness) + executable = args.output / "gpu-timestamps.exe" + subprocess.run([args.compiler, "-std=c++20", "-UNDEBUG", str(cpp), "-o", str(executable)], + check=True, timeout=45) + subprocess.run([str(executable), str(args.output)], check=True, timeout=10) + (args.output / "verification.json").write_text(json.dumps({ + "passed": True, "actual_helper": True, "actual_plume_timestamp_write": True, + "actual_native_completion_and_four_close_paths": True, "fake_gpu": True, + "checks": ["disabled no queries/maps", "actual queue frequency failure/zero", "partial allocations", "output failure", + "3/12 slots and all four close paths", "delayed completion", "one map per ticket", "wait/poll double-drain", + "same-slot reuse", "source-frame/image/present-failure metadata", "large absolute tick subtraction", + "failed/null map", "device removal", "short/invalid target", "unsubmitted shutdown", "reversed ticks", "unsafe reuse disables diagnostic"], + "limits": ["Mock COM/fences test ownership and ordering; no real GPU latency/utilization result."]}, indent=2) + "\n") + print("PASS: actual GPU timestamp helper, Plume writes and four native submission paths") + + +if __name__ == "__main__": + main() diff --git a/tools/gpu-aot/test_resize_fences.py b/tools/gpu-aot/test_resize_fences.py index 7eb180a..ef4b764 100644 --- a/tools/gpu-aot/test_resize_fences.py +++ b/tools/gpu-aot/test_resize_fences.py @@ -88,7 +88,13 @@ def body(path, signature): '''+resize+r''' } struct SwapChain {bool resize_ok=true,empty=false;int resizes=0;bool resize();bool isEmpty(){return empty;}}; +// Disabled optional timestamp boundary; test_gpu_timestamps covers its real enabled implementation. +struct GpuSubmissionTimestamps { + void Begin(auto*,u32,u64){};void End(auto*,u32,const char*,u32=~0u,u32=~0u){}; + void Submitted(u32,u64){};void Complete(u32,u64,u64,u64){}; +}; struct State { + GpuSubmissionTimestamps* gpu_timestamps=nullptr;u32 present_number=0; static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight; Backend backend=Backend::kD3D12; std::array frame_submitted{}; diff --git a/tools/gpu-aot/test_submission_waits.py b/tools/gpu-aot/test_submission_waits.py index 698a012..fe339bb 100644 --- a/tools/gpu-aot/test_submission_waits.py +++ b/tools/gpu-aot/test_submission_waits.py @@ -70,7 +70,13 @@ def main(): }; struct Swap {bool needsResize(){return false;}}; } +// Disabled optional timestamp boundary; test_gpu_timestamps covers its real enabled implementation. +struct GpuSubmissionTimestamps { + void Begin(auto*,u32,u64){};void End(auto*,u32,const char*,u32=~0u,u32=~0u){}; + void Submitted(u32,u64){};void Complete(u32,u64,u64,u64){}; +}; struct State { + GpuSubmissionTimestamps* gpu_timestamps=nullptr;u32 present_number=0; static constexpr u32 kFramesInFlight=3;u32 command_slot_count=kFramesInFlight; Backend backend=Backend::kD3D12; std::array frame_submitted{}; From c4196e4cef51d3925b2d5cfcdd7e8b96a7ff6483 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:27:17 +0200 Subject: [PATCH 18/40] Publish prelinked shader banks transactionally Retain the previous output until checked writes, flush, sync and close succeed, then replace atomically on POSIX or Windows. Add an explicit bounded prelink-worker budget and byte-preserving large-literal emission with MSVC fallback. Validation: production publication failure fixture passed POSIX and Windows API shim branches; original unchecked output strategy was rejected. Actual emitted literal/fallback bytes and miniz round-trip passed; worker-budget fixture rejected invalid budgets before spawning. --- tools/gpu-aot/CMakeLists.txt | 9 +- tools/gpu-aot/prelink_jobs.h | 27 ++++ tools/gpu-aot/prelink_shader_cache.cpp | 162 +++++++++++++++++-- tools/gpu-aot/test_prelink_emission.py | 120 ++++++++++++++ tools/gpu-aot/test_prelink_publication.py | 187 ++++++++++++++++++++++ tools/gpu-aot/test_prelink_workers.py | 92 +++++++++++ 6 files changed, 587 insertions(+), 10 deletions(-) create mode 100644 tools/gpu-aot/prelink_jobs.h create mode 100644 tools/gpu-aot/test_prelink_emission.py create mode 100644 tools/gpu-aot/test_prelink_publication.py create mode 100644 tools/gpu-aot/test_prelink_workers.py diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index 31183b4..aaf2d61 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -61,7 +61,7 @@ endif() if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") find_package(Python3 COMPONENTS Interpreter QUIET) if(Python3_Interpreter_FOUND) - foreach(check IN ITEMS command_slots constant_timing frame_cadence cull_state draw_diagnostic + foreach(check IN ITEMS command_slots constant_timing frame_cadence cull_state draw_diagnostic prelink_workers prelink_emission frame_ring_transaction submission_waits pipeline_failures completion_callbacks cpu_resource_wait draw_upload query_retirement) add_test(NAME ${check} @@ -70,6 +70,13 @@ if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") --compiler ${CMAKE_CXX_COMPILER}) set_tests_properties(${check} PROPERTIES TIMEOUT 60) endforeach() + if(UNIX) + add_test(NAME prelink_publication + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_prelink_publication.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/prelink_publication + --compiler ${CMAKE_CXX_COMPILER}) + set_tests_properties(prelink_publication PROPERTIES TIMEOUT 60) + endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/plume_d3d12.cpp") add_test(NAME resize_fences COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_resize_fences.py diff --git a/tools/gpu-aot/prelink_jobs.h b/tools/gpu-aot/prelink_jobs.h new file mode 100644 index 0000000..35d1726 --- /dev/null +++ b/tools/gpu-aot/prelink_jobs.h @@ -0,0 +1,27 @@ +/** Portable worker-budget selection for the offline DXC prelinker. */ +#pragma once + +#include +#include +#include +#include +#include + +namespace lego_gpu_aot { + +inline std::optional PrelinkWorkerCount(size_t jobs, unsigned hardware, + const char* requested) { + // Retain the existing unset default, including its unknown-hardware fallback. + unsigned workers = hardware ? hardware : 4; + if (requested) { + unsigned parsed = 0; + const char* const end = requested + std::strlen(requested); + const auto result = std::from_chars(requested, end, parsed, 10); + if (result.ec != std::errc{} || result.ptr != end || parsed == 0) + return std::nullopt; + workers = std::min(workers, parsed); + } + return static_cast(std::min(jobs, workers)); +} + +} // namespace lego_gpu_aot diff --git a/tools/gpu-aot/prelink_shader_cache.cpp b/tools/gpu-aot/prelink_shader_cache.cpp index fc8b4b7..589f342 100644 --- a/tools/gpu-aot/prelink_shader_cache.cpp +++ b/tools/gpu-aot/prelink_shader_cache.cpp @@ -13,26 +13,160 @@ */ #include #include +#include #include #include +#include +#include +#include #include +#include #include #include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#include +#include +#else +#include +#endif #define MINIZ_HEADER_FILE_ONLY #include #include #include "gpu_native/dxc_link.h" #include "gpu/shaders/shader_cache.h" +#include "prelink_jobs.h" namespace { +// Both prelink targets publish through this transaction. A failed write, +// flush, close or replacement must leave the previously generated bank intact. +class PrelinkOutput { + public: + explicit PrelinkOutput(const char* destination) : destination_(destination) { + static std::atomic serial{0}; +#ifdef _WIN32 + const auto process = GetCurrentProcessId(); +#else + const auto process = getpid(); +#endif + for (unsigned attempt = 0; attempt < 64; ++attempt) { + temporary_ = destination_; + temporary_ += ".tmp." + std::to_string(process) + "." + + std::to_string(serial.fetch_add(1, std::memory_order_relaxed)); +#ifdef _WIN32 + int descriptor = -1; + const auto error = _wsopen_s(&descriptor, temporary_.c_str(), + _O_CREAT | _O_EXCL | _O_WRONLY | _O_BINARY, _SH_DENYRW, + _S_IREAD | _S_IWRITE); + if (error) { + errno = error; + if (error == EEXIST) continue; + temporary_.clear(); + return; + } + file_ = _wfdopen(descriptor, L"wb"); + if (!file_) _close(descriptor); +#else + const int descriptor = open(temporary_.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666); + if (descriptor < 0) { + if (errno == EEXIST) continue; + temporary_.clear(); + return; + } + file_ = fdopen(descriptor, "wb"); + if (!file_) close(descriptor); +#endif + return; + } + // Every attempted name belonged to another transaction; do not remove it. + temporary_.clear(); + } + + PrelinkOutput(const PrelinkOutput&) = delete; + PrelinkOutput& operator=(const PrelinkOutput&) = delete; + ~PrelinkOutput() { + if (file_) std::fclose(file_); + if (!temporary_.empty()) { + std::error_code ignored; + std::filesystem::remove(temporary_, ignored); + } + } + + FILE* file() const { return file_; } + + bool commit() { + if (!file_) return false; + bool success = std::ferror(file_) == 0; + if (std::fflush(file_) != 0) success = false; +#ifdef _WIN32 + if (success && _commit(_fileno(file_)) != 0) success = false; +#else + if (success && fsync(fileno(file_)) != 0) success = false; +#endif + if (std::fclose(file_) != 0) success = false; + file_ = nullptr; + if (!success) return false; +#ifdef _WIN32 + if (!MoveFileExW(temporary_.c_str(), destination_.c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) return false; +#else + if (std::rename(temporary_.c_str(), destination_.c_str()) != 0) return false; +#endif + temporary_.clear(); + return true; + } + + private: + std::filesystem::path destination_; + std::filesystem::path temporary_; + FILE* file_ = nullptr; +}; + void EmitBytes(FILE* f, const uint8_t* data, size_t size) { + // Integer initializers create one AST node per byte. The linked bank can + // contain tens of millions of bytes, so emit a single string initializer + // instead. MSVC limits concatenated string length, so retain its original + // initializer form. Large banks still require the COFF embedding path to + // avoid Clang's per-byte constant-evaluation memory cost. The + // recorded compressed size excludes the string form's trailing C++ NUL. + std::fputs("#if defined(_MSC_VER) && !defined(__clang__)\n{\n", f); + if (!size) std::fputs("0,", f); for (size_t i = 0; i < size; ++i) { std::fprintf(f, "%u,", data[i]); if ((i & 31) == 31) std::fputc('\n', f); } - std::fputc('\n', f); + std::fputs("\n}\n#else\n" + "#if defined(__clang__)\n" + "#pragma clang diagnostic push\n" + "#pragma clang diagnostic ignored \"-Woverlength-strings\"\n" + "#endif\n", f); + constexpr size_t kLineBytes = 256; + constexpr char kHex[] = "0123456789ABCDEF"; + if (!size) std::fputs("\"\"\n", f); + for (size_t begin = 0; begin < size; begin += kLineBytes) { + const size_t count = std::min(kLineBytes, size - begin); + char line[kLineBytes * 4 + 3]; + line[0] = '"'; + for (size_t i = 0; i < count; ++i) { + const uint8_t byte = data[begin + i]; + line[1 + i * 4] = '\\'; + line[2 + i * 4] = 'x'; + line[3 + i * 4] = kHex[byte >> 4]; + line[4 + i * 4] = kHex[byte & 15]; + } + line[1 + count * 4] = '"'; + line[2 + count * 4] = '\n'; + std::fwrite(line, 1, count * 4 + 3, f); + } + std::fputs("#if defined(__clang__)\n" + "#pragma clang diagnostic pop\n" + "#endif\n#endif\n", f); } } // namespace @@ -71,6 +205,14 @@ int main(int argc, char** argv) { } while (s); } + const auto worker_count = lego_gpu_aot::PrelinkWorkerCount( + jobs.size(), std::thread::hardware_concurrency(), + std::getenv("LEGO_GPU_PRELINK_JOBS")); + if (!worker_count) { + std::fprintf(stderr, "prelink: LEGO_GPU_PRELINK_JOBS must be a positive unsigned decimal integer\n"); + return 1; + } + std::map> specLibs; for (const auto& j : jobs) specLibs.try_emplace(j.masked); for (auto& [value, blob] : specLibs) { @@ -118,10 +260,8 @@ int main(int argc, char** argv) { } }; { - unsigned hw = std::thread::hardware_concurrency(); - if (!hw) hw = 4; std::vector pool; - for (unsigned t = 0; t < hw; ++t) pool.emplace_back(worker); + for (unsigned t = 0; t < *worker_count; ++t) pool.emplace_back(worker); for (auto& t : pool) t.join(); } if (failed.load()) return 1; @@ -165,9 +305,10 @@ int main(int argc, char** argv) { return 1; } - FILE* f = std::fopen(argv[1], "wb"); + PrelinkOutput output(argv[1]); + FILE* f = output.file(); if (!f) { - std::fprintf(stderr, "prelink: cannot open %s\n", argv[1]); + std::fprintf(stderr, "prelink: cannot create temporary output for %s\n", argv[1]); return 1; } std::fprintf(f, "#include \"gpu/shaders/linked_shader_cache.h\"\n"); @@ -181,14 +322,17 @@ int main(int argc, char** argv) { std::fprintf(f, "};\n"); std::fprintf(f, "const size_t g_linkedShaderCacheEntryCount = %zu;\n", entries.size()); - std::fprintf(f, "const uint8_t g_compressedLinkedDxilCache[] = {\n"); + std::fprintf(f, "const uint8_t g_compressedLinkedDxilCache[] =\n"); EmitBytes(f, comp.data(), comp_len); - std::fprintf(f, "};\n"); + std::fprintf(f, ";\n"); std::fprintf(f, "const size_t g_linkedDxilCacheCompressedSize = %zu;\n", static_cast(comp_len)); std::fprintf(f, "const size_t g_linkedDxilCacheDecompressedSize = %zu;\n", blob.size()); - std::fclose(f); + if (!output.commit()) { + std::fprintf(stderr, "prelink: cannot publish %s; previous output preserved\n", argv[1]); + return 1; + } std::printf("prelink: %zu variant(s) from %zu spec-constant shader(s), " "%zu -> %zu bytes\n", diff --git a/tools/gpu-aot/test_prelink_emission.py b/tools/gpu-aot/test_prelink_emission.py new file mode 100644 index 0000000..c095bcf --- /dev/null +++ b/tools/gpu-aot/test_prelink_emission.py @@ -0,0 +1,120 @@ +"""Compile actual prelink emission and verify literal/fallback bytes and deflate ABI.""" +import argparse +from pathlib import Path +import subprocess + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('output', type=Path) +parser.add_argument('--compiler', default='clang++') +args = parser.parse_args() +root = Path(__file__).resolve().parents[2] +source = (root / 'tools/gpu-aot/prelink_shader_cache.cpp').read_text() +start = source.index('void EmitBytes(') +opening = source.index('{', start) +depth, end = 1, opening + 1 +while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 +helper = source[start:end] +start = source.index(' std::fprintf(f, "const uint8_t g_compressedLinkedDxilCache[]') +end = source.index(' if (!output.commit())', start) +footer = source[start:end] + ' assert(std::fclose(f) == 0);\n' +args.output.mkdir(parents=True, exist_ok=True) +output = args.output.resolve() +miniz = root / 'thirdparty/miniz' +includes = ['-I' + str(miniz), '-I' + str(root / 'rexlego/src')] +common = [args.compiler, '-std=c++20', '-UNDEBUG', *includes] +producer = r''' +#include +#include +#include +#include +#include +#include +#define MINIZ_HEADER_FILE_ONLY +#include +''' + helper + r''' +int main(int argc, char** argv) { + assert(argc == 2); + FILE* f = std::fopen(argv[1], "wb"); assert(f); + std::fputs("#include \"gpu/shaders/linked_shader_cache.h\"\n", f); + for (size_t size : {0, 1, 255, 256, 257, 65535, 65536, 131073}) { + std::vector bytes(size); + for (size_t i = 0; i < size; ++i) bytes[i] = uint8_t(i); + std::fprintf(f, "const uint8_t case_%zu[] =\n", size); + EmitBytes(f, bytes.data(), size); + std::fputs(";\n", f); + } + std::vector blob(8192); + for (size_t i = 0; i < blob.size(); ++i) blob[i] = uint8_t((i * 197) ^ (i >> 5)); + mz_ulong comp_len = mz_compressBound(static_cast(blob.size())); + std::vector comp(comp_len); + assert(mz_compress2(comp.data(), &comp_len, blob.data(), + static_cast(blob.size()), MZ_UBER_COMPRESSION) == MZ_OK); +''' + footer + '\n}\n' +producer_cpp = output / 'emit.cpp' +producer_cpp.write_text(producer) +producer_exe = output / 'emit.exe' +miniz_object = output / 'miniz.o' +subprocess.run([*common, '-c', str(miniz / 'miniz.cpp'), '-o', str(miniz_object)], + check=True, timeout=45) +subprocess.run([*common, str(producer_cpp), str(miniz_object), '-o', str(producer_exe)], + check=True, timeout=45) +bank = output / 'emitted-bank.cpp' +subprocess.run([str(producer_exe), str(bank)], check=True, timeout=10) +emitted = bank.read_text() +assert '#if defined(_MSC_VER) && !defined(__clang__)' in emitted +assert '\\x00' in emitted and '\\xFF' in emitted and '\\x22' in emitted and '\\x5C' in emitted +consumer = r''' +#include +#include +#include +#include +#define MINIZ_HEADER_FILE_ONLY +#include +// Only generated initializer selection is varied. Standard/miniz headers +// above use the actual host compiler configuration. +#if defined(TEST_MSVC_FALLBACK) +#undef __clang__ +#define _MSC_VER 1940 +#endif +#include "emitted-bank.cpp" +template void Check(const uint8_t (&bytes)[N], size_t payload) { +#if defined(TEST_MSVC_FALLBACK) + assert(N == (payload ? payload : 1)); +#else + assert(N == payload + 1 && bytes[payload] == 0); +#endif + for (size_t i = 0; i < payload; ++i) assert(bytes[i] == uint8_t(i)); +} +int main() { +''' + ''.join(f' Check(case_{size}, {size});\n' for size in + (0, 1, 255, 256, 257, 65535, 65536, 131073)) + r''' + assert(g_linkedDxilCacheDecompressedSize == 8192); +#if defined(TEST_MSVC_FALLBACK) + assert(sizeof(g_compressedLinkedDxilCache) == g_linkedDxilCacheCompressedSize); +#else + assert(sizeof(g_compressedLinkedDxilCache) == g_linkedDxilCacheCompressedSize + 1); + assert(g_compressedLinkedDxilCache[g_linkedDxilCacheCompressedSize] == 0); +#endif + std::vector decoded(g_linkedDxilCacheDecompressedSize); + mz_ulong length = decoded.size(); + assert(mz_uncompress(decoded.data(), &length, g_compressedLinkedDxilCache, + g_linkedDxilCacheCompressedSize) == MZ_OK); + assert(length == decoded.size()); + for (size_t i = 0; i < decoded.size(); ++i) + assert(decoded[i] == uint8_t((i * 197) ^ (i >> 5))); +} +''' +consumer_cpp = output / 'check.cpp' +consumer_cpp.write_text(consumer) +for name, definitions in [('strings', []), ('msvc-fallback', ['-DTEST_MSVC_FALLBACK'])]: + binary = output / (name + '.exe') + # Clang's intentional large-literal warning must be narrowly suppressed by + # the real emitted branch, even when the consumer uses warnings as errors. + warnings = ['-Werror', '-Woverlength-strings'] if 'clang' in Path(args.compiler).name else [] + subprocess.run([*common, *warnings, *definitions, str(consumer_cpp), + str(miniz_object), '-o', str(binary)], check=True, timeout=45) + subprocess.run([str(binary)], check=True, timeout=10) +print('PASS: actual emitted literal/MSVC fallback preserve every byte at line/64KiB boundaries; ' + 'production compressed metadata excludes literal NUL and miniz round trip matches') diff --git a/tools/gpu-aot/test_prelink_publication.py b/tools/gpu-aot/test_prelink_publication.py new file mode 100644 index 0000000..7b6fe0b --- /dev/null +++ b/tools/gpu-aot/test_prelink_publication.py @@ -0,0 +1,187 @@ +"""Exercise the actual prelink output transaction and emitter under I/O failures. + +The Windows variant uses controlled API shims on the host, proving branch/flag +selection and failure propagation; it does not replace a Windows SDK compile. +""" +import argparse +from pathlib import Path +import subprocess +import tempfile + + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('output', type=Path) +parser.add_argument('--compiler', default='clang++') +parser.add_argument('--negative-control', action='store_true', + help='Use the original direct, unchecked publication strategy; it must fail') +args = parser.parse_args() +root = Path(__file__).resolve().parents[2] +source = (root / 'tools/gpu-aot/prelink_shader_cache.cpp').read_text() +helper = source[source.index('class PrelinkOutput {'):source.index('void EmitBytes(')] +start = source.index('void EmitBytes(') +emitter = source[start:source.index('\n} // namespace', start)] +start = source.index(' PrelinkOutput output(argv[1]);') +publication = source[start:source.index('\n std::printf("prelink: %zu variant', start)] +# Only I/O boundaries are intercepted. The actual production transaction, +# emission body, main failure returns and complete generated metadata remain. +helper = helper.replace('std::fflush(', 'FaultFlush(').replace('std::fclose(', 'FaultClose(') +helper = helper.replace('fsync(', 'FaultSync(').replace('std::rename(', 'FaultReplace(') +publication = publication.replace(' if (!f) {', ' InjectWriteFailure(f);\n if (!f) {', 1) +if args.negative_control: + publication = publication.replace(' PrelinkOutput output(argv[1]);\n FILE* f = output.file();', + ' FILE* f = std::fopen(argv[1], "wb");') + begin = publication.index(' if (!output.commit())') + publication = publication[:begin] + ' std::fclose(f);\n' + +preamble = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +enum class Fault { None, Write, BufferedWrite, Flush, Sync, Close, Replace }; +Fault fault=Fault::None; +std::filesystem::path observed; +std::string prior; +int close_calls=0; +std::string Read(const std::filesystem::path& path) { + std::ifstream stream(path,std::ios::binary); + return std::string(std::istreambuf_iterator(stream),{}); +} +void Write(const std::filesystem::path& path, const std::string& bytes) { + std::ofstream stream(path,std::ios::binary); stream<=0); + assert(dup2(full,fileno(file))>=0); close(full); +} +#ifdef TEST_WINDOWS_API +// Standard library headers above retain their host ABI. Only the actual +// production Windows branch below is selected and its OS boundaries shimmed. +#define _WIN32 +constexpr int _O_CREAT=O_CREAT,_O_EXCL=O_EXCL,_O_WRONLY=O_WRONLY,_O_BINARY=0x8000; +constexpr int _SH_DENYRW=0x10,_S_IREAD=0400,_S_IWRITE=0200; +constexpr unsigned MOVEFILE_REPLACE_EXISTING=1,MOVEFILE_WRITE_THROUGH=8; +unsigned GetCurrentProcessId() { return static_cast(getpid()); } +int _wsopen_s(int* fd,const char* name,int flags,int share,int permissions) { + assert(flags==(_O_CREAT|_O_EXCL|_O_WRONLY|_O_BINARY)); + assert(share==_SH_DENYRW && permissions==(_S_IREAD|_S_IWRITE)); + *fd=open(name,O_CREAT|O_EXCL|O_WRONLY,0600); return *fd<0?errno:0; +} +FILE* _wfdopen(int fd,const wchar_t* mode) { + assert(std::wstring(mode)==L"wb"); return fdopen(fd,"wb"); +} +int _close(int fd) { return close(fd); } +int _fileno(FILE* file) { return fileno(file); } +int _commit(int fd) { return FaultSync(fd); } +bool MoveFileExW(const char* from,const char* to,unsigned flags) { + assert(flags==(MOVEFILE_REPLACE_EXISTING|MOVEFILE_WRITE_THROUGH)); + return FaultReplace(from,to)==0; +} +#endif +''' +body = r''' +int Produce(const std::filesystem::path& path) { + const std::string destination=path.string(); + const char* argv[]={nullptr,destination.c_str()}; + struct OutEntry { uint64_t hash; uint32_t masked,offset,size; }; + std::vector entries{{0x1234,1,0,32}}; + std::vector blob(32,7),comp{0,255,34,92,128}; + size_t comp_len=comp.size(); +''' + publication + r''' + return 0; +} +bool HasTemporary(const std::filesystem::path& parent,const std::string& prefix) { + for(const auto& row:std::filesystem::directory_iterator(parent)) + if(row.path().filename().string().find(prefix)==0) return true; + return false; +} +int main(int argc,char** argv) { + assert(argc==2); std::filesystem::path root=argv[1]; + std::filesystem::create_directories(root); + auto target=root/"bank with spaces.cpp"; + Write(target,"previous validated bank\n"); observed=target; prior=Read(target); + // A crashed/concurrent transaction's temporary file must not be opened or + // removed. Exclusive creation skips its name instead of truncating it. + auto collision=std::filesystem::path(target.string()+".tmp."+std::to_string(getpid())+".0"); + Write(collision,"other transaction"); + assert(Produce(target)==0); + assert(Read(collision)=="other transaction"); std::filesystem::remove(collision); + std::string complete=Read(target); + assert(complete.find("const size_t g_linkedDxilCacheCompressedSize = 5;")!=std::string::npos); + assert(complete.find("\\x00\\xFF\\x22\\x5C\\x80")!=std::string::npos); + assert(!HasTemporary(root,"bank with spaces.cpp.tmp.")); + for(Fault value:{Fault::Write,Fault::BufferedWrite,Fault::Flush,Fault::Sync,Fault::Close,Fault::Replace}) { + fault=value; Write(target,complete); prior=complete; close_calls=0; + assert(Produce(target)==1 && "failed publication reported success"); + assert(Read(target)==complete && "failed publication destroyed previous bank"); + assert(close_calls==1 && "temporary FILE must close once"); + assert(!HasTemporary(root,"bank with spaces.cpp.tmp.")); + } + fault=Fault::None; observed.clear(); + // Real rename/MoveFile replacement failure, without an injected return code. + auto directory_target=root/"existing directory"; + std::filesystem::create_directory(directory_target); + Write(directory_target/"keep","untouched"); + assert(Produce(directory_target)==1); + assert(Read(directory_target/"keep")=="untouched"); + assert(!HasTemporary(root,"existing directory.tmp.")); + auto impossible=root/"parent is a file"; Write(impossible,"keep parent"); + assert(Produce(impossible/"output.cpp")==1); assert(Read(impossible)=="keep parent"); + // An abandoned transaction/exception path cleans only its sibling file. + Write(target,complete); + { PrelinkOutput aborted(target.string().c_str()); assert(aborted.file()); + std::fputs("incomplete",aborted.file()); } + assert(Read(target)==complete && !HasTemporary(root,"bank with spaces.cpp.tmp.")); + auto fresh=root/"fresh.cpp"; assert(Produce(fresh)==0); assert(Read(fresh)==complete); + { PrelinkOutput committed(target.string().c_str()); assert(committed.file()); + std::fputs("replacement",committed.file()); assert(committed.commit()); + assert(!committed.commit()); } + assert(Read(target)=="replacement"); +} +''' +args.output.mkdir(parents=True, exist_ok=True) +output = args.output.resolve() +for name, defines in (('posix', []), ('windows-api', ['-DTEST_WINDOWS_API'])): + cpp = output / (name + '.cpp') + executable = output / (name + '.exe') + cpp.write_text(preamble + helper + emitter + body) + subprocess.run([args.compiler, '-std=c++20', '-UNDEBUG', *defines, + str(cpp), '-o', str(executable)], check=True, timeout=20) + # Outputs are separate per invocation, preserving the failed negative-control evidence. + run_directory = Path(tempfile.mkdtemp(prefix=name + '-run-', dir=output)) + result = subprocess.run([str(executable), str(run_directory)], + text=True, capture_output=True, timeout=8) + if args.negative_control: + assert result.returncode != 0 and 'failed publication reported success' in result.stderr, result.stderr + else: + assert result.returncode == 0, result.stderr +print('PASS: actual prelink emitter/transaction preserves prior banks on write, buffered flush, ' + 'sync, close, replacement and creation failure; exclusive siblings and cleanup checked ' + 'for POSIX and controlled Windows API branches' if not args.negative_control else + 'PASS: original unchecked direct-publication strategy rejected by the production fixture') diff --git a/tools/gpu-aot/test_prelink_workers.py b/tools/gpu-aot/test_prelink_workers.py new file mode 100644 index 0000000..d039c60 --- /dev/null +++ b/tools/gpu-aot/test_prelink_workers.py @@ -0,0 +1,92 @@ +"""Run real prelink worker selection/pool bodies without DXC or actual threads.""" +import argparse +from pathlib import Path +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('output', type=Path) +p.add_argument('--compiler', default='clang++') +a = p.parse_args() +root = Path(__file__).resolve().parents[2] +source = (root / 'tools/gpu-aot/prelink_shader_cache.cpp').read_text() +selection_start = source.index(' const auto worker_count = ') +selection_end = source.index(' std::map> specLibs;', selection_start) +selection = source[selection_start:selection_end] +# Invalid budgets must fail before any DXC compiler/library work. +assert selection_end < source.index('CompileSpecConstantLib(', selection_end) +pool_start = source.index(' std::vector pool;') +pool_end = source.index('\n }', pool_start) +pool = source[pool_start:pool_end] +# Substitute only the platform observations and thread recorder. The production +# budget selection, error return, loop bound and join loop remain unchanged. +selection = selection.replace('std::thread::hardware_concurrency()', 'hardware') +selection = selection.replace('std::getenv("LEGO_GPU_PRELINK_JOBS")', 'requested') +pool = pool.replace('std::thread', 'RecordingThread') +header = (root / 'tools/gpu-aot/prelink_jobs.h').as_posix() +code = r''' +#include +#include +#include +#include +#include +#include +''' + '#include "' + header + '"\n' + r''' +unsigned created=0,joined=0,calls=0; +struct RecordingThread { + templateexplicit RecordingThread(F worker){++created;worker();} + void join(){++joined;} +}; +int Run(size_t job_count,unsigned hardware,const char* requested){ + std::vector jobs(job_count); +''' + selection + r''' + auto worker=[]{++calls;}; +''' + pool + r''' + return 0; +} +void Check(size_t jobs,unsigned hardware,const char* requested,unsigned expected){ + created=joined=calls=0; + assert(Run(jobs,hardware,requested)==0); + assert(created==expected&&joined==expected&&calls==expected); +} +int main(){ + Check(0,20,nullptr,0);Check(0,0,"1",0); + Check(1,20,nullptr,1);Check(3,20,nullptr,3);Check(30,20,nullptr,20); + Check(30,0,nullptr,4);Check(2,0,nullptr,2); + Check(30,20,"1",1);Check(3,20,"2",2);Check(3,20,"100",3); + Check(30,20,"100",20);Check(30,0,"100",4);Check(30,0,"2",2); + Check(30,20,"0002",2); + const auto maximum=std::to_string(UINT_MAX); + Check(30,20,maximum.c_str(),20); + for(const char* bad:{"","0","0000","-1","+1"," 1","1 ","1\n","1x","0x2", + "1.5","99999999999999999999999999999999999999"}){ + for(size_t jobs:{size_t{0},size_t{3}}){ + created=joined=calls=0; + assert(Run(jobs,20,bad)==1); + assert(created==0&&joined==0&&calls==0); + } + } + const auto overflow=std::to_string(static_cast(UINT_MAX)+1); + assert(!lego_gpu_aot::PrelinkWorkerCount(3,20,overflow.c_str())); + // Budget arithmetic must not narrow the size_t job count before capping. + assert(lego_gpu_aot::PrelinkWorkerCount(std::numeric_limits::max(),20,"1")==1); + // Every positive hint/request combination obeys all three ceilings. + for(unsigned hardware=0;hardware<32;++hardware){ + for(unsigned request=1;request<48;++request){ + const auto requested=std::to_string(request); + for(unsigned jobs=0;jobs<64;++jobs){ + const auto count=lego_gpu_aot::PrelinkWorkerCount(jobs,hardware,requested.c_str()); + assert(count&&*count<=jobs&&*count<=request&&*count<=(hardware?hardware:4)); + assert((*count==0)==(jobs==0)); + } + } + } + std::puts("PASS: actual prelink pool respects strict requested/hardware/job ceilings; invalid budgets create no workers"); +} +''' +a.output.mkdir(parents=True, exist_ok=True) +cpp = a.output.resolve() / 'prelink-workers.cpp' +exe = a.output.resolve() / 'prelink-workers.exe' +cpp.write_text(code) +subprocess.run([a.compiler, '-std=c++20', '-UNDEBUG', str(cpp), '-o', str(exe)], + check=True, timeout=45) +subprocess.run([str(exe)], check=True, timeout=10) From b79191b6eabd8b91f4dbc586d06642852726b615 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:59:06 +0200 Subject: [PATCH 19/40] Compare GPU timestamp overhead with fixed renderer settings --- tools/gpu-aot/regression/README.md | 7 ++- tools/gpu-aot/regression/compare_samples.py | 31 +++++++--- tools/gpu-aot/regression/test_regression.py | 68 +++++++++++++++++++-- 3 files changed, 92 insertions(+), 14 deletions(-) diff --git a/tools/gpu-aot/regression/README.md b/tools/gpu-aot/regression/README.md index 809da33..498e5e8 100644 --- a/tools/gpu-aot/regression/README.md +++ b/tools/gpu-aot/regression/README.md @@ -72,7 +72,12 @@ requires timer mode to match. Varying both settings rejects comparability. For a build supporting native GPU timestamps, add `--gpu-timestamps` to write `logs/gpu-timestamps.csv` separately from CPU cadence metrics. Query writes and -readback add diagnostic overhead; comparisons require the same probe presence. +readback add diagnostic overhead. Command-slot and timer comparisons require +the same probe presence. To measure probe overhead explicitly, keep slots, +timer mode and all other settings fixed and pass `--vary gpu-timestamps` to +`compare_samples.py`; it groups every accepted window as `off` or `on`, retains +rejected windows, and reports `on` relative to `off`. Only probe presence and +its verified private output path may differ; this mode changes no defaults. The spans measure elapsed queue execution within individual command submissions, not GPU utilization, CPU command recording or DXGI presentation. A descriptor only requests the probe; startup evidence and actual completed rows must confirm diff --git a/tools/gpu-aot/regression/compare_samples.py b/tools/gpu-aot/regression/compare_samples.py index 1596f09..061c8d0 100644 --- a/tools/gpu-aot/regression/compare_samples.py +++ b/tools/gpu-aot/regression/compare_samples.py @@ -10,26 +10,42 @@ from sample_cadence import complete_rows, stats +EXPERIMENT_FLAGS = {'command-slots': 'LEGO_NATIVE_COMMAND_SLOTS', + 'timer-wait': 'REX_TIMER_WAIT_BLOCKING', + 'gpu-timestamps': 'LEGO_NATIVE_GPU_TIMESTAMPS'} + + +def experimental_value(sample, varying): + if varying == 'command-slots': + return sample['slots'] + if varying == 'timer-wait': + return sample['timer_wait'] + return 'on' if 'LEGO_NATIVE_GPU_TIMESTAMPS' in sample['flags'] else 'off' + + def comparison_identity(sample, varying='command-slots'): + require(varying in EXPERIMENT_FLAGS, 'Unknown experimental variable') scene = sample['scene'] flags = {key: value for key, value in sample['flags'].items() if key.startswith(('LEGO_NATIVE_', 'LEGO_DUMP_', 'LEGO_GPU_', 'REX_TIMER_'))} for key in ('LEGO_NATIVE_FRAME_METRICS', 'LEGO_DUMP_MISSING_SHADERS', 'LEGO_NATIVE_SHADER_PACK', 'LEGO_NATIVE_SHADER_PACK_TRIGGER'): flags.pop(key, None) # Private paths differ; sampler verifies their ownership. - flags.pop('LEGO_NATIVE_COMMAND_SLOTS' if varying == 'command-slots' else 'REX_TIMER_WAIT_BLOCKING', None) + flags.pop(EXPERIMENT_FLAGS[varying], None) if 'LEGO_NATIVE_GPU_TIMESTAMPS' in flags: # Preserve instrumentation presence while normalizing its private path. flags['LEGO_NATIVE_GPU_TIMESTAMPS'] = 'enabled' return {'binaries': sample['binaries'], 'checkpoint': sample['checkpoint'], 'criteria': sample['criteria'], 'normalized_config': sample['normalized_config'], 'shader_packs': sample['shader_packs'], 'supporting_state': sample['supporting_state'], + 'slots': sample['slots'] if varying != 'command-slots' else None, + 'timer_wait': sample['timer_wait'] if varying != 'timer-wait' else None, 'scene_id': scene['scene_id'], 'camera_id': scene['camera_id'], 'renderer': scene['renderer'], 'figures': scene['figures'], 'flags': flags} def compare(paths, varying='command-slots'): - require(varying in ('command-slots', 'timer-wait'), 'Unknown experimental variable') + require(varying in EXPERIMENT_FLAGS, 'Unknown experimental variable') samples = [json.loads((path/'sample.json').read_text()) for path in paths] for path, sample in zip(paths, samples): if 'metrics' in sample: @@ -37,6 +53,7 @@ def compare(paths, varying='command-slots'): require(stats(rows) == sample['metrics'], 'Frozen CSV disagrees with sample metrics') outcomes = [{'source': str(path.resolve()), 'status': sample['status'], 'slots': sample['slots'], 'timer_wait': sample['timer_wait'], + 'gpu_timestamps': experimental_value(sample, 'gpu-timestamps'), 'metrics': sample.get('metrics'), 'rejection_reasons': sample.get('rejection_reasons', [])} for path, sample in zip(paths, samples)] accepted = [sample for sample in samples if sample['status'] == 'accepted'] @@ -45,14 +62,14 @@ def compare(paths, varying='command-slots'): if accepted else None) means = {} if comparable: - variable = 'slots' if varying == 'command-slots' else 'timer_wait' - for value in sorted({sample[variable] for sample in accepted}): - group = [sample for sample in accepted if sample[variable] == value] + for value in sorted({experimental_value(sample, varying) for sample in accepted}): + group = [sample for sample in accepted if experimental_value(sample, varying) == value] frames = sum(sample['metrics']['frames'] for sample in group) seconds = sum(sample['metrics']['recorded_seconds'] for sample in group) means[str(value)] = {'windows': len(group), 'frames': frames, 'recorded_seconds': seconds, 'fps': frames/seconds} - baseline, candidate = ('3', '12') if varying == 'command-slots' else ('spin', 'blocking') + baseline, candidate = {'command-slots': ('3', '12'), 'timer-wait': ('spin', 'blocking'), + 'gpu-timestamps': ('off', 'on')}[varying] gain = (100*(means[candidate]['fps']/means[baseline]['fps']-1) if candidate in means and baseline in means else None) return {'outcomes': outcomes, 'comparable_identity': comparable, 'accepted_aggregate': means, @@ -69,7 +86,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('samples', nargs='+', type=Path) parser.add_argument('--output', type=Path, required=True) - parser.add_argument('--vary', choices=('command-slots', 'timer-wait'), default='command-slots') + parser.add_argument('--vary', choices=tuple(EXPERIMENT_FLAGS), default='command-slots') args = parser.parse_args() require(not args.output.exists(), 'Refuse overwrite of a previous comparison') args.output.parent.mkdir(parents=True, exist_ok=True) diff --git a/tools/gpu-aot/regression/test_regression.py b/tools/gpu-aot/regression/test_regression.py index 1d1e2c5..3b2b1e1 100644 --- a/tools/gpu-aot/regression/test_regression.py +++ b/tools/gpu-aot/regression/test_regression.py @@ -479,8 +479,9 @@ def test_runner_refuses_invalid_environment_without_spawning(self): with self.assertRaisesRegex(ValueError, 'environment value'): runner.environment(plan) - def live_fixture(self, name='sample', slots=3): - run = Path(m.prepare(self.args(name, cadence_only=True, command_slots=slots))['run_root']) + def live_fixture(self, name='sample', slots=3, **run_options): + run = Path(m.prepare(self.args(name, cadence_only=True, command_slots=slots, + **run_options))['run_root']) manifest = json.loads((run/'regression-manifest.json').read_text()) execution = {'exit_code': None, 'launch_requested': True, 'route': '0', 'environment_overrides': manifest['launch_plan']['environment_overrides']} @@ -499,17 +500,17 @@ def live_fixture(self, name='sample', slots=3): scene_file.write_text(json.dumps(scene)) return run, scene_file - def sample_fixture(self, name='sample', slots=3, event=None): - run, scene = self.live_fixture(name, slots) + def sample_fixture(self, name='sample', slots=3, event=None, interval_ms=50, **run_options): + run, scene = self.live_fixture(name, slots, **run_options) ticks = [0.0] calls = [0] - points = (60, 120, 120, 720) + points = (60, 120, 120, 120+int(30000/interval_ms)) def sleep(seconds): ticks[0] += seconds def endpoint(_): index = calls[0]; calls[0] += 1 if index == 3 and event: with (run/'logs/game.log').open('a') as target:target.write(event+'\n') - rows = [{'frame':float(i), 'interval_ms':50.0, 'draw_calls':500.0, + rows = [{'frame':float(i), 'interval_ms':float(interval_ms), 'draw_calls':500.0, 'detailed_timing_enabled':0.0, 'frame_slot_wait_calls':8.0, 'callbacks_enqueued':42.0,'callbacks_executed':42.0,'callbacks_pending':8.0} for i in range(1, points[index]+1)] @@ -665,6 +666,61 @@ def test_comparison_preserves_gpu_probe_presence_but_normalizes_private_path(sel (b/'sample.json').write_text(json.dumps(report_b)) self.assertFalse(comparer.compare([a,b])['comparable_identity']) + def test_gpu_timestamp_experiment_varies_probe_only_and_preserves_every_outcome(self): + off, off_report = self.sample_fixture('timestamp-off') + on, on_report = self.sample_fixture('timestamp-on', gpu_timestamps=True) + slow, slow_report = self.sample_fixture('timestamp-on-slower', interval_ms=75, + gpu_timestamps=True) + rejected, bad = self.sample_fixture('timestamp-streaming', gpu_timestamps=True, + event='Native GPU: adopted guest texture 0x1234') + self.assertNotIn('LEGO_NATIVE_GPU_TIMESTAMPS', off_report['flags']) + self.assertNotEqual(on_report['flags']['LEGO_NATIVE_GPU_TIMESTAMPS'], + slow_report['flags']['LEGO_NATIVE_GPU_TIMESTAMPS']) + paths = [off, on, rejected, slow] + self.assertFalse(comparer.compare(paths)['comparable_identity']) + report = comparer.compare(paths, varying='gpu-timestamps') + self.assertTrue(report['comparable_identity']) + self.assertEqual(report['experimental_variable'], 'gpu-timestamps') + self.assertEqual([row['status'] for row in report['outcomes']], + ['accepted', 'accepted', 'rejected', 'accepted']) + self.assertEqual([row['gpu_timestamps'] for row in report['outcomes']], + ['off', 'on', 'on', 'on']) + self.assertEqual(report['outcomes'][2]['rejection_reasons'], bad['rejection_reasons']) + self.assertEqual(report['accepted_aggregate']['off']['windows'], 1) + aggregate = report['accepted_aggregate']['on'] + self.assertEqual(aggregate['windows'], 2) + self.assertEqual(aggregate['frames'], on_report['metrics']['frames']+ + slow_report['metrics']['frames']) + self.assertAlmostEqual(aggregate['fps'], 1000/60) + self.assertLess(aggregate['fps'], on_report['metrics']['fps']) + self.assertAlmostEqual(report['candidate_vs_baseline_fps_percent'], -100/6) + self.assertFalse(report['source_default_changed']) + only_rejected = comparer.compare([rejected], varying='gpu-timestamps') + self.assertIsNone(only_rejected['comparable_identity']) + self.assertEqual(only_rejected['accepted_aggregate'], {}) + self.assertEqual(len(only_rejected['outcomes']), 1) + + def test_gpu_timestamp_comparison_rejects_every_other_changed_setting(self): + off, _ = self.sample_fixture('timestamp-strict-off') + on, original = self.sample_fixture('timestamp-strict-on', gpu_timestamps=True) + path = on/'sample.json' + for field, value in (('slots', 12), ('timer_wait', 'blocking')): + report = json.loads(json.dumps(original)); report[field] = value + path.write_text(json.dumps(report)) + self.assertFalse(comparer.compare([off,on], varying='gpu-timestamps')['comparable_identity']) + for flag, value in (('LEGO_NATIVE_COMMAND_SLOTS', '12'), ('REX_TIMER_WAIT_BLOCKING', '1'), + ('LEGO_NATIVE_TIMING', '0'), ('LEGO_NATIVE_BUFFER_WINDOWS', '0')): + report = json.loads(json.dumps(original)); report['flags'][flag] = value + path.write_text(json.dumps(report)) + self.assertFalse(comparer.compare([off,on], varying='gpu-timestamps')['comparable_identity']) + report = json.loads(json.dumps(original)); report['scene']['camera_id'] = 'moved-camera' + path.write_text(json.dumps(report)) + self.assertFalse(comparer.compare([off,on], varying='gpu-timestamps')['comparable_identity']) + path.write_text(json.dumps(original)) + self.assertTrue(comparer.compare([off,on], varying='gpu-timestamps')['comparable_identity']) + with self.assertRaisesRegex(ValueError, 'Unknown experimental variable'): + comparer.compare([off,on], varying='all-settings') + def test_inventory_limit_stops_iteration_before_materialization_and_refuses_aliases(self): directory=self.root/'bounded-tree';directory.mkdir() for index in range(5):(directory/str(index)).write_bytes(b'x') From feb37a5aa094a6f36279d5d3cc91903653c22de6 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:30:45 +0200 Subject: [PATCH 20/40] Build and fingerprint linked shader banks after their producers --- rexlego/CMakeLists.txt | 168 +++++++++-- tools/gpu-aot/CMakeLists.txt | 33 +- tools/gpu-aot/embed_linked_bank.py | 236 +++++++++++++++ tools/gpu-aot/test_binary_cache_build.py | 284 ++++++++++++++++++ .../gpu-aot/test_native_build_fingerprint.py | 146 +++++++++ 5 files changed, 840 insertions(+), 27 deletions(-) create mode 100644 tools/gpu-aot/embed_linked_bank.py create mode 100644 tools/gpu-aot/test_binary_cache_build.py create mode 100644 tools/gpu-aot/test_native_build_fingerprint.py diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt index d6b68e6..578db24 100644 --- a/rexlego/CMakeLists.txt +++ b/rexlego/CMakeLists.txt @@ -171,6 +171,29 @@ if(LEGODIMENSIONS_NATIVE_GPU) "${CMAKE_CURRENT_BINARY_DIR}/generated/gpu_native/shaders") set(LEGO_NATIVE_DXC "${LEGO_DXC_ROOT}/bin/${LEGO_DXC_ARCH}/dxc.exe") + # Both archive prelinkers are built for the Windows target even when + # standalone host shader compilation uses a host-native DXC executable. + if(CMAKE_CROSSCOMPILING AND NOT CMAKE_CROSSCOMPILING_EMULATOR) + message(FATAL_ERROR + "Native GPU cross-build prelinkers require CMAKE_CROSSCOMPILING_EMULATOR, including when LEGO_HOST_DXC_EXECUTABLE is set") + endif() + set(LEGO_HOST_DXC_EXECUTABLE "" CACHE FILEPATH + "Host-native DXC executable for build-time host shader compilation") + if(LEGO_HOST_DXC_EXECUTABLE) + if(NOT EXISTS "${LEGO_HOST_DXC_EXECUTABLE}") + message(FATAL_ERROR "Host DXC not found: ${LEGO_HOST_DXC_EXECUTABLE}") + endif() + set(LEGO_NATIVE_DXC "${LEGO_HOST_DXC_EXECUTABLE}") + set(LEGO_NATIVE_DXC_COMMAND "${LEGO_NATIVE_DXC}") + elseif(CMAKE_CROSSCOMPILING) + if(NOT CMAKE_CROSSCOMPILING_EMULATOR) + message(FATAL_ERROR + "Cross-building needs LEGO_HOST_DXC_EXECUTABLE or CMAKE_CROSSCOMPILING_EMULATOR for Windows DXC") + endif() + set(LEGO_NATIVE_DXC_COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} "${LEGO_NATIVE_DXC}") + else() + set(LEGO_NATIVE_DXC_COMMAND "${LEGO_NATIVE_DXC}") + endif() foreach(LEGO_HOST_SHADER IN ITEMS copy_vs copy_color_ps resolve_color_ps depth_pack_ps depth_restore_ps) if(LEGO_HOST_SHADER STREQUAL "copy_vs") set(LEGO_HOST_SHADER_PROFILE vs_6_6) @@ -183,7 +206,7 @@ if(LEGODIMENSIONS_NATIVE_GPU) OUTPUT "${LEGO_HOST_SHADER_HEADER}" COMMAND ${CMAKE_COMMAND} -E make_directory "${LEGO_NATIVE_GENERATED_SHADER_DIR}" - COMMAND "${LEGO_NATIVE_DXC}" + COMMAND ${LEGO_NATIVE_DXC_COMMAND} -T ${LEGO_HOST_SHADER_PROFILE} -E main -HV 2021 -all-resources-bound -Wno-ignored-attributes -I "${LEGO_NATIVE_HOST_SHADER_DIR}" @@ -191,6 +214,7 @@ if(LEGODIMENSIONS_NATIVE_GPU) -Vn g_${LEGO_HOST_SHADER}_dxil "${LEGO_NATIVE_HOST_SHADER_DIR}/${LEGO_HOST_SHADER}.hlsl" DEPENDS + "${LEGO_NATIVE_DXC}" "${LEGO_NATIVE_HOST_SHADER_DIR}/${LEGO_HOST_SHADER}.hlsl" "${LEGO_NATIVE_HOST_SHADER_DIR}/copy_common.hlsli" COMMENT "Compiling ${LEGO_HOST_SHADER}.hlsl (DXIL)" @@ -246,19 +270,84 @@ if(LEGODIMENSIONS_NATIVE_GPU) add_custom_command( OUTPUT "${LEGO_NATIVE_LINKED_SHADER_CACHE}" - COMMAND "$" + COMMAND lego_gpu_prelink "${LEGO_NATIVE_LINKED_SHADER_CACHE}" DEPENDS lego_gpu_prelink COMMENT "Pre-linking LEGO specialization-constant shader variants" VERBATIM) - target_sources(legodimensions PRIVATE "${LEGO_NATIVE_LINKED_SHADER_CACHE}") - set_source_files_properties("${LEGO_NATIVE_LINKED_SHADER_CACHE}" - PROPERTIES SKIP_PRECOMPILE_HEADERS ON) + add_custom_target(lego_main_linked_cache + DEPENDS "${LEGO_NATIVE_LINKED_SHADER_CACHE}") + add_dependencies(legodimensions lego_main_linked_cache) + # Large compressed banks otherwise become millions of C++ initializer + # values. Keep their metadata in C++, but embed their unchanged bytes in a + # target COFF object when host LLVM tools support the target's MSVC ABI. + find_program(LEGO_HOST_LLVM_MC NAMES llvm-mc llvm-mc-22 llvm-mc-21 llvm-mc-20 + NO_CMAKE_FIND_ROOT_PATH) + set(LEGO_NATIVE_BINARY_CACHE_ARCH "") + # Windows native generators can retain the host CMAKE_SYSTEM_PROCESSOR + # when clang-cl targets another architecture. Use the compiler's proven + # architecture and pointer width; ARM64EC/X86/unknown retain C++ embedding. + if(CMAKE_CXX_SIZEOF_DATA_PTR EQUAL 8) + if(CMAKE_CXX_COMPILER_ARCHITECTURE_ID STREQUAL "x64") + set(LEGO_NATIVE_BINARY_CACHE_ARCH x86_64) + elseif(CMAKE_CXX_COMPILER_ARCHITECTURE_ID STREQUAL "ARM64") + set(LEGO_NATIVE_BINARY_CACHE_ARCH aarch64) + endif() + endif() + set(LEGO_NATIVE_BINARY_CACHE_SUPPORTED OFF) + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND + CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC" AND + LEGO_NATIVE_BINARY_CACHE_ARCH AND LEGO_HOST_LLVM_MC AND + EXISTS "${LEGO_HOST_LLVM_MC}" AND NOT IS_DIRECTORY "${LEGO_HOST_LLVM_MC}" AND + Python3_Interpreter_FOUND) + set(LEGO_NATIVE_BINARY_CACHE_SUPPORTED ON) + endif() + option(LEGO_NATIVE_BINARY_CACHE + "Embed the main linked DXIL bytes as COFF instead of a C++ initializer" + ${LEGO_NATIVE_BINARY_CACHE_SUPPORTED}) + if(LEGO_NATIVE_BINARY_CACHE) + if(NOT LEGO_NATIVE_BINARY_CACHE_SUPPORTED) + message(FATAL_ERROR + "LEGO_NATIVE_BINARY_CACHE needs x64/ARM64 Windows Clang with the MSVC ABI, Python3, and host llvm-mc; set LEGO_NATIVE_BINARY_CACHE=OFF for C++ embedding") + endif() + set(LEGO_NATIVE_BINARY_CACHE_SCRIPT + "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/embed_linked_bank.py") + set(LEGO_NATIVE_BINARY_CACHE_DIR + "${CMAKE_CURRENT_BINARY_DIR}/generated/gpu_native/linked-bank") + set(LEGO_NATIVE_BINARY_CACHE_METADATA + "${LEGO_NATIVE_BINARY_CACHE_DIR}/linked-bank-metadata.cpp") + set(LEGO_NATIVE_BINARY_CACHE_OBJECT + "${LEGO_NATIVE_BINARY_CACHE_DIR}/linked-bank.obj") + add_custom_command( + OUTPUT "${LEGO_NATIVE_BINARY_CACHE_METADATA}" "${LEGO_NATIVE_BINARY_CACHE_OBJECT}" + BYPRODUCTS + "${LEGO_NATIVE_BINARY_CACHE_DIR}/linked-bank.bin" + "${LEGO_NATIVE_BINARY_CACHE_DIR}/linked-bank.s" + "${LEGO_NATIVE_BINARY_CACHE_DIR}/linked-bank-manifest.json" + COMMAND "${Python3_EXECUTABLE}" "${LEGO_NATIVE_BINARY_CACHE_SCRIPT}" + "${LEGO_NATIVE_LINKED_SHADER_CACHE}" "${LEGO_NATIVE_BINARY_CACHE_DIR}" + --llvm-mc "${LEGO_HOST_LLVM_MC}" + --arch "${LEGO_NATIVE_BINARY_CACHE_ARCH}" + DEPENDS "${LEGO_NATIVE_LINKED_SHADER_CACHE}" + "${LEGO_NATIVE_BINARY_CACHE_SCRIPT}" "${LEGO_HOST_LLVM_MC}" + COMMENT "Embedding main linked DXIL bank as ${LEGO_NATIVE_BINARY_CACHE_ARCH} COFF" + VERBATIM) + target_sources(legodimensions PRIVATE + "${LEGO_NATIVE_BINARY_CACHE_METADATA}" "${LEGO_NATIVE_BINARY_CACHE_OBJECT}") + set_source_files_properties("${LEGO_NATIVE_BINARY_CACHE_METADATA}" + PROPERTIES GENERATED TRUE SKIP_PRECOMPILE_HEADERS ON) + set_source_files_properties("${LEGO_NATIVE_BINARY_CACHE_OBJECT}" + PROPERTIES GENERATED TRUE EXTERNAL_OBJECT TRUE) + else() + target_sources(legodimensions PRIVATE "${LEGO_NATIVE_LINKED_SHADER_CACHE}") + set_source_files_properties("${LEGO_NATIVE_LINKED_SHADER_CACHE}" + PROPERTIES SKIP_PRECOMPILE_HEADERS ON) + endif() set(LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE "${CMAKE_CURRENT_SOURCE_DIR}/out/native-gpu/runtime-linked-dxil-cache.cpp") add_custom_command( OUTPUT "${LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE}" - COMMAND "$" + COMMAND lego_gpu_prelink_runtime "${LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE}" DEPENDS lego_gpu_prelink_runtime COMMENT "Pre-linking LEGO runtime shader variants" @@ -270,9 +359,9 @@ if(LEGODIMENSIONS_NATIVE_GPU) src/gpu_native/runtime_linked_shader_cache.cpp PROPERTIES OBJECT_DEPENDS "${LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE}") - # Fingerprint the exact native renderer inputs. CMAKE_CONFIGURE_DEPENDS - # makes a header-only edit trigger reconfigure as well as recompilation, - # and the fingerprint printed at startup proves which executable is live. + # Fingerprint the exact native renderer inputs after generated shader banks + # exist. Hashing the linked cache during configure prevented a fresh build + # from reaching the prelink target that creates that same file. set(LEGO_NATIVE_GPU_FINGERPRINT_INPUTS "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/plume/plume_d3d12.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.cpp" @@ -341,26 +430,53 @@ if(LEGODIMENSIONS_NATIVE_GPU) "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/runtime_shader_cache.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/runtime_linked_shader_cache.cpp" "${LEGO_NATIVE_SHADER_CACHE}" + "${LEGO_NATIVE_MICROCODE_INDEX}" "${LEGO_NATIVE_LINKED_SHADER_CACHE}" - "${LEGO_NATIVE_RUNTIME_SHADER_CACHE}") - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS - ${LEGO_NATIVE_GPU_FINGERPRINT_INPUTS}) - set(LEGO_NATIVE_GPU_FINGERPRINT_MATERIAL "") - foreach(LEGO_NATIVE_GPU_INPUT IN LISTS LEGO_NATIVE_GPU_FINGERPRINT_INPUTS) - file(SHA256 "${LEGO_NATIVE_GPU_INPUT}" LEGO_NATIVE_GPU_INPUT_HASH) - string(APPEND LEGO_NATIVE_GPU_FINGERPRINT_MATERIAL - "${LEGO_NATIVE_GPU_INPUT_HASH}") - endforeach() - string(SHA256 LEGO_NATIVE_GPU_FINGERPRINT_FULL - "${LEGO_NATIVE_GPU_FINGERPRINT_MATERIAL}") - string(SUBSTRING "${LEGO_NATIVE_GPU_FINGERPRINT_FULL}" 0 16 - LEGO_NATIVE_GPU_FINGERPRINT) - configure_file(src/gpu_native/build_info.h.in - "${CMAKE_CURRENT_BINARY_DIR}/generated/native_gpu_build_info.h" @ONLY) + "${LEGO_NATIVE_RUNTIME_SHADER_CACHE}" + "${LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE}") + # Include every native helper header so changes to upload, bounds or + # completion logic cannot retain an older executable fingerprint. + file(GLOB_RECURSE LEGO_NATIVE_GPU_LOCAL_HEADERS CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/*.h") + list(APPEND LEGO_NATIVE_GPU_FINGERPRINT_INPUTS ${LEGO_NATIVE_GPU_LOCAL_HEADERS}) + list(REMOVE_DUPLICATES LEGO_NATIVE_GPU_FINGERPRINT_INPUTS) + set(LEGO_NATIVE_GPU_FINGERPRINT_SCRIPT + "${CMAKE_CURRENT_BINARY_DIR}/generate-native-gpu-fingerprint.cmake") + file(CONFIGURE OUTPUT "${LEGO_NATIVE_GPU_FINGERPRINT_SCRIPT}" CONTENT [=[ +set(LEGO_NATIVE_GPU_FINGERPRINT_INPUTS "@LEGO_NATIVE_GPU_FINGERPRINT_INPUTS@") +set(LEGO_NATIVE_GPU_FINGERPRINT_MATERIAL "") +foreach(LEGO_NATIVE_GPU_INPUT IN LISTS LEGO_NATIVE_GPU_FINGERPRINT_INPUTS) + file(SHA256 "${LEGO_NATIVE_GPU_INPUT}" LEGO_NATIVE_GPU_INPUT_HASH) + string(APPEND LEGO_NATIVE_GPU_FINGERPRINT_MATERIAL "${LEGO_NATIVE_GPU_INPUT_HASH}") +endforeach() +string(SHA256 LEGO_NATIVE_GPU_FINGERPRINT_FULL "${LEGO_NATIVE_GPU_FINGERPRINT_MATERIAL}") +string(SUBSTRING "${LEGO_NATIVE_GPU_FINGERPRINT_FULL}" 0 16 LEGO_NATIVE_GPU_FINGERPRINT) +configure_file("@CMAKE_CURRENT_SOURCE_DIR@/src/gpu_native/build_info.h.in" + "@CMAKE_CURRENT_BINARY_DIR@/generated/native_gpu_build_info.h" @ONLY) +file(WRITE "@CMAKE_CURRENT_BINARY_DIR@/native-gpu-build.txt" + "LEGO Dimensions native GPU ${LEGO_NATIVE_GPU_FINGERPRINT}\n") +]=] @ONLY) + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/generated/native_gpu_build_info.h" + "${CMAKE_CURRENT_BINARY_DIR}/native-gpu-build.txt" + COMMAND ${CMAKE_COMMAND} -P "${LEGO_NATIVE_GPU_FINGERPRINT_SCRIPT}" + DEPENDS ${LEGO_NATIVE_GPU_FINGERPRINT_INPUTS} + "${LEGO_NATIVE_GPU_FINGERPRINT_SCRIPT}" + "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/build_info.h.in" + COMMENT "Fingerprinting native renderer sources and generated shader banks" + VERBATIM) + add_custom_target(lego_native_gpu_fingerprint + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/generated/native_gpu_build_info.h" + "${CMAKE_CURRENT_BINARY_DIR}/native-gpu-build.txt") + # Make generators must complete both bank producers before the fingerprint + # and metadata compilation consume their outputs through separate paths. + add_dependencies(lego_native_gpu_fingerprint + lego_main_linked_cache lego_runtime_linked_cache) + add_dependencies(legodimensions lego_native_gpu_fingerprint) + target_sources(legodimensions PRIVATE + "${CMAKE_CURRENT_BINARY_DIR}/generated/native_gpu_build_info.h") target_include_directories(legodimensions PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/generated") - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/native-gpu-build.txt" - "LEGO Dimensions native GPU ${LEGO_NATIVE_GPU_FINGERPRINT}\n") add_custom_command(TARGET legodimensions POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_CURRENT_BINARY_DIR}/native-gpu-build.txt" diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index aaf2d61..f3a4695 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -61,7 +61,7 @@ endif() if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") find_package(Python3 COMPONENTS Interpreter QUIET) if(Python3_Interpreter_FOUND) - foreach(check IN ITEMS command_slots constant_timing frame_cadence cull_state draw_diagnostic prelink_workers prelink_emission + foreach(check IN ITEMS native_build_fingerprint command_slots constant_timing frame_cadence cull_state draw_diagnostic prelink_workers prelink_emission frame_ring_transaction submission_waits pipeline_failures completion_callbacks cpu_resource_wait draw_upload query_retirement) add_test(NAME ${check} @@ -70,6 +70,37 @@ if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") --compiler ${CMAKE_CXX_COMPILER}) set_tests_properties(${check} PROPERTIES TIMEOUT 60) endforeach() + # This fixture cross-compiles a tiny SDK-free Windows ABI cache and + # inspects/archives a real COFF blob, rather than linking a host GPU app. + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(LEGO_BINARY_CACHE_TEST_TOOLS_FOUND TRUE) + foreach(tool IN ITEMS mc readobj objcopy ar ranlib) + find_program(LEGO_BINARY_CACHE_TEST_${tool} NAMES llvm-${tool} + NO_CMAKE_FIND_ROOT_PATH) + if(NOT LEGO_BINARY_CACHE_TEST_${tool}) + set(LEGO_BINARY_CACHE_TEST_TOOLS_FOUND FALSE) + endif() + endforeach() + if(WIN32) + find_program(LEGO_BINARY_CACHE_TEST_GENERATOR NAMES ninja + NO_CMAKE_FIND_ROOT_PATH) + else() + find_program(LEGO_BINARY_CACHE_TEST_GENERATOR NAMES make gmake + NO_CMAKE_FIND_ROOT_PATH) + endif() + if(LEGO_BINARY_CACHE_TEST_TOOLS_FOUND AND LEGO_BINARY_CACHE_TEST_GENERATOR) + add_test(NAME binary_cache_build + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_binary_cache_build.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/binary_cache_build + --compiler ${CMAKE_CXX_COMPILER} + --llvm-mc ${LEGO_BINARY_CACHE_TEST_mc} + --llvm-readobj ${LEGO_BINARY_CACHE_TEST_readobj} + --llvm-objcopy ${LEGO_BINARY_CACHE_TEST_objcopy} + --llvm-ar ${LEGO_BINARY_CACHE_TEST_ar} + --llvm-ranlib ${LEGO_BINARY_CACHE_TEST_ranlib}) + set_tests_properties(binary_cache_build PROPERTIES TIMEOUT 60) + endif() + endif() if(UNIX) add_test(NAME prelink_publication COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_prelink_publication.py diff --git a/tools/gpu-aot/embed_linked_bank.py b/tools/gpu-aot/embed_linked_bank.py new file mode 100644 index 0000000..b9fe790 --- /dev/null +++ b/tools/gpu-aot/embed_linked_bank.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Stream a generated linked DXIL bank into metadata C++ and an MSVC ABI COFF blob. + +The original bank remains the input/provenance artifact. Only its compressed +byte-array definition moves to a read-only object section; headers, tables and +size constants retain their original source text. No C++ payload is compiled. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile + + +MAX_LINE = 64 * 1024 +MAX_METADATA = 16 * 1024 * 1024 +DECLARATION = re.compile(rb"const uint8_t g_compressedLinkedDxilCache\[\] =\s*(\{)?\s*") +HEX_LITERAL = re.compile(rb'"(?:\\x[0-9A-Fa-f]{2})*"') +SYMBOL = "?g_compressedLinkedDxilCache@@3QBEB" +OUTPUTS = ("linked-bank-metadata.cpp", "linked-bank.bin", "linked-bank.s", + "linked-bank.obj", "linked-bank-manifest.json") + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def identity(path): + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return {"bytes": path.stat().st_size, "sha256": digest.hexdigest()} + + +def stat_identity(stat): + return stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns + + +class Lines: + def __init__(self, source): + self.source = source + self.digest = hashlib.sha256() + self.bytes = 0 + self.number = 0 + + def read(self): + raw = self.source.readline(MAX_LINE + 1) + require(len(raw) <= MAX_LINE, "Generated source line exceeds 64 KiB") + self.digest.update(raw) + self.bytes += len(raw) + self.number += bool(raw) + return raw, raw.rstrip(b"\r\n").strip() + + def next(self): + raw, content = self.read() + require(raw, "Truncated generated bank") + return raw, content + + def expect(self, wanted): + _, content = self.next() + require(content == wanted, + f"Unexpected generated structure on line {self.number}: expected {wanted!r}") + + +def numeric(lines, sink, terminator): + digest = hashlib.sha256() + count = 0 + while True: + _, content = lines.next() + if content == terminator: + return count, digest.hexdigest() + require(not content or content.endswith(b","), "Numeric initializer must end in a comma") + pending = bytearray() + for token in content.split(b",")[:-1]: + token = token.strip() + require(1 <= len(token) <= 3 and token.isdigit(), "Invalid numeric payload token") + value = int(token) + require(value <= 255, "Numeric payload byte exceeds 255") + pending.append(value) + sink.write(pending) + digest.update(pending) + count += len(pending) + + +def literals(lines, sink=None): + digest = hashlib.sha256() + count = 0 + while True: + _, content = lines.next() + if content == b"#if defined(__clang__)": + break + require(HEX_LITERAL.fullmatch(content), "Invalid escaped payload literal") + decoded = bytes.fromhex(content[1:-1].replace(b"\\x", b"").decode("ascii")) + if sink is not None: + sink.write(decoded) + digest.update(decoded) + count += len(decoded) + lines.expect(b"#pragma clang diagnostic pop") + lines.expect(b"#endif") + lines.expect(b"#endif") + lines.expect(b";") + return count, digest.hexdigest() + + +def extract(source_path, metadata_path, binary_path): + before = source_path.stat() + prefix = bytearray() + footer = bytearray() + literal_identity = None + with source_path.open("rb") as source, metadata_path.open("wb") as metadata, \ + binary_path.open("wb") as binary: + require(stat_identity(before) == stat_identity(os.fstat(source.fileno())), + "Input changed before it could be opened") + lines = Lines(source) + while True: + raw, content = lines.next() + match = DECLARATION.fullmatch(content) + if match: + break + prefix.extend(raw) + require(len(prefix) <= MAX_METADATA, "Oversized linked table/header metadata") + metadata.write(raw) + if match[1]: + source_format = "numeric" + count, digest = numeric(lines, binary, b"};") + else: + source_format = "numeric-with-literal-fallback" + lines.expect(b"#if defined(_MSC_VER) && !defined(__clang__)") + lines.expect(b"{") + count, digest = numeric(lines, binary, b"}") + for wanted in (b"#else", b"#if defined(__clang__)", + b"#pragma clang diagnostic push", + b'#pragma clang diagnostic ignored "-Woverlength-strings"', b"#endif"): + lines.expect(wanted) + literal_identity = literals(lines) + while True: + raw, content = lines.read() + if not raw: + break + require(not DECLARATION.fullmatch(content), "Duplicate compressed-array definition") + footer.extend(raw) + require(len(prefix) + len(footer) <= MAX_METADATA, "Oversized linked metadata") + metadata.write(raw) + binary.flush() + os.fsync(binary.fileno()) + metadata.flush() + os.fsync(metadata.fileno()) + source_identity = {"bytes": lines.bytes, "sha256": lines.digest.hexdigest()} + require(stat_identity(before) == stat_identity(os.fstat(source.fileno())) == + stat_identity(source_path.stat()), "Input changed during extraction") + + def scalar(data, name): + matches = re.findall(rb"const size_t " + name + rb"\s*=\s*(\d+)\s*;", data) + require(len(matches) == 1, f"Expected exactly one {name.decode()} constant") + return int(matches[0]) + + entries = scalar(prefix, b"g_linkedShaderCacheEntryCount") + compressed = scalar(footer, b"g_linkedDxilCacheCompressedSize") + decompressed = scalar(footer, b"g_linkedDxilCacheDecompressedSize") + if compressed == 0: + # The C++ numeric emitter uses a zero sentinel for an empty array; + # there are no compressed payload bytes or implicit string NULs to embed. + require((count == 0 and digest == hashlib.sha256(b"").hexdigest()) or + (count == 1 and digest == hashlib.sha256(b"\0").hexdigest()), + "Invalid empty-array numeric sentinel") + with binary_path.open("wb"): + pass + count, digest = 0, hashlib.sha256(b"").hexdigest() + require(compressed == count, "Compressed size metadata differs from numeric payload length") + require(literal_identity is None or literal_identity == (count, digest), + "Numeric and literal payload bytes differ") + require(b"g_compressedLinkedDxilCache" not in prefix + footer, + "Metadata still contains a compressed-array definition/reference") + return {"input_format": source_format, "input": {"path": str(source_path), **source_identity}, + "compressed_payload": {"bytes": count, "sha256": digest}, + "decompressed_bytes": decompressed, "linked_entries": entries, + "numeric_and_literal_identical": literal_identity is not None, + "metadata_preserved": True, "payload_type": "const uint8_t[]", + "implicit_string_nul_embedded": False} + + +def assembly(binary_path): + # Quoted assembler names preserve MSVC C++ array mangling. JSON quoting is + # used only for assembler string syntax; subprocess never invokes a shell. + return ('.section .rdata,"dr"\n.p2align 4\n' + f'.globl "{SYMBOL}"\n"{SYMBOL}":\n' + f'.incbin {json.dumps(str(binary_path), ensure_ascii=False)}\n') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + parser.add_argument("output", type=Path, help="Directory for five generated linked-bank artifacts") + parser.add_argument("--llvm-mc", default="llvm-mc") + parser.add_argument("--arch", choices=("x86_64", "aarch64"), default="x86_64") + parser.add_argument("--timeout", type=int, default=60, help="Assembly timeout in seconds") + args = parser.parse_args() + require(args.timeout > 0, "Assembly timeout must be positive") + source = args.input.resolve(strict=True) + output = args.output.resolve() + require(source.is_file(), "Input must be a generated source file") + require(source not in [output / name for name in OUTPUTS], "Output cannot replace the input") + output.mkdir(parents=True, exist_ok=True) + input_identity = stat_identity(source.stat()) + with tempfile.TemporaryDirectory(prefix=".linked-bank-stage-", dir=output) as temporary: + stage = Path(temporary) + record = extract(source, stage / OUTPUTS[0], stage / OUTPUTS[1]) + stage_assembly = stage / OUTPUTS[2] + stage_assembly.write_text(assembly(stage / OUTPUTS[1]), encoding="utf-8") + target = args.arch + "-pc-windows-msvc" + subprocess.run([args.llvm_mc, "-triple=" + target, "-filetype=obj", + str(stage_assembly), "-o", str(stage / OUTPUTS[3])], + check=True, timeout=args.timeout) + require((stage / OUTPUTS[3]).stat().st_size > 0, "Assembler produced an empty object") + # The published assembly refers to the published blob, not its temporary + # name. Assembly changes no payload/metadata and is not an input to this run. + stage_assembly.write_text(assembly(output / OUTPUTS[1]), encoding="utf-8") + record.update(target=target, coff_symbol=SYMBOL, assembler=args.llvm_mc, + outputs={name: identity(stage / name) for name in OUTPUTS[:-1]}) + (stage / OUTPUTS[4]).write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + require(stat_identity(source.stat()) == input_identity, + "Input changed before publication; previous outputs preserved") + # All parsing, comparison and assembly complete before publishing. Each + # replacement is atomic; the manifest is the final completion marker. + for name in OUTPUTS: + os.replace(stage / name, output / name) + print(json.dumps(record, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/gpu-aot/test_binary_cache_build.py b/tools/gpu-aot/test_binary_cache_build.py new file mode 100644 index 0000000..966a556 --- /dev/null +++ b/tools/gpu-aot/test_binary_cache_build.py @@ -0,0 +1,284 @@ +"""Build actual native binary-cache CMake rules with a tiny Windows COFF bank.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import time +import zlib + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--compiler', default='clang++') + parser.add_argument('--cmake', default='cmake') + parser.add_argument('--generator', default='Ninja' if os.name == 'nt' else 'Unix Makefiles') + parser.add_argument('--source-root', type=Path, + default=Path(__file__).resolve().parents[2], + help='Production source root, or an explicit isolated review snapshot') + for tool in ('mc', 'readobj', 'objcopy', 'ar', 'ranlib'): + parser.add_argument('--llvm-' + tool) + args = parser.parse_args() + root = args.source_root.resolve() + text = (root / 'rexlego/CMakeLists.txt').read_text() + start = text.index(' add_custom_target(lego_main_linked_cache') + end = text.index(' set(LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE', start) + integration = text[start:end] + dependency_start = text.index(' add_dependencies(lego_native_gpu_fingerprint\n') + dependency_end = text.index(')', dependency_start) + 1 + fingerprint_order = text[dependency_start:dependency_end] + tools = {name: getattr(args, name.replace('-', '_')) or shutil.which(name) + for name in ('llvm-mc', 'llvm-readobj', 'llvm-objcopy', 'llvm-ar', 'llvm-ranlib')} + assert all(tools.values()), 'host LLVM assembler/object/archive tools are required' + args.output.mkdir(parents=True, exist_ok=True) + output = Path(tempfile.mkdtemp(prefix='fixture with spaces ', dir=args.output.resolve())) + source = output / 'source with spaces/rexlego' + build = output / 'build with spaces' + source.mkdir(parents=True) + script = source.parent / 'tools/gpu-aot/embed_linked_bank.py' + script.parent.mkdir(parents=True) + shutil.copyfile(root / 'tools/gpu-aot/embed_linked_bank.py', script) + include = source / 'include' + header = include / 'gpu/shaders/linked_shader_cache.h' + header.parent.mkdir(parents=True) + shutil.copyfile(root / 'rexlego/src/gpu_native/linked_shader_cache.h', header) + # The actual ABI header is compiled unchanged. These minimal platform + # standard-header shims remove any dependency on a Windows SDK installation. + (include / 'cstddef').write_text('#pragma once\nusing size_t=__SIZE_TYPE__;\n') + (include / 'cstdint').write_text('#pragma once\nusing uint8_t=unsigned char;\n' + 'using uint32_t=unsigned int;\nusing uint64_t=unsigned long long;\n') + (source / 'probe.cpp').write_text('#include "gpu/shaders/linked_shader_cache.h"\n' + 'const uint8_t* bank_address(){return g_compressedLinkedDxilCache;}\n') + (source / 'copy-bank.cmake').write_text(''' +file(APPEND "${OUTPUT}.calls" "built\\n") +execute_process(COMMAND "${CMAKE_COMMAND}" -E copy "${INPUT}" "${OUTPUT}" + COMMAND_ERROR_IS_FATAL ANY) +''') + raw = source / 'main-raw.cpp' + linked = source / 'main-linked.cpp' + + def bank(version): + expanded = bytes(range(256))[::version] * 4 + payload = zlib.compress(expanded) + prefix = ('#include "gpu/shaders/linked_shader_cache.h"\n' + 'LinkedShaderCacheEntry g_linkedShaderCacheEntries[] = {\n' + f'{{ 0x1234, 0, 0, {len(expanded)} }},\n}};\n' + 'const size_t g_linkedShaderCacheEntryCount = 1;\n') + footer = (f'const size_t g_linkedDxilCacheCompressedSize = {len(payload)};\n' + f'const size_t g_linkedDxilCacheDecompressedSize = {len(expanded)};\n') + body = ''.join(','.join(str(value) for value in payload[at:at+32]) + ',\n' + for at in range(0, len(payload), 32)) + raw.write_text(prefix + 'const uint8_t g_compressedLinkedDxilCache[] = {\n' + body + '};\n' + footer) + return payload, (prefix + footer).encode() + + payload, metadata = bank(1) + cmake = '''cmake_minimum_required(VERSION 3.25) +project(BinaryCacheFixture LANGUAGES CXX) +if(DEFINED FIXTURE_ARCHITECTURE) + set(CMAKE_CXX_COMPILER_ARCHITECTURE_ID "${FIXTURE_ARCHITECTURE}") +endif() +if(DEFINED FIXTURE_POINTER_BYTES) + set(CMAKE_CXX_SIZEOF_DATA_PTR "${FIXTURE_POINTER_BYTES}") +endif() +find_package(Python3 COMPONENTS Interpreter REQUIRED) +add_library(legodimensions STATIC probe.cpp) +target_include_directories(legodimensions PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include") +set(LEGO_NATIVE_LINKED_SHADER_CACHE "${CMAKE_CURRENT_SOURCE_DIR}/main-linked.cpp") +add_custom_command(OUTPUT "${LEGO_NATIVE_LINKED_SHADER_CACHE}" + COMMAND ${CMAKE_COMMAND} + "-DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/main-raw.cpp" + "-DOUTPUT=${LEGO_NATIVE_LINKED_SHADER_CACHE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/copy-bank.cmake" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/main-raw.cpp" + VERBATIM) +''' + after = ''' +add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/fingerprint.txt" + COMMAND ${CMAKE_COMMAND} -E copy "${LEGO_NATIVE_LINKED_SHADER_CACHE}" + "${CMAKE_CURRENT_BINARY_DIR}/fingerprint.txt" + DEPENDS "${LEGO_NATIVE_LINKED_SHADER_CACHE}" + VERBATIM) +add_custom_target(lego_native_gpu_fingerprint + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/fingerprint.txt") +add_custom_target(lego_runtime_linked_cache) +add_dependencies(legodimensions lego_native_gpu_fingerprint) +get_target_property(fixture_sources legodimensions SOURCES) +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/sources.txt" "${fixture_sources}") +''' + (source / 'CMakeLists.txt').write_text(cmake + integration + after + fingerprint_order) + configure = [args.cmake, '-S', str(source), '-B', str(build), '-G', args.generator, + '-DCMAKE_SYSTEM_NAME=Windows', '-DCMAKE_SYSTEM_PROCESSOR=AMD64', + '-DCMAKE_CXX_COMPILER=' + args.compiler, + '-DCMAKE_CXX_COMPILER_TARGET=x86_64-pc-windows-msvc', + '-DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY', + '-DCMAKE_CXX_FLAGS=-nostdinc++', '-DCMAKE_BUILD_TYPE=Release', + '-DCMAKE_AR=' + tools['llvm-ar'], '-DCMAKE_RANLIB=' + tools['llvm-ranlib'], + '-DLEGO_HOST_LLVM_MC=' + tools['llvm-mc']] + + def run(command, success=True): + result = subprocess.run(command, text=True, capture_output=True, timeout=30) + with (output / 'commands.log').open('a') as log: + log.write(json.dumps(command) + '\n' + result.stdout + result.stderr) + assert (result.returncode == 0) == success, result.stdout + result.stderr + return result + + run(configure) + assert not linked.exists(), 'configure must not run the bank producer' + assert 'linked-bank-metadata.cpp' in (build / 'sources.txt').read_text() + assert 'linked-bank.obj' in (build / 'sources.txt').read_text() + assert str(linked) not in (build / 'sources.txt').read_text() + artifacts = build / 'generated/gpu_native/linked-bank' + archive = build / 'legodimensions.lib' + calls = Path(str(linked) + '.calls') + + def verify(expected_payload, expected_metadata, expected_calls, target_build=None, + machine='AMD64', target='x86_64-pc-windows-msvc'): + current = target_build or build + current_artifacts = current / 'generated/gpu_native/linked-bank' + current_archive = current / 'legodimensions.lib' + run([args.cmake, '--build', str(current), '--target', 'legodimensions', '--parallel', '1']) + assert calls.read_text().splitlines() == ['built'] * expected_calls + assert (current / 'fingerprint.txt').read_bytes() == linked.read_bytes() == raw.read_bytes() + assert (current_artifacts / 'linked-bank.bin').read_bytes() == expected_payload + assert (current_artifacts / 'linked-bank-metadata.cpp').read_bytes() == expected_metadata + manifest = json.loads((current_artifacts / 'linked-bank-manifest.json').read_text()) + assert manifest['target'] == target + assert manifest['compressed_payload']['bytes'] == len(expected_payload) + assert manifest['compressed_payload']['sha256'] == hashlib.sha256(expected_payload).hexdigest() + assert not manifest['implicit_string_nul_embedded'] + symbols = run([tools['llvm-readobj'], '--file-headers', '--symbols', + str(current_artifacts / 'linked-bank.obj')]).stdout + assert 'IMAGE_FILE_MACHINE_' + machine in symbols + assert '?g_compressedLinkedDxilCache@@3QBEB' in symbols + metadata_objects = list(current.rglob('linked-bank-metadata.cpp.obj')) + assert len(metadata_objects) == 1 + metadata_headers = run([tools['llvm-readobj'], '--file-headers', + str(metadata_objects[0])]).stdout + assert 'IMAGE_FILE_MACHINE_' + machine in metadata_headers + extracted = current_artifacts / 'extracted-rdata.bin' + run([tools['llvm-objcopy'], '--dump-section', '.rdata=' + str(extracted), + str(current_artifacts / 'linked-bank.obj'), + str(current_artifacts / 'inspection-copy.obj')]) + assert extracted.read_bytes() == expected_payload + # Actual target archiver must consume the generated external object, + # not merely produce independent files that never reach the target. + members = run([tools['llvm-ar'], 't', str(current_archive)]).stdout + assert 'linked-bank.obj' in members and 'linked-bank-metadata.cpp.obj' in members + + verify(payload, metadata, 1) + tracked = [archive, artifacts / 'linked-bank.obj', artifacts / 'linked-bank-metadata.cpp', + build / 'fingerprint.txt'] + stamps = [path.stat().st_mtime_ns for path in tracked] + verify(payload, metadata, 1) + assert [path.stat().st_mtime_ns for path in tracked] == stamps, 'unchanged build must be a no-op' + time.sleep(1.1) # Make timestamp resolution; no compiler runs during this wait. + payload, metadata = bank(2) + verify(payload, metadata, 2) + assert archive.stat().st_mtime_ns != stamps[0] + (artifacts / 'linked-bank.obj').unlink() + verify(payload, metadata, 2) + + # Target architecture comes from compiler detection, independently of the + # host CPU. A spoofed/unknown system processor must not reject proven x64. + wrong_host = output / 'x64 unknown host' + command = list(configure) + command[command.index('-B') + 1] = str(wrong_host) + run(command + ['-DCMAKE_SYSTEM_PROCESSOR=unknown', '-DLEGO_NATIVE_BINARY_CACHE=ON']) + verify(payload, metadata, 2, target_build=wrong_host) + + # An actual ARM64 compiler target on an AMD64 host reproduces the old + # architecture-selection error. Inspect both real COFF producers, not + # merely configure variables or the assembly text. + compiler_targets = run([args.compiler, '--print-targets']).stdout + assembler_targets = run([tools['llvm-mc'], '--version']).stdout + arm64_tested = 'aarch64' in compiler_targets and 'aarch64' in assembler_targets + if arm64_tested: + arm64 = output / 'arm64 target amd64 host' + command = list(configure) + command[command.index('-B') + 1] = str(arm64) + run(command + ['-DCMAKE_CXX_COMPILER_TARGET=aarch64-pc-windows-msvc', + '-DCMAKE_SYSTEM_PROCESSOR=AMD64', '-DLEGO_NATIVE_BINARY_CACHE=ON']) + verify(payload, metadata, 2, target_build=arm64, + machine='ARM64', target='aarch64-pc-windows-msvc') + else: + print('UNTESTED: supplied compiler/assembler do not both advertise AArch64') + + # Actual i686 detection and controlled unsupported compiler-attribute + # cases exercise the exact production gate; host CPU names cannot mask + # ARM64EC/X86/unknown architectures or a non-64-bit pointer ABI. + for name, arguments in ( + ('missing tools', ['-DLEGO_HOST_LLVM_MC=' + str(output / 'absent llvm-mc')]), + ('actual x86 target', ['-DCMAKE_CXX_COMPILER_TARGET=i686-pc-windows-msvc']), + ('arm64ec attribute', ['-DFIXTURE_ARCHITECTURE=ARM64EC']), + ('x86 attribute', ['-DFIXTURE_ARCHITECTURE=X86']), + ('unknown attribute', ['-DFIXTURE_ARCHITECTURE=']), + ('wrong pointer ABI', ['-DFIXTURE_POINTER_BYTES=4'])): + target = output / name + command = list(configure) + command[command.index('-B') + 1] = str(target) + run(command + ['-DLEGO_NATIVE_BINARY_CACHE=ON'] + arguments, success=False) + + # Direct production-parser cases cover both accepted source formats, + # branch mismatch and failed assembly without running DXC/prelinking. + parser_source = output / 'parser input.cpp' + parser_output = output / 'parser outputs' + declaration = 'const uint8_t g_compressedLinkedDxilCache[] = {\n' + prefix, remainder = raw.read_text().split(declaration, 1) + numeric, footer = remainder.split('};\n', 1) + literals = ''.join('"' + ''.join('\\x%02X' % byte for byte in payload[at:at+256]) + '"\n' + for at in range(0, len(payload), 256)) + dual = (prefix + 'const uint8_t g_compressedLinkedDxilCache[] =\n' + '#if defined(_MSC_VER) && !defined(__clang__)\n{\n' + numeric + '}\n#else\n' + '#if defined(__clang__)\n#pragma clang diagnostic push\n' + '#pragma clang diagnostic ignored "-Woverlength-strings"\n#endif\n' + literals + + '#if defined(__clang__)\n#pragma clang diagnostic pop\n#endif\n#endif\n;\n' + footer) + invoke = [sys.executable, str(script), str(parser_source), str(parser_output), + '--llvm-mc', tools['llvm-mc']] + for label, contents in (('numeric', raw.read_text()), ('numeric-with-literal-fallback', dual)): + parser_source.write_text(contents) + run(invoke) + record = json.loads((parser_output / 'linked-bank-manifest.json').read_text()) + assert record['input_format'] == label + assert record['numeric_and_literal_identical'] == (label != 'numeric') + assert (parser_output / 'linked-bank.bin').read_bytes() == payload + assert (parser_output / 'linked-bank-metadata.cpp').read_bytes() == metadata + extracted = parser_output / 'inspection.bin' + run([tools['llvm-objcopy'], '--dump-section', '.rdata=' + str(extracted), + str(parser_output / 'linked-bank.obj'), str(parser_output / 'inspection.obj')]) + assert extracted.read_bytes() == payload + + output_names = ('linked-bank-metadata.cpp', 'linked-bank.bin', 'linked-bank.s', + 'linked-bank.obj', 'linked-bank-manifest.json') + def prior_outputs(): + return {name: (hashlib.sha256((parser_output / name).read_bytes()).hexdigest(), + (parser_output / name).stat().st_mtime_ns) for name in output_names} + preserved = prior_outputs() + for invalid in (dual.replace('\\x%02X' % payload[0], '\\x%02X' % (payload[0] ^ 1), 1), + raw.read_text().replace(declaration, declaration + '256,\n', 1), + raw.read_text().replace('g_linkedDxilCacheCompressedSize = %d;' % len(payload), + 'g_linkedDxilCacheCompressedSize = %d;' % (len(payload)+1))): + parser_source.write_text(invalid) + run(invoke, success=False) + assert prior_outputs() == preserved, 'parser failure modified published outputs' + parser_source.write_text(dual) + failed_assembler = list(invoke) + # Python rejects llvm-mc's -triple option on every host; this is a real + # nonzero subprocess result, without shell scripts or platform launchers. + failed_assembler[-1] = sys.executable + run(failed_assembler, success=False) + assert prior_outputs() == preserved, 'assembler failure modified published outputs' + assert not list(parser_output.glob('.linked-bank-stage-*')), 'failed staging directory leaked' + print('PASS: real binary-cache CMake rules produce byte-exact MSVC ABI COFF and metadata; ' + 'Make consumers share one producer, target archives external object, unchanged no-op, ' + 'input/object changes rebuild, compiler architecture/pointer ABI checked; ' + 'numeric/dual parser and failed publication preserved all prior outputs; ' + 'ARM64 target verified=' + str(arm64_tested)) + + +if __name__ == '__main__': + main() diff --git a/tools/gpu-aot/test_native_build_fingerprint.py b/tools/gpu-aot/test_native_build_fingerprint.py new file mode 100644 index 0000000..d1cbffd --- /dev/null +++ b/tools/gpu-aot/test_native_build_fingerprint.py @@ -0,0 +1,146 @@ +"""Build the production native fingerprint rule against synthetic shader banks.""" + +import argparse +import hashlib +import os +from pathlib import Path +import re +import subprocess +import tempfile + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + parser.add_argument("--cmake", default="cmake") + parser.add_argument("--generator", default="Ninja" if os.name == "nt" else "Unix Makefiles") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + text = (root / "rexlego/CMakeLists.txt").read_text() + begin = text.index(" # Fingerprint the exact native renderer inputs") + end = text.index("\nendif()", begin) + production = text[begin:end] + output = args.output.resolve() + # The test needs a fresh tree: generated archives must be absent at configure. + output.mkdir(parents=True, exist_ok=True) + output = Path(tempfile.mkdtemp(prefix="fixture-", dir=output)) + source = output / "source/rexlego" + build = output / "build" + source.mkdir(parents=True) + for header in (root / "rexlego/src/gpu_native").rglob("*.h"): + path = source / "src/gpu_native" / header.relative_to(root / "rexlego/src/gpu_native") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("// synthetic native helper header\n") + input_files = [] + for relative in re.findall(r'"\$\{CMAKE_CURRENT_SOURCE_DIR\}/([^"\n]+)"', production): + if "*" in relative: + continue + path = (source / relative).resolve() + if path in input_files or path.suffix not in (".cpp", ".h", ".hlsl", ".hlsli", ".in"): + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("// synthetic fingerprint input\n") + input_files.append(path) + template = source / "src/gpu_native/build_info.h.in" + template.write_text((root / "rexlego/src/gpu_native/build_info.h.in").read_text()) + raw = source / "main-raw.cpp" + runtime = source / "runtime-raw.cpp" + microcode_index = source / "main-index.cpp" + linked = source / "main-linked.cpp" + runtime_linked = source / "runtime-linked.cpp" + raw.write_text("main archive v1\n") + runtime.write_text("runtime archive v1\n") + microcode_index.write_text("microcode lookup index v1\n") + (source / "main.cpp").write_text( + '#include "native_gpu_build_info.h"\n#include \n' + 'int main(){std::puts(legodimensions::gpu_native::kNativeGpuBuildFingerprint);}\n') + (source / "copy-bank.cmake").write_text(''' +file(APPEND "${OUTPUT}.calls" "built\\n") +execute_process(COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${INPUT}" "${OUTPUT}" + COMMAND_ERROR_IS_FATAL ANY) +''') + cmake = '''cmake_minimum_required(VERSION 3.25) +project(NativeFingerprintFixture LANGUAGES CXX) +add_executable(legodimensions main.cpp) +set(LEGO_NATIVE_SHADER_CACHE "${CMAKE_CURRENT_SOURCE_DIR}/main-raw.cpp") +set(LEGO_NATIVE_MICROCODE_INDEX "${CMAKE_CURRENT_SOURCE_DIR}/main-index.cpp") +set(LEGO_NATIVE_RUNTIME_SHADER_CACHE "${CMAKE_CURRENT_SOURCE_DIR}/runtime-raw.cpp") +set(LEGO_NATIVE_LINKED_SHADER_CACHE "${CMAKE_CURRENT_SOURCE_DIR}/main-linked.cpp") +set(LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE "${CMAKE_CURRENT_SOURCE_DIR}/runtime-linked.cpp") +foreach(pair IN ITEMS "main" "runtime") + add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/${pair}-linked.cpp" + COMMAND ${CMAKE_COMMAND} + "-DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/${pair}-raw.cpp" + "-DOUTPUT=${CMAKE_CURRENT_SOURCE_DIR}/${pair}-linked.cpp" + -P "${CMAKE_CURRENT_SOURCE_DIR}/copy-bank.cmake" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${pair}-raw.cpp" + VERBATIM) +endforeach() +add_custom_target(lego_runtime_linked_cache DEPENDS "${LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE}") +add_custom_target(lego_main_linked_cache DEPENDS "${LEGO_NATIVE_LINKED_SHADER_CACHE}") +add_dependencies(legodimensions lego_runtime_linked_cache) +''' + (source / "CMakeLists.txt").write_text(cmake + production) + + def run(command): + return subprocess.run(command, check=True, text=True, capture_output=True, timeout=30) + + configure = run([args.cmake, "-S", str(source), "-B", str(build), "-G", args.generator, + "-DCMAKE_CXX_COMPILER=" + args.compiler, "-DCMAKE_BUILD_TYPE=Release"]) + (output / "configure.log").write_text(configure.stdout + configure.stderr) + assert not linked.exists() and not runtime_linked.exists() + assert not (build / "generated/native_gpu_build_info.h").exists() + + def check_build(): + result = run([args.cmake, "--build", str(build), "--parallel", "2"]) + with (output / "build.log").open("a") as log: + log.write(result.stdout + result.stderr) + assert linked.read_bytes() == raw.read_bytes() + assert runtime_linked.read_bytes() == runtime.read_bytes() + # Parse the actual generated hash script's ordered input list rather + # than assuming that the fixture knows which files production fingerprints. + script = (build / "generate-native-gpu-fingerprint.cmake").read_text() + paths = re.search(r'set\(LEGO_NATIVE_GPU_FINGERPRINT_INPUTS "([^"]+)"\)', script).group(1).split(";") + assert str(microcode_index) in paths + hashes = "".join(hashlib.sha256(Path(path).read_bytes()).hexdigest() for path in paths) + expected = hashlib.sha256(hashes.encode()).hexdigest()[:16] + header = (build / "generated/native_gpu_build_info.h").read_text() + assert f'"{expected}"' in header + assert (build / "native-gpu-build.txt").read_text() == f"LEGO Dimensions native GPU {expected}\n" + binary = build / ("legodimensions.exe" if (build / "legodimensions.exe").exists() else "legodimensions") + assert run([str(binary)]).stdout.strip() == expected + return expected + + first = check_build() + assert Path(str(linked) + ".calls").read_text().splitlines() == ["built"] + assert Path(str(runtime_linked) + ".calls").read_text().splitlines() == ["built"] + # Changing either bank must re-link, regenerate provenance, and recompile + # the executable that includes the fingerprint header. + raw.write_text("main archive v2\n") + second = check_build() + assert second != first + assert Path(str(linked) + ".calls").read_text().splitlines() == ["built", "built"] + assert Path(str(runtime_linked) + ".calls").read_text().splitlines() == ["built"] + runtime.write_text("runtime archive v2\n") + third = check_build() + assert third != second + assert Path(str(runtime_linked) + ".calls").read_text().splitlines() == ["built", "built"] + # An index-only edit changes which shader an existing bank supplies. Its + # provenance must change independently of any bank producer invocation. + microcode_index.write_text("microcode lookup index v2\n") + fourth = check_build() + assert fourth != third + assert Path(str(linked) + ".calls").read_text().splitlines() == ["built", "built"] + assert Path(str(runtime_linked) + ".calls").read_text().splitlines() == ["built", "built"] + # Header-only changes are build dependencies without a reconfigure/read + # of not-yet-generated banks during the initial CMake configure. + (source / "src/gpu_native/index_draw.h").write_text("// synthetic bounds helper v2\n") + fifth = check_build() + assert fifth != fourth + assert check_build() == fifth + print("PASS: fresh native fingerprint configure; generated banks precede hashing; bank/index/header changes update executable provenance") + + +if __name__ == "__main__": + main() From bcb2408e6ab9e544029c2c7aa5c9b3a8a58c4b1c Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:38:51 +0200 Subject: [PATCH 21/40] Add bounded buffer allocation diagnostics with zero disabled setup --- .../src/gpu_native/buffer_alloc_diagnostic.h | 46 ++++++++++++++++++ rexlego/src/gpu_native/buffers.cpp | 29 ++++++++++++ tools/gpu-aot/CMakeLists.txt | 2 +- tools/gpu-aot/test_buffer_alloc_diagnostic.py | 47 +++++++++++++++++++ tools/gpu-aot/test_buffer_variants.py | 14 ++++++ 5 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 rexlego/src/gpu_native/buffer_alloc_diagnostic.h create mode 100644 tools/gpu-aot/test_buffer_alloc_diagnostic.py diff --git a/rexlego/src/gpu_native/buffer_alloc_diagnostic.h b/rexlego/src/gpu_native/buffer_alloc_diagnostic.h new file mode 100644 index 0000000..e0e2fbe --- /dev/null +++ b/rexlego/src/gpu_native/buffer_alloc_diagnostic.h @@ -0,0 +1,46 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace legodimensions::gpu_native { +// Recording-lock serialized, metadata-only diagnostic. A trigger arms the +// following frame; allocation activity during startup consumes no record budget. +class BufferAllocationDiagnostic { + public: + static std::optional ParseGuest(std::string_view value) { + if (value.size() != 8) return std::nullopt; + uint32_t guest = 0; + const auto parsed = std::from_chars(value.data(), value.data() + value.size(), guest, 16); + if (parsed.ec != std::errc{} || parsed.ptr != value.data() + value.size()) return std::nullopt; + return guest; + } + BufferAllocationDiagnostic(std::optional guest = {}, + std::filesystem::path trigger = {}) : guest_(guest), trigger_(std::move(trigger)) {} + bool Select(uint32_t frame, uint32_t guest) { + if (!guest_ || trigger_.empty() || guest != *guest_ || records_ >= 128) return false; + if (!armed_) { + if (checked_ && checked_frame_ == frame) return false; + checked_ = true; + checked_frame_ = frame; + std::error_code error; + if (!std::filesystem::is_regular_file(trigger_, error) || error || + !std::filesystem::remove(trigger_, error) || error) return false; + armed_ = true; + armed_frame_ = frame; + } + if (frame == armed_frame_) return false; + ++records_; + return true; + } + private: + std::optional guest_; + std::filesystem::path trigger_; + bool checked_ = false, armed_ = false; + uint32_t checked_frame_ = 0, armed_frame_ = 0, records_ = 0; +}; +} // namespace legodimensions::gpu_native diff --git a/rexlego/src/gpu_native/buffers.cpp b/rexlego/src/gpu_native/buffers.cpp index 2b503fb..06a2be6 100644 --- a/rexlego/src/gpu_native/buffers.cpp +++ b/rexlego/src/gpu_native/buffers.cpp @@ -1,5 +1,7 @@ #include "gpu_native/buffers.h" #include "gpu_native/buffer_header.h" +#include "gpu_native/buffer_alloc_diagnostic.h" +#include "gpu_native/long_probe.h" #include "gpu_native/memory_watch.h" #include "gpu_native/vertex_upload.h" @@ -48,6 +50,7 @@ struct BufferResource { u64 window_cache_bytes = 0; u64 window_use_serial = 0; u64 last_window_use = 0; + u32 window_offset = 0; // Diagnostic origin within the parent buffer. }; std::mutex g_buffers_mutex; @@ -98,6 +101,26 @@ std::shared_ptr AdoptBuffer(u32 address, BufferKind kind) { return resource; } +void TraceBufferAllocation(const BufferResource& resource, const VertexByteOrder& order, + u64 old_hash, u64 new_hash, u64 conversion_hash, size_t previous_variants, + bool evicted) { + if (!LongProbeEnabled()) return; + static BufferAllocationDiagnostic diagnostic = [] { + const char* guest = std::getenv("LEGO_NATIVE_BUFFER_ALLOC_GUEST"); + const char* trigger = std::getenv("LEGO_NATIVE_BUFFER_ALLOC_TRIGGER"); + return BufferAllocationDiagnostic(guest ? BufferAllocationDiagnostic::ParseGuest(guest) : + std::nullopt, trigger ? trigger : ""); + }(); + if (!diagnostic.Select(g_probe_frame.load(), resource.guest_address)) return; + LongProbeEvent("buffer_allocation", false, "guest=", resource.guest_address, + "mirror=", resource.mirror_address, "kind=", u32(resource.kind), "bytes=", resource.length, + "window_offset=", resource.window_offset, "stride=", order.stride, + "phase=", order.stream_offset, "reversal_hash=", conversion_hash, + "old_content=", old_hash, "new_content=", new_hash, + "reason=", old_hash != new_hash ? "content_changed" : "variant_miss", + "previous_variants=", previous_variants, "evicted=", evicted); +} + void ByteSwapElements(void* destination, const void* source, size_t size, u32 element_size) { auto* dst = static_cast(destination); @@ -254,6 +277,8 @@ plume::RenderBuffer* ResolveBufferContents(const std::shared_ptr g_buffer_timing.hash_ms += std::chrono::duration( std::chrono::steady_clock::now() - hash_start).count(); } + const u64 previous_content_hash = resource->content_hash; + const size_t previous_variants = resource->variants.size(); if (resource->content_hash != content_hash) { for (auto& [key, version] : resource->variants) HostDevice::RetireResource(std::move(version)); resource->variants.clear(); @@ -317,6 +342,7 @@ plume::RenderBuffer* ResolveBufferContents(const std::shared_ptr } if (!converted.empty()) std::memcpy(mapped, converted.data(), converted.size()); next_buffer->unmap(); + const bool variant_evicted = resource->variants.size() >= 8; if (resource->variants.size() >= 8) { auto old = resource->variants.begin(); HostDevice::RetireResource(std::move(old->second)); @@ -327,6 +353,8 @@ plume::RenderBuffer* ResolveBufferContents(const std::shared_ptr resource->content_hash = content_hash; resource->byte_order_hash = byte_order_hash; if (!watch_hit || audit_watch) resource->cpu_stamp = std::move(next_stamp); + TraceBufferAllocation(*resource, conversion_order, previous_content_hash, + content_hash, byte_order_hash, previous_variants, variant_evicted); return resource->buffer.get(); } @@ -363,6 +391,7 @@ BufferResourceView ResolveBufferResourceWindow(u32 guest_address, BufferKind kin resource->format = parent->format; resource->kind = kind; resource->owns_guest_memory = false; + resource->window_offset = offset; // A shared mesh pool has far more than 64 draw ranges. Evicting begin() // repeatedly discarded newly inserted ranges and allocated GPU buffers // for them again on every frame. Bound bytes as well as entry count and diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index f3a4695..ddf4b07 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -61,7 +61,7 @@ endif() if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") find_package(Python3 COMPONENTS Interpreter QUIET) if(Python3_Interpreter_FOUND) - foreach(check IN ITEMS native_build_fingerprint command_slots constant_timing frame_cadence cull_state draw_diagnostic prelink_workers prelink_emission + foreach(check IN ITEMS buffer_alloc_diagnostic native_build_fingerprint command_slots constant_timing frame_cadence cull_state draw_diagnostic prelink_workers prelink_emission frame_ring_transaction submission_waits pipeline_failures completion_callbacks cpu_resource_wait draw_upload query_retirement) add_test(NAME ${check} diff --git a/tools/gpu-aot/test_buffer_alloc_diagnostic.py b/tools/gpu-aot/test_buffer_alloc_diagnostic.py new file mode 100644 index 0000000..4a78264 --- /dev/null +++ b/tools/gpu-aot/test_buffer_alloc_diagnostic.py @@ -0,0 +1,47 @@ +"""Check actual bounded buffer-allocation trigger state without GPU assets.""" +import argparse +from pathlib import Path +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('output', type=Path) +p.add_argument('--compiler', default='clang++') +a = p.parse_args() +root = Path(__file__).resolve().parents[2] +a.output.mkdir(parents=True, exist_ok=True) +source = a.output / 'buffer-alloc-diagnostic.cpp' +source.write_text(r''' +#include "gpu_native/buffer_alloc_diagnostic.h" +#include +#include +using namespace legodimensions::gpu_native; +int main(int argc,char**argv) { + assert(argc==2); + const auto path=std::filesystem::path(argv[1]); + std::filesystem::remove(path); + using D=BufferAllocationDiagnostic; + assert(D::ParseGuest("849a4070")==0x849A4070); + for(const auto bad:{"","849A407","849A40700","0x849A4070","849A407G"," 49A4070","-49A4070"}) + assert(!D::ParseGuest(bad)); + D disabled({},path),no_path(0x849A4070,{}),active(0x849A4070,path); + assert(!active.Select(10,0x849A4070)); + {std::ofstream file(path);file<<"arm";} + assert(!disabled.Select(11,0x849A4070)&&std::filesystem::exists(path)); + assert(!no_path.Select(11,0x849A4070)); + assert(!active.Select(10,0x849A4070)&&std::filesystem::exists(path)); // One check/frame. + assert(!active.Select(11,0x849A4090)&&std::filesystem::exists(path)); // Exact guest filter. + assert(!active.Select(11,0x849A4070)&&!std::filesystem::exists(path)); + assert(!active.Select(11,0x849A4070)); // Do not begin in a partial frame. + for(unsigned n=0;n<128;++n) assert(active.Select(12+n/4,0x849A4070)); + {std::ofstream file(path);} + for(unsigned n=0;n<1000;++n) assert(!active.Select(100+n,0x849A4070)); + assert(std::filesystem::exists(path)); // Whole-process cap cannot rearm. + std::filesystem::remove(path); +} +''') +exe = a.output / 'buffer-alloc-diagnostic' +subprocess.run([a.compiler, '-std=c++20', '-O2', '-I' + str(root / 'rexlego/src'), + str(source), '-o', str(exe)], check=True, timeout=45) +subprocess.run([str(exe.resolve()), str((a.output / 'allocation.trigger').resolve())], + check=True, timeout=10) +print('PASS exact guest filter, disabled state, one poll/frame, deferred arming, 128-record cap') diff --git a/tools/gpu-aot/test_buffer_variants.py b/tools/gpu-aot/test_buffer_variants.py index c735a1f..f3fa93d 100644 --- a/tools/gpu-aot/test_buffer_variants.py +++ b/tools/gpu-aot/test_buffer_variants.py @@ -86,6 +86,12 @@ def body(signature): return true; } '''+resource+r''' +struct AllocationRecord {u64 old_hash,new_hash,conversion;size_t previous;bool evicted;}; +std::vector allocation_records; +void TraceBufferAllocation(const BufferResource&,const VertexByteOrder&,u64 old_hash, + u64 new_hash,u64 conversion,size_t previous,bool evicted){ + allocation_records.push_back({old_hash,new_hash,conversion,previous,evicted}); +} std::shared_ptr resource; auto AdoptBuffer(u32,BufferKind)->std::shared_ptr{return resource;} struct BufferResourceView { @@ -123,17 +129,25 @@ def body(signature): auto* first=ResolveBufferResource(1,BufferKind::kVertex,order);assert(first); const auto original_bytes=first->data; assert(HostDevice::device.allocations==1); + assert(allocation_records.size()==1&&allocation_records.back().old_hash==0&& + allocation_records.back().new_hash!=0&&allocation_records.back().previous==0); order.stream_offset=160;assert(ResolveBufferResource(1,BufferKind::kVertex,order)==first); assert(HostDevice::device.allocations==1); + assert(allocation_records.size()==1); // A reused conversion is not an allocation. auto* plain=ResolveBufferResource(1,BufferKind::kVertex,{});assert(plain&&plain!=first); assert(ResolveBufferResource(1,BufferKind::kVertex,order)==first); assert(HostDevice::device.allocations==2); + assert(allocation_records.size()==2&&allocation_records.back().previous==1&& + allocation_records.back().old_hash==allocation_records.back().new_hash); memory.bytes[32+164]^=0xFF; // CPU update between draws in the same frame. auto* changed=ResolveBufferResource(1,BufferKind::kVertex,order);assert(changed&&changed!=first); assert(changed->data!=original_bytes&&first->data==original_bytes); assert(resource->variants.size()==1&&HostDevice::held.size()==2); + assert(allocation_records.size()==3&&allocation_records.back().previous==2&& + allocation_records.back().old_hash!=allocation_records.back().new_hash); for(u32 phase=0;phase<16;++phase){order.stream_offset=phase;assert(ResolveBufferResource(1,BufferKind::kVertex,order));} assert(resource->variants.size()==8); // Evicted versions survive their fence. + assert(allocation_records.back().evicted&&allocation_records.back().previous==8); assert(first->data==original_bytes); order.stream_offset=2048;assert(!ResolveBufferResource(1,BufferKind::kVertex,order)); resource->kind=BufferKind::kIndex;resource->variants.clear();resource->buffer.reset();resource->format=1; From 28d83370668cd953b1d474eed80ff7ae1be908b0 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:38:51 +0200 Subject: [PATCH 22/40] Use RenderDoc include interface for optional native capture --- rexlego/CMakeLists.txt | 2 +- rexlego/src/gpu_native/renderdoc_probe.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt index 578db24..3be54f3 100644 --- a/rexlego/CMakeLists.txt +++ b/rexlego/CMakeLists.txt @@ -118,7 +118,7 @@ if(LEGODIMENSIONS_NATIVE_GPU) target_include_directories(legodimensions PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/miniz" "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/zstd") - target_link_libraries(legodimensions PRIVATE plume d3d12 dxgi rex::xxhash) + target_link_libraries(legodimensions PRIVATE plume d3d12 dxgi rex::xxhash renderdoc) # Generated locally by tools/gpu-aot. Keeping it out of Git avoids a # multi-megabyte generated source file while still making an enabled build diff --git a/rexlego/src/gpu_native/renderdoc_probe.h b/rexlego/src/gpu_native/renderdoc_probe.h index 9b261e9..547d2b2 100644 --- a/rexlego/src/gpu_native/renderdoc_probe.h +++ b/rexlego/src/gpu_native/renderdoc_probe.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include namespace legodimensions::gpu_native { From 305e2341e4c918e63aed935a92185cd080caa4b1 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:40:52 +0200 Subject: [PATCH 23/40] Pin stop-safe timers and host codegen template fixes --- rexglue-sdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rexglue-sdk b/rexglue-sdk index 17e3675..0541b80 160000 --- a/rexglue-sdk +++ b/rexglue-sdk @@ -1 +1 @@ -Subproject commit 17e3675767067339581018c7e722e1d8d65de201 +Subproject commit 0541b80c9bbc3b96a4dd6f5783638eea19917e1b From 62b361f7f2de98863c1063f3d673fa8d3c60de4d Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:43:28 +0200 Subject: [PATCH 24/40] Test production cross-build codegen and shader producer rules --- tools/gpu-aot/CMakeLists.txt | 8 + tools/gpu-aot/test_cross_build_producers.py | 196 ++++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 tools/gpu-aot/test_cross_build_producers.py diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index ddf4b07..e645cad 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -70,6 +70,14 @@ if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") --compiler ${CMAKE_CXX_COMPILER}) set_tests_properties(${check} PROPERTIES TIMEOUT 60) endforeach() + # Verify the actual SDK project template when its submodule is present. + if(UNIX AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/resources/templates/init/rexglue_cmake.inja") + add_test(NAME cross_build_producers + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_cross_build_producers.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/cross_build_producers + --compiler ${CMAKE_CXX_COMPILER}) + set_tests_properties(cross_build_producers PROPERTIES TIMEOUT 60) + endif() # This fixture cross-compiles a tiny SDK-free Windows ABI cache and # inspects/archives a real COFF blob, rather than linking a host GPU app. if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") diff --git a/tools/gpu-aot/test_cross_build_producers.py b/tools/gpu-aot/test_cross_build_producers.py new file mode 100644 index 0000000..3cb2534 --- /dev/null +++ b/tools/gpu-aot/test_cross_build_producers.py @@ -0,0 +1,196 @@ +"""Configure and run production codegen/DXC/prelink rules with synthetic tools.""" +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + + +def rule(text, output): + start = text.index(' add_custom_command(\n OUTPUT "${' + output + '}"') + end = text.index("VERBATIM)", start) + len("VERBATIM)") + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--compiler", default="clang++") + parser.add_argument("--cmake", default="cmake") + parser.add_argument("--generator", default="Ninja" if os.name == "nt" else "Unix Makefiles") + args = parser.parse_args() + if os.name == "nt": + parser.error("This Linux-to-Windows cross-build fixture requires Unix executable scripts") + root = Path(__file__).resolve().parents[2] + native = (root / "rexlego/CMakeLists.txt").read_text() + sdk_template = root / "rexglue-sdk/resources/templates/init/rexglue_cmake.inja" + if not sdk_template.exists(): + parser.error("Initialize rexglue-sdk to verify its actual codegen template") + template = sdk_template.read_text() + codegen = template[template.index("# A cross-build can reuse"): + template.index("# Include DLL module shared library targets")] + codegen = re.sub(r'\{\{ cmake_var\("([^"\n]+)"\) \}\}', + lambda match: "${" + match[1] + "}", codegen) + codegen = codegen.replace("{{ names.snake_case }}", "legodimensions").replace( + "{{ entrypoint_out_dir }}", "generated/default") + assert "{{" not in codegen + shaders = native[native.index(" set(LEGO_NATIVE_HOST_SHADER_DIR"): + native.index(" add_executable(lego_gpu_microcode_index")] + main_prelink = rule(native, "LEGO_NATIVE_LINKED_SHADER_CACHE") + runtime_prelink = rule(native, "LEGO_NATIVE_RUNTIME_LINKED_SHADER_CACHE") + args.output.mkdir(parents=True, exist_ok=True) + output = Path(tempfile.mkdtemp(prefix="producers-", dir=args.output.resolve())) + source = output / "source with spaces" + source.mkdir() + shader_dir = source / "src/gpu_native/shaders" + shader_dir.mkdir(parents=True) + for shader in ("copy_vs", "copy_color_ps", "resolve_color_ps", "depth_pack_ps", "depth_restore_ps"): + (shader_dir / (shader + ".hlsl")).write_text("// synthetic shader\n") + (shader_dir / "copy_common.hlsli").write_text("// synthetic shared shader header\n") + (source / "legodimensions_manifest.toml").write_text("# synthetic manifest\n") + (source / "main.cpp").write_text("int main() {}\n") + (source / "producer.cpp").write_text(r''' +#include +#include +#include +int main(int argc,char** argv) { + if(argc<2)return 2; + std::filesystem::path output=argv[argc-1]; + bool codegen=std::string(argv[1])=="codegen"; + if(codegen) { + if(argc!=4||std::string(argv[2])!="--pointer_table_scan")return 3; + output=output.parent_path()/"generated/default/codegen.build.stamp"; + } + std::filesystem::create_directories(output.parent_path()); + std::ofstream(output)<<"// synthetic producer output\n"; + if(codegen) { + std::string path=output.generic_string(),escaped; + for(char ch:path){if(ch==' ')escaped+='\\';escaped+=ch;} + std::ofstream(output.parent_path()/"codegen.d")< Date: Wed, 7 Oct 2026 07:45:56 +0200 Subject: [PATCH 25/40] Keep GPU-first depth sampling channels consistent and retryable --- rexlego/src/gpu_native/textures.cpp | 31 ++----- tools/gpu-aot/CMakeLists.txt | 10 +++ tools/gpu-aot/test_texture_depth_upload.py | 96 +++++++++++++++++++++- 3 files changed, 109 insertions(+), 28 deletions(-) diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp index a339faf..7af2d26 100644 --- a/rexlego/src/gpu_native/textures.cpp +++ b/rexlego/src/gpu_native/textures.cpp @@ -756,7 +756,7 @@ bool EnsureDepthSamplingMirror(TextureResource& resource) { plume::RenderSwizzle::R, plume::RenderSwizzle::G, plume::RenderSwizzle::B, plume::RenderSwizzle::A, plume::RenderSwizzle::ZERO, plume::RenderSwizzle::ONE}; - const u32 swizzle = NativeTextureSwizzle(resource.guest_format, + const u32 swizzle = NativeTextureSwizzle(resource.guest_format & 0x3Fu, resource.guest_fetch.swizzle); view_desc.componentMapping = plume::RenderComponentMapping( swizzles[swizzle & 7], swizzles[(swizzle >> 3) & 7], @@ -1676,31 +1676,10 @@ bool ResolveTextureFromSurface(u32 destination_texture, u32 source_surface, (adopted_destination->guest_fetch.swizzle == 0xB48 || adopted_destination->guest_fetch.swizzle == 0x688)) { auto& dst = *adopted_destination; - if (!dst.sampled_texture) { - auto* device = HostDevice::Device(); - plume::RenderTextureDesc desc; - desc.dimension = plume::RenderTextureDimension::TEXTURE_2D; - desc.width = dst.width; desc.height = dst.height; - desc.depth = 1; desc.mipLevels = 1; desc.arraySize = 1; - desc.multisampling.sampleCount = plume::RenderSampleCount::COUNT_1; - desc.format = plume::RenderFormat::R32_FLOAT; - desc.flags |= plume::RenderTextureFlag::RENDER_TARGET; - desc.committed = true; - dst.sampled_texture = device->createTexture(desc); - if (!dst.sampled_texture || - !static_cast(dst.sampled_texture.get())->d3d) return false; - plume::RenderTextureViewDesc view; - view.dimension = plume::RenderTextureViewDimension::TEXTURE_2D; - view.format = plume::RenderFormat::R32_FLOAT; - view.mipLevels = 1; - if (dst.guest_fetch.swizzle == 0xB48) - view.componentMapping = plume::RenderComponentMapping(plume::RenderSwizzle::R, - plume::RenderSwizzle::G, plume::RenderSwizzle::ONE, plume::RenderSwizzle::ONE); - dst.sampled_view = dst.sampled_texture->createTextureView(view); - if (!dst.sampled_view) return false; - dst.sampled_descriptor_index = HostDevice::RegisterTexture( - dst.sampled_texture.get(), dst.sampled_view.get()); - } + // CPU-first and GPU-first sampling must compose the same Xenos depth + // channel expansion with the guest swizzle. Publish storage/view/descriptor + // together so an allocation failure can be retried safely. + if (!EnsureDepthSamplingMirror(dst)) return false; if (!HostDevice::ResolveHdrColor(source->texture.get(), source->descriptor_index, dst.sampled_texture.get(), dst.width, dst.height, 1.0f, ColorResolveDestination::kDepthFloat32, true, &*region)) return false; diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index e645cad..86cfe66 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -70,6 +70,16 @@ if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") --compiler ${CMAKE_CXX_COMPILER}) set_tests_properties(${check} PROPERTIES TIMEOUT 60) endforeach() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/shaders/bytecode/d3d12_5_1/texture_load_depth_unorm_cs.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/d3d12/texture_cache.cpp" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/pipeline/texture/util.cpp" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/pipeline/texture/cache.cpp") + add_test(NAME texture_depth_upload + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_texture_depth_upload.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/texture_depth_upload + --compiler ${CMAKE_CXX_COMPILER}) + set_tests_properties(texture_depth_upload PROPERTIES TIMEOUT 60) + endif() # Verify the actual SDK project template when its submodule is present. if(UNIX AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/resources/templates/init/rexglue_cmake.inja") add_test(NAME cross_build_producers diff --git a/tools/gpu-aot/test_texture_depth_upload.py b/tools/gpu-aot/test_texture_depth_upload.py index de466e4..3da5245 100644 --- a/tools/gpu-aot/test_texture_depth_upload.py +++ b/tools/gpu-aot/test_texture_depth_upload.py @@ -2,6 +2,7 @@ import argparse import hashlib import json +import os from pathlib import Path import subprocess @@ -20,6 +21,8 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('output', type=Path) parser.add_argument('--compiler', default='clang++') + parser.add_argument('--legacy-source', type=Path, + help='Require the previous actual GPU-first branch to fail the parity oracle') args = parser.parse_args() root = Path(__file__).resolve().parents[2] source_path = root / 'rexlego/src/gpu_native/textures.cpp' @@ -35,6 +38,16 @@ def main(): cursor = 0 for op in operations: cursor = sdk.index(op, cursor) + len(op) + sdk_table_path = root / 'rexglue-sdk/src/graphics/d3d12/texture_cache.cpp' + sdk_table = sdk_table_path.read_text() + depth_entry = sdk_table.split('// k_24_8\n', 1)[1].split('// k_24_8_FLOAT', 1)[0] + assert 'DXGI_FORMAT_R32_FLOAT' in depth_entry + assert 'xenos::XE_GPU_TEXTURE_SWIZZLE_RRRR' in depth_entry + sdk_swizzle_path = root / 'rexglue-sdk/src/graphics/pipeline/texture/cache.cpp' + sdk_swizzle = function(sdk_swizzle_path.read_text(), + 'uint32_t TextureCache::GuestToHostSwizzle(') + sdk_swizzle = sdk_swizzle.replace('TextureCache::GuestToHostSwizzle', 'SdkDepthSwizzle') + sdk_swizzle = sdk_swizzle.replace('xenos::XE_GPU_TEXTURE_SWIZZLE_0', '4u') bodies = '\n'.join(function(text, sig) for sig in [ 'bool EnsureDepthSamplingMirror(', 'bool UploadGuestDepthTexture(', 'TextureResourceView ResolveTextureResource(']) @@ -91,14 +104,17 @@ def main(): ''' harness += tiled + r''' } } } +''' + harness += sdk_swizzle + r''' namespace plume { enum class RenderFormat { UNKNOWN, R32_FLOAT, R32G32B32A32_FLOAT, D32_FLOAT_S8 }; enum class RenderTextureDimension { TEXTURE_2D }; enum class RenderTextureViewDimension { TEXTURE_2D }; enum class RenderSampleCount { COUNT_1 }; enum RenderTextureFlag { RENDER_TARGET=1 }; -enum class RenderSwizzle { R,G,B,A,ZERO,ONE }; -struct RenderComponentMapping { std::array lanes{}; +enum class RenderSwizzle { R,G,B,A,ZERO,ONE,IDENTITY }; +struct RenderComponentMapping { std::array lanes{ + RenderSwizzle::IDENTITY,RenderSwizzle::IDENTITY,RenderSwizzle::IDENTITY,RenderSwizzle::IDENTITY}; RenderComponentMapping()=default; RenderComponentMapping(RenderSwizzle r,RenderSwizzle g,RenderSwizzle b,RenderSwizzle a):lanes{r,g,b,a}{} }; struct RenderTextureDesc { RenderTextureDimension dimension{};u32 width{},height{},depth{},mipLevels{},arraySize{},flags{};bool committed{}; @@ -248,6 +264,55 @@ def main(): for(u32 y=0;y expected{}; + const auto sdk_swizzle=SdkDepthSwizzle(swizzle,0); // Actual table above proves RRRR. + for(u32 lane=0;lane<4;++lane)expected[lane]=plume::RenderSwizzle((sdk_swizzle>>(3*lane))&7); + for(bool gpu_first:{true,false}){ + auto resource=std::make_shared();textures[resource->guest_address]=resource; + resource->guest_fetch.swizzle=swizzle;Fill(*resource,2,true); + plume::RenderCommandList commands; + if(!gpu_first)assert(UploadTextureResource(resource->guest_address,&commands,false)); + assert(ResolveDepth(resource)); + assert(resource->sampled_view->desc.componentMapping.lanes==expected && "depth_mirror_swizzle_parity"); + const auto descriptor=resource->sampled_descriptor_index; + const auto registrations=HostDevice::registrations; + auto view=ResolveTextureResource(resource->guest_address); + assert(view.view==resource->sampled_view.get() && view.descriptor_index==descriptor && !view.depth); + readable=false;++generation;assert(UploadTextureResource(resource->guest_address,&commands,true));readable=true; + assert(ResolveDepth(resource)); + assert(resource->sampled_descriptor_index==descriptor && HostDevice::registrations==registrations); + } + } + // Native-created resources retain full D3DFORMAT flags; the atlas guard and + // SDK intrinsic expansion both use the low six-bit Xenos format ID. + for(u32 swizzle:{0xB48u,0x688u}){ + auto resource=std::make_shared();textures[resource->guest_address]=resource; + resource->guest_format=0x80000016u;resource->owns_guest_memory=true;resource->guest_fetch.swizzle=swizzle; + assert(ResolveDepth(resource)); + const auto sdk_swizzle=SdkDepthSwizzle(swizzle,0); + for(u32 lane=0;lane<4;++lane) + assert(resource->sampled_view->desc.componentMapping.lanes[lane]==plume::RenderSwizzle((sdk_swizzle>>(3*lane))&7) + && "full_depth_format_swizzle_parity"); + } + // GPU-first allocation must be atomic and retryable at every failure boundary. + // Failed conversion may retain a complete mirror but cannot grant authority. + for(int failure=0;failure<5;++failure){ + auto resource=std::make_shared();textures[resource->guest_address]=resource; + if(failure==0)device.fail_texture=true; + if(failure==1)device.fail_native=true; + if(failure==2)plume::fail_view=true; + if(failure==3)HostDevice::fail_descriptor=true; + if(failure==4)HostDevice::fail_resolve=true; + assert(!ResolveDepth(resource));assert(!resource->resolved_on_host && !resource->sampled_valid); + if(failure<4)assert(!resource->sampled_texture && !resource->sampled_view && resource->sampled_descriptor_index==~0u); + const auto descriptor=resource->sampled_descriptor_index;const auto registrations=HostDevice::registrations; + device.fail_texture=device.fail_native=plume::fail_view=HostDevice::fail_descriptor=HostDevice::fail_resolve=false; + assert(ResolveDepth(resource));assert(resource->resolved_on_host && resource->sampled_valid); + if(failure==4)assert(resource->sampled_descriptor_index==descriptor && HostDevice::registrations==registrations); + } // Actual non-square tiled rectangles, endian modes and stencil-independent normalized endpoints. for(bool tiled:{false,true})for(u32 endian=0;endian<4;++endian){ auto resource=std::make_shared();textures[resource->guest_address]=resource;Fill(*resource,endian,tiled); @@ -339,9 +404,36 @@ def main(): '-I', str(root / 'rexlego/src/gpu_native'), str(fixture), '-o', str(binary)], check=True, timeout=45) subprocess.run([str(binary)], check=True, timeout=10) + legacy_result = None + if args.legacy_source: + legacy_text = args.legacy_source.read_text() + legacy_begin = legacy_text.index(' // Xenos k_24_8 sampling') + legacy_end = legacy_text.index(' // Preserve both host depth', legacy_begin) + legacy_fixture = args.output.resolve() / 'legacy-gpu-first.cpp' + legacy_binary = args.output.resolve() / 'legacy-gpu-first.exe' + assert harness.count(resolve_branch) == 1 + legacy_fixture.write_text(harness.replace(resolve_branch, legacy_text[legacy_begin:legacy_end], 1)) + subprocess.run([args.compiler, '-std=c++20', '-UNDEBUG', + '-I', str(root / 'rexlego/src/gpu_native'), str(legacy_fixture), '-o', str(legacy_binary)], + check=True, timeout=45) + def disable_core(): + import resource + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + legacy = subprocess.run([str(legacy_binary)], capture_output=True, text=True, timeout=10, + preexec_fn=disable_core if os.name == 'posix' else None) + assert legacy.returncode != 0 and 'depth_mirror_swizzle_parity' in legacy.stderr, legacy.stderr + legacy_result = {'source_sha256': hashlib.sha256(args.legacy_source.read_bytes()).hexdigest(), + 'returncode': legacy.returncode, 'expected_assertion': 'depth_mirror_swizzle_parity'} (args.output / 'verification.json').write_text(json.dumps({ 'production_source_sha256': hashlib.sha256(source_path.read_bytes()).hexdigest(), 'sdk_depth_loader_sha256': hashlib.sha256(sdk_path.read_bytes()).hexdigest(), + 'sdk_depth_table_sha256': hashlib.sha256(sdk_table_path.read_bytes()).hexdigest(), + 'sdk_swizzle_source_sha256': hashlib.sha256(sdk_swizzle_path.read_bytes()).hexdigest(), + 'depth_mirror_orders': ['CPU upload then GPU resolve', 'GPU resolve before CPU upload'], + 'depth_mirror_swizzles': ['B48', '688'], + 'depth_mirror_full_d3d_format_gpu_first': '80000016', + 'gpu_first_failure_retry_cases': 5, + 'legacy_negative': legacy_result, 'decoder_sha256': hashlib.sha256((root / 'rexlego/src/gpu_native/texture_depth_upload.h').read_bytes()).hexdigest(), 'actual_bodies': ['UploadGuestDepthTexture', 'EnsureDepthSamplingMirror', 'ResolveTextureResource', 'UploadTextureResource entry guards', From b7832f899b289c986abf6921b6c5c45a9956e49a Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:47:26 +0200 Subject: [PATCH 26/40] Batch immutable draw state reads and count binding requests --- rexlego/src/gpu_native/device.cpp | 18 ++ rexlego/src/gpu_native/device.h | 15 ++ rexlego/src/gpu_native/draw.cpp | 54 ++++- .../src/gpu_native/state_call_diagnostic.h | 128 +++++++++++ tools/gpu-aot/CMakeLists.txt | 2 +- tools/gpu-aot/test_draw_device_view.py | 204 ++++++++++++++++++ tools/gpu-aot/test_frame_ring_transaction.py | 1 + tools/gpu-aot/test_gpu_timestamps.py | 1 + tools/gpu-aot/test_state_call_diagnostic.py | 187 ++++++++++++++++ tools/gpu-aot/test_submission_waits.py | 1 + 10 files changed, 603 insertions(+), 8 deletions(-) create mode 100644 rexlego/src/gpu_native/state_call_diagnostic.h create mode 100644 tools/gpu-aot/test_draw_device_view.py create mode 100644 tools/gpu-aot/test_state_call_diagnostic.py diff --git a/rexlego/src/gpu_native/device.cpp b/rexlego/src/gpu_native/device.cpp index 5eff88b..7db495f 100644 --- a/rexlego/src/gpu_native/device.cpp +++ b/rexlego/src/gpu_native/device.cpp @@ -231,6 +231,9 @@ constexpr u32 kFirstTextureSlot = 3; std::mutex g_mutex; std::unique_ptr g_state; +// Accessed under g_mutex. Do not reset on device recreation: a command-list +// address may be reused, but its recording identity must never be reused. +u64 g_recording_serial = 0; std::unique_ptr CreateInterface(Backend backend) { switch (backend) { @@ -594,6 +597,9 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) { if (state.command_list_open) { return state.command_lists[state.frame_slot].get(); } + // Reserve zero for a closed/invalid view and fail before any ring mutation + // if the process-wide serial can no longer advance. + if (g_recording_serial == UINT64_MAX) return nullptr; if ((state.framebuffers.empty() || state.swap_chain->needsResize()) && !RebuildSwapChain(state)) { return nullptr; @@ -629,6 +635,7 @@ plume::RenderCommandList* BeginFrameCommandsLocked(State& state) { if (state.gpu_timestamps) state.gpu_timestamps->Begin(commands, slot, u64(state.present_number) + 1); InitializeNullTextures(state, commands); + ++g_recording_serial; state.command_list_open = true; return commands; } @@ -758,6 +765,17 @@ bool HostDevice::IsReady() { return g_state && g_state->device && g_state->queue && g_state->swap_chain; } +DrawDeviceView HostDevice::CurrentDrawDeviceView() { + std::lock_guard lock(g_mutex); + if (!g_state) return {}; + const auto& state = *g_state; + return {state.device.get(), state.pipeline_layout.get(), + state.texture_descriptors.get(), state.sampler_descriptors.get(), + state.null_vertex_buffer.get(), + state.command_list_open ? state.command_lists[state.frame_slot].get() : nullptr, + state.command_list_open ? g_recording_serial : 0}; +} + plume::RenderDevice* HostDevice::Device() { std::lock_guard lock(g_mutex); return g_state ? g_state->device.get() : nullptr; diff --git a/rexlego/src/gpu_native/device.h b/rexlego/src/gpu_native/device.h index 192553b..9533ac8 100644 --- a/rexlego/src/gpu_native/device.h +++ b/rexlego/src/gpu_native/device.h @@ -49,6 +49,18 @@ struct DrawUploadSlice { explicit operator bool() const { return buffer && mapped; } }; +// Immutable device objects plus the identity of the currently open recording. +// The caller must hold LockRecording throughout every use of these pointers. +struct DrawDeviceView { + plume::RenderDevice* device = nullptr; + plume::RenderPipelineLayout* pipeline_layout = nullptr; + plume::RenderDescriptorSet* texture_descriptors = nullptr; + plume::RenderDescriptorSet* sampler_descriptors = nullptr; + plume::RenderBuffer* null_vertex_buffer = nullptr; + plume::RenderCommandList* commands = nullptr; + u64 recording_serial = 0; +}; + class HostDevice { public: // Hold across the whole CPU recording operation, not only list acquisition. @@ -57,6 +69,9 @@ class HostDevice { static bool Create(rex::ui::Window* window, Backend backend = Backend::kD3D12); static void Shutdown(); static bool IsReady(); + // One state-lock acquisition replaces repeated immutable draw getters. + // Closed recordings return commands=nullptr and recording_serial=0. + static DrawDeviceView CurrentDrawDeviceView(); static plume::RenderDevice* Device(); static plume::RenderCommandQueue* Queue(); diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp index 03ead98..8a94e96 100644 --- a/rexlego/src/gpu_native/draw.cpp +++ b/rexlego/src/gpu_native/draw.cpp @@ -48,6 +48,7 @@ #include "gpu_native/long_probe.h" #include "gpu_native/renderdoc_probe.h" #include "gpu_native/vertex_declarations.h" +#include "gpu_native/state_call_diagnostic.h" REXCVAR_DEFINE_BOOL(gpu_native_buffer_windows, false, "GPU", "Upload and retain only the vertex buffer ranges fetched by native draws") @@ -251,6 +252,40 @@ plume::RenderRect CullDrawScissor(plume::RenderRect scissor, return scissor; } +StateCallCapture* NativeStateCalls() { + static std::unique_ptr capture = [] { + const char* output = std::getenv("LEGO_NATIVE_STATE_CALLS"); + if (!output || !*output) return std::unique_ptr{}; + const char* trigger = std::getenv("LEGO_NATIVE_STATE_CALLS_TRIGGER"); + return std::make_unique(output, trigger ? trigger : ""); + }(); + return capture.get(); +} + +void BindNativeDrawLayout(plume::RenderCommandList* commands, + const DrawDeviceView& view, StateCallLedger* diagnostic) { + if (diagnostic) diagnostic->Layout(view.pipeline_layout); + commands->setGraphicsPipelineLayout(view.pipeline_layout); +} + +void BindNativeDrawDescriptorSets(plume::RenderCommandList* commands, + const DrawDeviceView& view, StateCallLedger* diagnostic) { + if (diagnostic) { + for (u32 i = 0; i < 3; ++i) diagnostic->Descriptor(view.texture_descriptors, i); + diagnostic->Descriptor(view.sampler_descriptors, 3); + } + commands->setGraphicsDescriptorSet(view.texture_descriptors, 0); + commands->setGraphicsDescriptorSet(view.texture_descriptors, 1); + commands->setGraphicsDescriptorSet(view.texture_descriptors, 2); + commands->setGraphicsDescriptorSet(view.sampler_descriptors, 3); +} + +void BindNativeDrawPipeline(plume::RenderCommandList* commands, + plume::RenderPipeline* pipeline, StateCallLedger* diagnostic) { + if (diagnostic) diagnostic->Pipeline(pipeline); + commands->setPipeline(pipeline); +} + bool BindConstants( plume::RenderCommandList* commands, const D3DDevice* device, const DrawBindings& bindings, const VertexDeclarationView& declaration, bool capture_constants) { @@ -960,6 +995,13 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, if (!commands) { return false; } + const auto draw_device = HostDevice::CurrentDrawDeviceView(); + if (draw_device.commands != commands || !draw_device.recording_serial || + !draw_device.pipeline_layout || !draw_device.texture_descriptors || + !draw_device.sampler_descriptors || !draw_device.null_vertex_buffer) return false; + StateCallLedger* state_calls = nullptr; + if (auto* capture = NativeStateCalls()) + state_calls = capture->BeginDraw(commands, draw_device.recording_serial); timing.Next(); std::array texture_barriers; @@ -1095,14 +1137,11 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, { ConstantTimer constant_timing(ConstantCost::kBindings); - commands->setGraphicsPipelineLayout(HostDevice::PipelineLayout()); + BindNativeDrawLayout(commands, draw_device, state_calls); if (enable_stencil && (key.depth_control & 1u)) static_cast(commands)->d3d->OMSetStencilRef( state_bytes[NativeStencilWordOffset(key.raster_control, key.depth_control, polygonal) + 3]); - commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 0); - commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 1); - commands->setGraphicsDescriptorSet(HostDevice::TextureDescriptorSet(), 2); - commands->setGraphicsDescriptorSet(HostDevice::SamplerDescriptorSet(), 3); + BindNativeDrawDescriptorSets(commands, draw_device, state_calls); } auto constants = BindConstants(commands, device, bindings, declaration, count<=6); if (!constants) { @@ -1116,7 +1155,7 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, commands->setFramebuffer(framebuffer); } timing.Next(); - commands->setPipeline(pipeline); + BindNativeDrawPipeline(commands, pipeline, state_calls); // 83FB82F0 writes normalized R,G,B,A blend constants at +10464..10476. const auto* guest_blend_factor = reinterpret_cast( reinterpret_cast(device) + 10464); @@ -1146,7 +1185,7 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, std::array views; std::array slots; - auto* null_buffer = HostDevice::NullVertexBuffer(); + auto* null_buffer = draw_device.null_vertex_buffer; std::array vertex_buffers; static const bool window_uploads = [] { const char* value = std::getenv("LEGO_NATIVE_BUFFER_WINDOWS"); @@ -1624,6 +1663,7 @@ u32 SwapHook(D3DDevice* /*device*/, u32 front_buffer, u32 /*parameters*/) { HostDevice::PollCompletionCallbacks(); auto recording = HostDevice::LockRecording(); const auto texture = ResolveTextureResource(front_buffer); + if (auto* capture = NativeStateCalls()) capture->Present(g_probe_frame.load()); if (!HostDevice::PresentTexture(texture.texture, texture.descriptor_index) && g_unsupported_draw_logs++ < 20) { LogUnknownTexture(front_buffer, "present"); diff --git a/rexlego/src/gpu_native/state_call_diagnostic.h b/rexlego/src/gpu_native/state_call_diagnostic.h new file mode 100644 index 0000000..9e0f1ff --- /dev/null +++ b/rexlego/src/gpu_native/state_call_diagnostic.h @@ -0,0 +1,128 @@ +// Counts guest draw binding requests; equal requests are not a D3D skip proof. +#pragma once + +#include +#include +#include +#include +#include + +namespace legodimensions::gpu_native { + +struct StateCallCounts { + uint64_t draws = 0, dropped_draws = 0, recordings = 0; + uint64_t layouts = 0, equal_layouts = 0; + uint64_t pipelines = 0, equal_pipelines = 0; + std::array descriptors{}, equal_descriptors{}; +}; + +class StateCallLedger { + public: + static constexpr uint32_t kMaxDraws = 4096; + bool BeginDraw(const void* commands, uint64_t serial) { + if (counts_.draws >= kMaxDraws) { + ++counts_.dropped_draws; + return false; + } + ++counts_.draws; + if (commands != commands_ || serial != serial_) { + ++counts_.recordings; + commands_ = commands; + serial_ = serial; + layout_ = pipeline_ = nullptr; + descriptors_.fill(nullptr); + } + return true; + } + void Layout(const void* layout) { + ++counts_.layouts; + if (layout_ && layout_ == layout) ++counts_.equal_layouts; + // A changed root signature invalidates its table comparisons. + if (layout_ != layout) descriptors_.fill(nullptr); + layout_ = layout; + } + void Descriptor(const void* descriptor, uint32_t slot) { + if (slot >= descriptors_.size()) return; + ++counts_.descriptors[slot]; + if (descriptors_[slot] && descriptors_[slot] == descriptor) + ++counts_.equal_descriptors[slot]; + descriptors_[slot] = descriptor; + } + void Pipeline(const void* pipeline) { + ++counts_.pipelines; + if (pipeline_ && pipeline_ == pipeline) ++counts_.equal_pipelines; + pipeline_ = pipeline; + } + const StateCallCounts& Counts() const { return counts_; } + void Reset() { *this = {}; } + + private: + StateCallCounts counts_; + const void* commands_ = nullptr; + uint64_t serial_ = 0; + const void* layout_ = nullptr; + const void* pipeline_ = nullptr; + std::array descriptors_{}; +}; + +// One finite capture per process. Optional trigger arms the next present +// interval; polling and file output occur only at the guest present hook. +class StateCallCapture { + public: + static constexpr uint32_t kMaxFrames = 120; + StateCallCapture(std::filesystem::path output, std::filesystem::path trigger) + : output_(std::move(output)), trigger_(std::move(trigger)), + armed_(!output_.empty() && trigger_.empty()) {} + StateCallLedger* BeginDraw(const void* commands, uint64_t serial) { + if (!armed_ || !commands || !serial || exhausted_) return nullptr; + return ledger_.BeginDraw(commands, serial) ? &ledger_ : nullptr; + } + void Present(uint32_t frame) { + if (output_.empty() || exhausted_) return; + if (!armed_) { + std::error_code error; + if (!std::filesystem::is_regular_file(trigger_, error) || error) return; + if (!std::filesystem::remove(trigger_, error) || error) return; + armed_ = true; + return; + } + if (!file_.is_open()) { + // A diagnostic cannot overwrite existing evidence. + std::error_code error; + if (std::filesystem::exists(output_, error) || error) { + exhausted_ = true; + return; + } + file_.open(output_, std::ios::out); + if (!file_) { exhausted_ = true; return; } + file_ << "# scope=guest_draw_requests;equal=adjacent_guest_request;" + "unobserved_helper_compute_ray_or_external_calls_not_measured;" + "max_frames=120;max_draws_per_frame=4096\n" + "frame,draws,dropped_draws,recordings,layout_calls,layout_equal," + "pso_calls,pso_equal,table0_calls,table0_equal,table1_calls,table1_equal," + "table2_calls,table2_equal,table3_calls,table3_equal\n"; + } + const auto& count = ledger_.Counts(); + file_ << frame << ',' << count.draws << ',' << count.dropped_draws << ',' + << count.recordings << ',' << count.layouts << ',' << count.equal_layouts + << ',' << count.pipelines << ',' << count.equal_pipelines; + for (size_t i = 0; i < count.descriptors.size(); ++i) + file_ << ',' << count.descriptors[i] << ',' << count.equal_descriptors[i]; + file_ << '\n'; + file_.flush(); + ledger_.Reset(); + if (!file_ || ++frames_ >= kMaxFrames) { + exhausted_ = true; + file_.close(); + } + } + + private: + std::filesystem::path output_, trigger_; + StateCallLedger ledger_; + std::ofstream file_; + uint32_t frames_ = 0; + bool armed_ = false, exhausted_ = false; +}; + +} // namespace legodimensions::gpu_native diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index 86cfe66..58fe845 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -61,7 +61,7 @@ endif() if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") find_package(Python3 COMPONENTS Interpreter QUIET) if(Python3_Interpreter_FOUND) - foreach(check IN ITEMS buffer_alloc_diagnostic native_build_fingerprint command_slots constant_timing frame_cadence cull_state draw_diagnostic prelink_workers prelink_emission + foreach(check IN ITEMS draw_device_view state_call_diagnostic buffer_alloc_diagnostic native_build_fingerprint command_slots constant_timing frame_cadence cull_state draw_diagnostic prelink_workers prelink_emission frame_ring_transaction submission_waits pipeline_failures completion_callbacks cpu_resource_wait draw_upload query_retirement) add_test(NAME ${check} diff --git a/tools/gpu-aot/test_draw_device_view.py b/tools/gpu-aot/test_draw_device_view.py new file mode 100644 index 0000000..e164c1c --- /dev/null +++ b/tools/gpu-aot/test_draw_device_view.py @@ -0,0 +1,204 @@ +"""Exercise production draw snapshots, recording identity, and lifecycle locks.""" +import argparse +import ast +from pathlib import Path +import subprocess + + +def function(source, signature): + start = source.index(signature) + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--compiler', default='clang++') + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + production = (root / 'rexlego/src/gpu_native/device.cpp').read_text() + header = (root / 'rexlego/src/gpu_native/device.h').read_text() + # Reuse the established fake driver, while compiling actual call-site + # bodies below; this fixture does not reimplement snapshot/serial logic. + existing = ast.parse((root / 'tools/gpu-aot/test_frame_ring_transaction.py').read_text()) + harness = next(ast.literal_eval(n.value) for n in ast.walk(existing) + if isinstance(n, ast.Assign) and any(isinstance(t, ast.Name) and t.id == 'harness' for t in n.targets)) + harness = harness.replace('#include ', '#include \n#include \n#include \n#include \n#include \n#include ') + harness = harness.replace('namespace plume {', ''' +namespace plume { +enum class RenderCommandListType {DIRECT}; +enum class RenderFormat {B8G8R8A8_UNORM}; +struct RenderSwapChainDesc {RenderSwapChainDesc(void*,RenderFormat,u32){}}; +struct Queue;struct Swap; +struct RenderPipelineLayout {};struct RenderDescriptorSet {};struct RenderBuffer {}; +''', 1) + harness = harness.replace('struct Device {', '''struct Device { + std::unique_ptr createCommandQueue(RenderCommandListType); + int getDescription(){return 0;} +''', 1) + harness = harness.replace(' int waits=0;', ''' int waits=0,submissions=0; + std::unique_ptr createSwapChain(RenderSwapChainDesc); + void executeCommandLists(RenderCommandList* commands,RenderCommandFence*) {assert(commands);++submissions;} +''', 1) + harness = harness.replace('void begin(){++begins;}', 'void begin(){++begins;}void end(){}', 1) + harness = harness.replace('// Disabled optional timestamp boundary;', ''' +namespace plume { +using RenderDevice=Device; +struct RenderInterface {auto createDevice(){return std::make_unique();}}; +std::unique_ptr Device::createCommandQueue(RenderCommandListType){return std::make_unique();} +std::unique_ptr Queue::createSwapChain(RenderSwapChainDesc){return std::make_unique();} +} +namespace rex::ui {struct Window {void* GetNativeWindowHandle(){return this;}};} +using HWND=void*;using HRESULT=int; +struct ID3D12Debug {void EnableDebugLayer(){}void Release(){}}; +#define IID_PPV_ARGS(x) x +#define SUCCEEDED(x) ((x)>=0) +int D3D12GetDebugInterface(ID3D12Debug**){return -1;} +const char* NameOf(Backend){return "fake";}void InitializeLogoCapture(){} +constexpr const char* kNativeGpuBuildFingerprint="fixture"; +bool CpuMemoryWatchEnabled(){return false;} +// Disabled optional timestamp boundary;''', 1) + harness = harness.replace('struct GpuSubmissionTimestamps {', '''struct GpuSubmissionTimestamps { + static std::unique_ptr Create(auto*,auto*,u32,const char*,const char*){return {};} + int Frequency(){return 1;}void Shutdown(u32){} +''', 1) + harness = harness.replace('GpuSubmissionTimestamps* gpu_timestamps=nullptr;', 'std::unique_ptr gpu_timestamps;', 1) + harness = harness.replace('struct State {', ''' +enum class SyncReason {kOther,kIdle,kFence,kResource,kCallback,kQueryBegin,kQueryRelease,kCount}; +struct State { + std::unique_ptr render_interface; + std::unique_ptr idle_fence; + std::unique_ptr pipeline_layout; + std::unique_ptr texture_descriptors,sampler_descriptors; + std::unique_ptr null_vertex_buffer; + struct CallbackQueue {size_t size(){return 0;}} completion_callbacks; + u64 last_submission=0,sync_calls=0;double sync_ms=0; + std::array(SyncReason::kCount)> sync_reason_calls{}; + std::array(SyncReason::kCount)> sync_reason_ms{}; +''', 1) + harness += '\n' + function(header, 'struct DrawDeviceView {') + ';\n' + harness += r''' +struct CountingMutex { + std::mutex mutex;u64 acquisitions=0; + void lock(){mutex.lock();++acquisitions;}void unlock(){mutex.unlock();} +}; +CountingMutex g_mutex; +std::unique_ptr g_state; +struct HostDevice { + static std::unique_lock LockRecording(); + static DrawDeviceView CurrentDrawDeviceView(); + static bool Create(rex::ui::Window*,Backend); + static void Shutdown(); + static bool Synchronize(SyncReason); + static bool SubmitRecordedWork(); +}; +void ResetDrawResources(){}void ResetQueryResources(){}void ResetShaderResources(){} +void ResetBufferResources(){}void ResetTextureResources(){}void ResetDrawBindings(){} +void ResetVertexDeclarations(){}void ShutdownCpuMemoryWatch(){} +auto CreateInterface(Backend backend){return backend==Backend::kD3D12? + std::make_unique():std::unique_ptr{};} +bool CreatePipelineLayout(State& state){ + state.pipeline_layout=std::make_unique(); + state.texture_descriptors=std::make_unique(); + state.sampler_descriptors=std::make_unique();return true; +} +bool CreatePresentPipeline(State&){return true;} +bool CreateNullVertexBuffer(State& state){state.null_vertex_buffer=std::make_unique();return true;} +bool CreateNullTextures(State&){return true;} +''' + bodies = '\n'.join(function(production, signature) for signature in ( + 'bool CreateFrameRing(', 'void RefreshCompletedSubmissionsLocked(', + 'void MarkSubmissionLocked(', 'bool WaitForSubmissionLocked(', + 'bool WaitForSubmittedFramesLocked(', 'bool RebuildSwapChain(', + 'plume::RenderCommandList* BeginFrameCommandsLocked(', + 'std::unique_lock HostDevice::LockRecording(', + 'DrawDeviceView HostDevice::CurrentDrawDeviceView(', + 'bool HostDevice::Create(', 'void HostDevice::Shutdown(', + 'bool HostDevice::Synchronize(', 'bool HostDevice::SubmitRecordedWork(')) + # Actual acquire-failure close/submit transition, including frame advance. + present = function(production, 'bool HostDevice::PresentTexture(') + start = present.index(' if (!acquired || image_index >= state.framebuffers.size()) {') + close = function(present[start:], ' if (!acquired || image_index >= state.framebuffers.size()) {') + bodies += '\nbool AcquireFailed(State& state){bool acquired=false;u32 image_index=0,slot=state.frame_slot;auto* commands=state.command_lists[slot].get();\n' + close + '\nreturn true;}\n' + tests = r''' +int main(){ + auto recording=HostDevice::LockRecording(); + assert(!HostDevice::CurrentDrawDeviceView().commands); + rex::ui::Window window; + assert(!HostDevice::Create(nullptr,Backend::kD3D12));assert(g_recording_serial==0); + assert(!HostDevice::Create(&window,Backend::kVulkan));assert(g_recording_serial==0); + assert(HostDevice::Create(&window,Backend::kD3D12)); + auto closed=HostDevice::CurrentDrawDeviceView(); + assert(closed.device&&closed.pipeline_layout&&closed.texture_descriptors&&closed.sampler_descriptors&&closed.null_vertex_buffer); + assert(!closed.commands&&!closed.recording_serial); + auto& state=*g_state; + auto* commands=BeginFrameCommandsLocked(state); + const auto locks_before=g_mutex.acquisitions; + auto first=HostDevice::CurrentDrawDeviceView(); + assert(g_mutex.acquisitions==locks_before+1); + assert(commands&&first.commands==commands&&first.recording_serial==1); + assert(first.device==state.device.get()&&first.pipeline_layout==state.pipeline_layout.get()); + assert(first.texture_descriptors==state.texture_descriptors.get()&&first.sampler_descriptors==state.sampler_descriptors.get()); + assert(first.null_vertex_buffer==state.null_vertex_buffer.get()); + for(int i=0;i<100;++i){assert(BeginFrameCommandsLocked(state)==commands);assert(HostDevice::CurrentDrawDeviceView().recording_serial==1);} + // Helper/present counters are not recording identities. + ++state.present_number;assert(HostDevice::CurrentDrawDeviceView().recording_serial==1); + assert(HostDevice::Synchronize(SyncReason::kFence)); + assert(!HostDevice::CurrentDrawDeviceView().commands&&!HostDevice::CurrentDrawDeviceView().recording_serial); + assert(BeginFrameCommandsLocked(state)==commands);assert(HostDevice::CurrentDrawDeviceView().recording_serial==2); + assert(HostDevice::SubmitRecordedWork());assert(!HostDevice::CurrentDrawDeviceView().recording_serial); + assert(BeginFrameCommandsLocked(state));assert(HostDevice::CurrentDrawDeviceView().recording_serial==3); + assert(!AcquireFailed(state));assert(!HostDevice::CurrentDrawDeviceView().recording_serial); + assert(BeginFrameCommandsLocked(state));assert(HostDevice::CurrentDrawDeviceView().recording_serial==4); + // Failed wait cannot begin, reclaim or advance the recording sequence. + assert(HostDevice::SubmitRecordedWork()); + state.frame_submitted[state.frame_slot]=true; + state.frame_fences[state.frame_slot]->native.completed=UINT64_MAX; + const auto serial=g_recording_serial;auto* previous=state.command_lists[state.frame_slot].get();const auto previous_begins=previous->begins; + assert(!BeginFrameCommandsLocked(state));assert(g_recording_serial==serial&&!state.command_list_open&&previous->begins==previous_begins); + state.frame_fences[state.frame_slot]->native.completed=0; + assert(BeginFrameCommandsLocked(state));assert(g_recording_serial==serial+1); + // Production Create fast path and Shutdown must both respect the retained + // recording guard, including the snapshot's raw pointer lifetime. + std::latch create_started(1); + auto create=std::async(std::launch::async,[&]{create_started.count_down();return HostDevice::Create(&window,Backend::kD3D12);}); + create_started.wait();assert(create.wait_for(std::chrono::milliseconds(30))==std::future_status::timeout); + recording.unlock();assert(create.wait_for(std::chrono::seconds(3))==std::future_status::ready&&create.get()); + recording=HostDevice::LockRecording(); + auto live=HostDevice::CurrentDrawDeviceView();std::latch shutdown_started(1); + auto shutdown=std::async(std::launch::async,[&]{shutdown_started.count_down();HostDevice::Shutdown();}); + shutdown_started.wait();assert(shutdown.wait_for(std::chrono::milliseconds(30))==std::future_status::timeout); + assert(HostDevice::CurrentDrawDeviceView().device==live.device&&g_state); + recording.unlock();assert(shutdown.wait_for(std::chrono::seconds(3))==std::future_status::ready);shutdown.get(); + recording=HostDevice::LockRecording();assert(!HostDevice::CurrentDrawDeviceView().device); + const auto before_recreate=g_recording_serial; + assert(HostDevice::Create(&window,Backend::kD3D12));assert(BeginFrameCommandsLocked(*g_state)); + assert(HostDevice::CurrentDrawDeviceView().recording_serial==before_recreate+1); + HostDevice::Shutdown(); + // Exhausted serial refuses the first recording before allocator/resource mutation. + assert(HostDevice::Create(&window,Backend::kD3D12));g_recording_serial=UINT64_MAX; + auto& overflow=*g_state;const auto begins=overflow.command_lists[overflow.frame_slot]->begins; + overflow.draw_upload_offset=512; + assert(!BeginFrameCommandsLocked(overflow)); + assert(overflow.command_lists[overflow.frame_slot]->begins==begins&&overflow.draw_upload_offset==512&&!overflow.command_list_open); + HostDevice::Shutdown();assert(Tracked::live==0); + std::cout<<"PASS: production snapshot identity, same-open/rebegin/submit/sync/acquire failure/recreation/overflow, retained recording excludes Create/Shutdown.\n"; +} +''' + args.output.mkdir(parents=True, exist_ok=True) + source = args.output.resolve() / 'draw-device-view.cpp' + binary = args.output.resolve() / 'draw-device-view' + source.write_text(harness + bodies + tests) + subprocess.run([args.compiler, '-std=c++20', '-UNDEBUG', '-pthread', + '-I' + str(root / 'rexlego/src'), str(source), '-o', str(binary)], + check=True, timeout=45) + subprocess.run([str(binary)], check=True, timeout=15) + + +if __name__ == '__main__': + main() diff --git a/tools/gpu-aot/test_frame_ring_transaction.py b/tools/gpu-aot/test_frame_ring_transaction.py index 3f46781..ade0551 100644 --- a/tools/gpu-aot/test_frame_ring_transaction.py +++ b/tools/gpu-aot/test_frame_ring_transaction.py @@ -37,6 +37,7 @@ def main(): #include #include "gpu_native/descriptor_retirement.h" using u32=std::uint32_t;using u64=std::uint64_t; +u64 g_recording_serial=0; using legodimensions::gpu_native::DescriptorRetirement; #define REXLOG_INFO(...) ((void)0) #define REXLOG_ERROR(...) ((void)0) diff --git a/tools/gpu-aot/test_gpu_timestamps.py b/tools/gpu-aot/test_gpu_timestamps.py index e0ab52d..df995a9 100644 --- a/tools/gpu-aot/test_gpu_timestamps.py +++ b/tools/gpu-aot/test_gpu_timestamps.py @@ -74,6 +74,7 @@ def main(): #include #include using u32=uint32_t;using u64=uint64_t;using UINT64=unsigned long long;using HRESULT=long; +u64 g_recording_serial=0; #define FAILED(x) ((x)<0) #define SUCCEEDED(x) ((x)>=0) #define REXLOG_ERROR(...) ((void)0) diff --git a/tools/gpu-aot/test_state_call_diagnostic.py b/tools/gpu-aot/test_state_call_diagnostic.py new file mode 100644 index 0000000..f35cfe5 --- /dev/null +++ b/tools/gpu-aot/test_state_call_diagnostic.py @@ -0,0 +1,187 @@ +"""Exercise actual guest state-request counters and unchanged API emissions.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + + +def function(source, signature): + begin = source.index(signature) + end = source.index('{', begin) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[begin:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--compiler', default='clang++') + parser.add_argument('--negative-control', action='store_true', + help='Deliberately ignore recording serial; oracle must fail') + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + root = Path(__file__).resolve().parents[2] + source = (root/'rexlego/src/gpu_native/draw.cpp').read_text() + bodies = '\n'.join(function(source, name) for name in ( + 'StateCallCapture* NativeStateCalls(', 'void BindNativeDrawLayout(', + 'void BindNativeDrawDescriptorSets(', 'void BindNativeDrawPipeline(')) + dispatch = function(source, 'bool DispatchDraw(') + assert dispatch.count('HostDevice::CurrentDrawDeviceView()') == 1 + assert 'draw_device.commands != commands' in dispatch + assert 'capture->BeginDraw(commands, draw_device.recording_serial)' in dispatch + assert 'auto* null_buffer = draw_device.null_vertex_buffer;' in dispatch + assert dispatch.index('BindNativeDrawLayout(') < dispatch.index('BindNativeDrawDescriptorSets(') + assert dispatch.index('BindNativeDrawDescriptorSets(') < dispatch.index('BindConstants(') + assert dispatch.index('BindConstants(') < dispatch.index('BindNativeDrawPipeline(') + assert 'capture->Present(g_probe_frame.load())' in function(source, 'u32 SwapHook(') + include = root/'rexlego/src' + if args.negative_control: + header = (include/'gpu_native/state_call_diagnostic.h').read_text() + old = 'commands != commands_ || serial != serial_' + assert header.count(old) == 1 + header = header.replace(old, 'commands != commands_') + include = args.output/'negative-include' + (include/'gpu_native').mkdir(parents=True, exist_ok=True) + (include/'gpu_native/state_call_diagnostic.h').write_text(header) + cpp = args.output/'state-call-diagnostic.cpp' + cpp.write_text(r''' +#include +#include +#include +#include +#include +#include +#include +#include "gpu_native/state_call_diagnostic.h" +using namespace legodimensions::gpu_native; +using u32=uint32_t; +void Check(bool value,const char* reason){if(!value){std::cerr< calls; + void setGraphicsPipelineLayout(RenderPipelineLayout*){calls.push_back("layout");} + void setGraphicsDescriptorSet(RenderDescriptorSet*,u32 slot){calls.push_back("table"+std::to_string(slot));} + void setPipeline(RenderPipeline*){calls.push_back("pipeline");} +}; +} +struct DrawDeviceView {plume::RenderPipelineLayout* pipeline_layout; + plume::RenderDescriptorSet* texture_descriptors;plume::RenderDescriptorSet* sampler_descriptors;}; +''' + bodies + r''' +int main(int argc,char** argv){ + if(argc==3){ + auto* enabled=NativeStateCalls();Check(enabled!=nullptr,"explicit opt-in accessor"); + plume::RenderCommandList own; + Check(enabled->BeginDraw(&own,1)!=nullptr,"opt-in captures valid recording"); + enabled->Present(1); + Check(std::filesystem::exists(argv[2]),"opt-in writes designated aggregate"); + return 0; + } + Check(argc==2,"expected own output directory"); + // The actual cached opt-in accessor performs no capture when unset/empty. + Check(NativeStateCalls()==nullptr,"disabled capture must be null"); + plume::RenderCommandList commands; + plume::RenderPipelineLayout layout,other_layout; + plume::RenderDescriptorSet texture,sampler,other_texture; + plume::RenderPipeline graphics,compute,ray; + DrawDeviceView view{&layout,&texture,&sampler}; + auto bind=[&](StateCallLedger* ledger){ + BindNativeDrawLayout(&commands,view,ledger); + BindNativeDrawDescriptorSets(&commands,view,ledger); + BindNativeDrawPipeline(&commands,&graphics,ledger); + }; + bind(nullptr);bind(nullptr); + Check(commands.calls.size()==12,"disabled original API count"); + const std::vector order{"layout","table0","table1","table2","table3","pipeline"}; + Check(std::equal(order.begin(),order.end(),commands.calls.begin()),"original binding order"); + StateCallLedger ledger; + Check(ledger.BeginDraw(&commands,1),"first recording");bind(&ledger); + Check(ledger.BeginDraw(&commands,1),"same recording");bind(&ledger); + Check(ledger.Counts().recordings==1&&ledger.Counts().equal_layouts==1&& + ledger.Counts().equal_pipelines==1,"same-recording equality"); + for(u32 i=0;i<4;++i)Check(ledger.Counts().descriptors[i]==2&& + ledger.Counts().equal_descriptors[i]==1,"independent table slot equality"); + // Same physical command pointer is reused after Reset; serial is decisive. + Check(ledger.BeginDraw(&commands,2),"reopened same command pointer");bind(&ledger); + Check(ledger.Counts().recordings==2&&ledger.Counts().equal_pipelines==1&& + ledger.Counts().equal_layouts==1,"recording_reset_rebind"); + for(u32 i=0;i<4;++i)Check(ledger.Counts().equal_descriptors[i]==1,"recording resets tables"); + // Observed layout changes invalidate the comparison for every root table. + view.pipeline_layout=&other_layout; + Check(ledger.BeginDraw(&commands,2),"layout change draw");bind(&ledger); + for(u32 i=0;i<4;++i)Check(ledger.Counts().equal_descriptors[i]==1,"changed layout resets tables"); + view.texture_descriptors=&other_texture; + Check(ledger.BeginDraw(&commands,2),"descriptor change draw");bind(&ledger); + for(u32 i=0;i<3;++i)Check(ledger.Counts().equal_descriptors[i]==1,"changed texture table"); + Check(ledger.Counts().equal_descriptors[3]==2,"independent sampler table"); + // The scope is guest requests. Unobserved helper/compute/ray binds must + // still be emitted, even though adjacent guest requests can compare equal. + const auto before=commands.calls.size(); + commands.setPipeline(&compute);commands.setPipeline(&ray); + Check(ledger.BeginDraw(&commands,2),"draw after unobserved pipeline");bind(&ledger); + Check(commands.calls.size()==before+8,"interleaved pipeline bindings preserved"); + const auto repeats=ledger.Counts().equal_pipelines; + Check(repeats>1,"scope explicitly counts adjacent guest requests only"); + ledger.Reset(); + for(u32 i=0;i #include "gpu_native/descriptor_retirement.h" using u32=std::uint32_t;using u64=std::uint64_t; +u64 g_recording_serial=0; using legodimensions::gpu_native::DescriptorRetirement; #define REXLOG_ERROR(...) ((void)0) #define REXLOG_WARN(...) ((void)0) From f40512bfc895492896567f2bc60e81065a2e7386 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:57:20 +0200 Subject: [PATCH 27/40] Register supported GPU checks and preserve optional dependency coverage --- tools/gpu-aot/CMakeLists.txt | 151 ++++---- tools/gpu-aot/test_draw_upload.py | 9 +- tools/gpu-aot/test_shader_binding_failures.py | 344 ++++++++++++++++++ 3 files changed, 433 insertions(+), 71 deletions(-) create mode 100644 tools/gpu-aot/test_shader_binding_failures.py diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index 58fe845..9f352d3 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -2,10 +2,13 @@ cmake_minimum_required(VERSION 3.25) project(NativeGpuPortableChecks LANGUAGES CXX) include(CTest) +if(BUILD_TESTING) + find_package(Python3 3.11 COMPONENTS Interpreter QUIET) +endif() + # The private-run harness uses Linux /proc, flock and reflinks. No game assets # are required by its synthetic preparation, sampling and comparison checks. -if(BUILD_TESTING AND UNIX) - find_package(Python3 3.11 COMPONENTS Interpreter QUIET) +if(BUILD_TESTING AND CMAKE_SYSTEM_NAME STREQUAL "Linux") if(Python3_Interpreter_FOUND) add_test(NAME renderer_regression COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/regression/test_regression.py) @@ -13,11 +16,10 @@ if(BUILD_TESTING AND UNIX) endif() endif() - # No SDK, generated game code, game assets, DXC, or graphics device required. set(CHECKS blend_state buffer_header color_resolve_swizzle descriptor_retirement pixel_shader_filter polygon_offset pool_copy portrait_identity present_rect - resolve_region shader_texture_usage texture_alpha4_upload texture_number + resolve_region shader_texture_usage texture_2d_upload texture_alpha4_upload texture_number texture_volume_upload tile_extent tonemap_dof vertex_byte_order vertex_upload window_mode) foreach(check IN LISTS CHECKS) @@ -33,61 +35,70 @@ foreach(check IN LISTS CHECKS) add_test(NAME ${check} COMMAND test_${check}) endforeach() -# Validate the public native-renderer dependency with real source and fake COM. -if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") - find_package(Python3 COMPONENTS Interpreter QUIET) - if(Python3_Interpreter_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/plume_d3d12.cpp") - add_test(NAME plume_command_creation - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_plume_command_creation.py - ${CMAKE_CURRENT_BINARY_DIR}/fixtures/plume_command_creation - --compiler ${CMAKE_CXX_COMPILER}) - set_tests_properties(plume_command_creation PROPERTIES TIMEOUT 60) - add_test(NAME gpu_timestamps - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_gpu_timestamps.py - ${CMAKE_CURRENT_BINARY_DIR}/fixtures/gpu_timestamps - --compiler ${CMAKE_CXX_COMPILER}) - set_tests_properties(gpu_timestamps PROPERTIES TIMEOUT 60) - if(UNIX AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/.git") - add_test(NAME prepare_plume - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_prepare_plume.py - ${CMAKE_CURRENT_BINARY_DIR}/fixtures/prepare_plume) - set_tests_properties(prepare_plume PROPERTIES TIMEOUT 60) - endif() - endif() -endif() - -# Compile production bodies with deterministic allocation and fence failures. -# These use GNU-style compiler arguments and require no SDK or game assets. -if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") - find_package(Python3 COMPONENTS Interpreter QUIET) +# These fixtures compile production function bodies with fake memory and fences. +# Base fixtures need no SDK, shader archive, generated game code or GPU. +# Optional fixtures below consume actual dependency sources when initialized. +# Their compiler flags target the GNU-style drivers (including clang++ on Windows). +# These Python drivers compile and execute host binaries without CMake's cross +# flags or emulator; only register them for a native host configuration. +if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING + AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") if(Python3_Interpreter_FOUND) - foreach(check IN ITEMS draw_device_view state_call_diagnostic buffer_alloc_diagnostic native_build_fingerprint command_slots constant_timing frame_cadence cull_state draw_diagnostic prelink_workers prelink_emission - frame_ring_transaction submission_waits pipeline_failures - completion_callbacks cpu_resource_wait draw_upload query_retirement) - add_test(NAME ${check} - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_${check}.py - ${CMAKE_CURRENT_BINARY_DIR}/fixtures/${check} - --compiler ${CMAKE_CXX_COMPILER}) - set_tests_properties(${check} PROPERTIES TIMEOUT 60) - endforeach() + set(PRODUCTION_CHECKS draw_device_view state_call_diagnostic buffer_alloc_diagnostic + command_slots completion_callbacks constant_timing cpu_resource_wait cull_state + draw_diagnostic draw_index_bounds draw_upload frame_cadence frame_ring_transaction + framebuffer_owners memory_watch nonindexed_buffer_windows quad_buffer_windows + pipeline_failures shader_binding_failures prelink_emission prelink_workers + query_retirement shader_bounds submission_waits texture_binding_failures + texture_source_residency texture_authority_probe) + find_program(LEGO_GPU_CHECK_NINJA NAMES ninja NO_CMAKE_FIND_ROOT_PATH) + find_program(LEGO_GPU_CHECK_MAKE NAMES make gmake NO_CMAKE_FIND_ROOT_PATH) + if(LEGO_GPU_CHECK_NINJA) + set(LEGO_GPU_CHECK_GENERATOR "Ninja") + elseif(LEGO_GPU_CHECK_MAKE) + set(LEGO_GPU_CHECK_GENERATOR "Unix Makefiles") + endif() + if(LEGO_GPU_CHECK_GENERATOR) + list(APPEND PRODUCTION_CHECKS native_build_fingerprint) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/thirdparty/xxHash/xxhash.h") + list(APPEND PRODUCTION_CHECKS shader_capture_dedup) + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/thirdparty/xxHash/xxhash.c") + list(APPEND PRODUCTION_CHECKS precompiled_pack) + endif() + endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/shaders/bytecode/d3d12_5_1/texture_load_depth_unorm_cs.h" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/d3d12/texture_cache.cpp" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/pipeline/texture/util.cpp" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/pipeline/texture/cache.cpp") - add_test(NAME texture_depth_upload - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_texture_depth_upload.py - ${CMAKE_CURRENT_BINARY_DIR}/fixtures/texture_depth_upload - --compiler ${CMAKE_CXX_COMPILER}) - set_tests_properties(texture_depth_upload PROPERTIES TIMEOUT 60) + list(APPEND PRODUCTION_CHECKS texture_depth_upload) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../research/reblue/thirdparty/XenosRecomp/XenosRecomp/shader_recompiler.cpp" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../research/reblue/thirdparty/XenosRecomp/thirdparty/fmt/include/fmt/format.h") + list(APPEND PRODUCTION_CHECKS unmapped_vertex_fetch) endif() - # Verify the actual SDK project template when its submodule is present. + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/plume_d3d12.cpp") + list(APPEND PRODUCTION_CHECKS plume_command_creation gpu_timestamps resize_fences) + endif() + if(UNIX) + list(APPEND PRODUCTION_CHECKS prelink_publication) + endif() + # The cross-build fixture uses Unix host-tool scripts and reads the SDK's + # actual generated-project template; an asset-free checkout may lack it. if(UNIX AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/resources/templates/init/rexglue_cmake.inja") - add_test(NAME cross_build_producers - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_cross_build_producers.py - ${CMAKE_CURRENT_BINARY_DIR}/fixtures/cross_build_producers - --compiler ${CMAKE_CXX_COMPILER}) - set_tests_properties(cross_build_producers PROPERTIES TIMEOUT 60) + list(APPEND PRODUCTION_CHECKS cross_build_producers) endif() + foreach(check IN LISTS PRODUCTION_CHECKS) + set(extra_arguments) + if(check STREQUAL "native_build_fingerprint") + list(APPEND extra_arguments --generator "${LEGO_GPU_CHECK_GENERATOR}") + endif() + add_test(NAME ${check} + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_${check}.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/${check} + --compiler ${CMAKE_CXX_COMPILER} ${extra_arguments}) + set_tests_properties(${check} PROPERTIES TIMEOUT 60) + endforeach() # This fixture cross-compiles a tiny SDK-free Windows ABI cache and # inspects/archives a real COFF blob, rather than linking a host GPU app. if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") @@ -119,28 +130,28 @@ if(BUILD_TESTING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") set_tests_properties(binary_cache_build PROPERTIES TIMEOUT 60) endif() endif() - if(UNIX) - add_test(NAME prelink_publication - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_prelink_publication.py - ${CMAKE_CURRENT_BINARY_DIR}/fixtures/prelink_publication - --compiler ${CMAKE_CXX_COMPILER}) - set_tests_properties(prelink_publication PROPERTIES TIMEOUT 60) - endif() - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/plume_d3d12.cpp") - add_test(NAME resize_fences - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_resize_fences.py - ${CMAKE_CURRENT_BINARY_DIR}/fixtures/resize_fences - --compiler ${CMAKE_CXX_COMPILER}) - set_tests_properties(resize_fences PROPERTIES TIMEOUT 60) - endif() + else() + message(STATUS "Python3 unavailable: production-body fixtures are disabled") endif() endif() -# These fixtures exercise the real extraction CLI with synthetic inputs. -if(BUILD_TESTING) - find_package(Python3 COMPONENTS Interpreter QUIET) - find_program(LEGO_DOTNET_EXECUTABLE NAMES dotnet) - if(Python3_Interpreter_FOUND AND LEGO_DOTNET_EXECUTABLE) +# This offline Git preparation check invokes no C++ compiler. +if(BUILD_TESTING AND UNIX AND Python3_Interpreter_FOUND + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/.git" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/plume_d3d12.cpp") + find_package(Git QUIET) + if(Git_FOUND) + add_test(NAME prepare_plume + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_prepare_plume.py + ${CMAKE_CURRENT_BINARY_DIR}/fixtures/prepare_plume) + set_tests_properties(prepare_plume PROPERTIES TIMEOUT 60) + endif() +endif() + +# Extraction CLI fixtures do not depend on the selected C++ compiler frontend. +if(BUILD_TESTING AND Python3_Interpreter_FOUND) + find_program(LEGO_DOTNET_EXECUTABLE NAMES dotnet NO_CMAKE_FIND_ROOT_PATH) + if(LEGO_DOTNET_EXECUTABLE) foreach(check IN ITEMS extract_paths image_shader_extract embedded_shader_extract) add_test(NAME ${check} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_${check}.py diff --git a/tools/gpu-aot/test_draw_upload.py b/tools/gpu-aot/test_draw_upload.py index 162aa3c..2209390 100644 --- a/tools/gpu-aot/test_draw_upload.py +++ b/tools/gpu-aot/test_draw_upload.py @@ -88,6 +88,7 @@ def function(signature): bool timing_enabled=true; bool NativeTextureTimingEnabled(){return timing_enabled;} std::unique_ptr g_state=std::make_unique();std::mutex g_mutex; +u64 g_recording_serial=0; struct HostDevice { static auto LockRecording(){static std::recursive_mutex m;return std::unique_lock(m);} static DrawUploadSlice AllocateDrawUpload(u32 size); @@ -96,23 +97,28 @@ def function(signature): int main() { assert(!HostDevice::AllocateDrawUpload(0)); auto first=HostDevice::AllocateDrawUpload(8816);assert(first&&first.offset==0); + assert(g_recording_serial==1); std::memset(first.mapped,0xA5,8816); auto second=HostDevice::AllocateDrawUpload(8816);assert(second.offset==8960&&second.buffer==first.buffer); std::memset(second.mapped,0x5A,8816); for(int i=0;i<550;++i){auto s=HostDevice::AllocateDrawUpload(8816);assert(s&&s.offset%256==0);std::memset(s.mapped,i&255,8816);} assert(g_state->device->allocations==2);assert(g_state->queue->waits==0); + assert(g_recording_serial==1); // Arena reuse stays in the same recording. for(int i=0;i<8816;++i) assert(static_cast(first.mapped)[i]==0xA5); g_state->command_list_open=false;g_state->frame_submitted[0]=true;g_state->frame_slot=1; auto other=HostDevice::AllocateDrawUpload(8816);assert(other&&other.buffer!=first.buffer); + assert(g_recording_serial==2); std::memset(other.mapped,0xCC,8816);assert(g_state->queue->waits==0); assert(static_cast(first.mapped)[0]==0xA5); g_state->command_list_open=false;g_state->frame_submitted[1]=true;g_state->frame_slot=0; auto reuse=HostDevice::AllocateDrawUpload(8816);assert(reuse.buffer==first.buffer&&reuse.offset==0); + assert(g_recording_serial==3); // Reusing a slot starts a distinct recording. assert(g_state->queue->waits==1); // Reset only after old fence wait. assert(g_state->frame_slot_wait_calls==1 && g_state->frame_slot_wait_ms>=0); timing_enabled=false; g_state->command_list_open=false;g_state->frame_submitted[0]=true; auto untimed=HostDevice::AllocateDrawUpload(8816);assert(untimed); + assert(g_recording_serial==4); assert(g_state->queue->waits==2 && g_state->frame_slot_wait_calls==2); // Cheap count survives timing-off. std::memset(reuse.mapped,0xDD,8816);assert(static_cast(other.mapped)[0]==0xCC); auto large=HostDevice::AllocateDrawUpload(5*1024*1024);assert(large&&large.offset==0); @@ -120,6 +126,7 @@ def function(signature): auto tail=HostDevice::AllocateDrawUpload(1);assert(tail&&tail.offset==0); assert(g_state->device->allocations==5); g_state->device->fail=true;assert(!HostDevice::AllocateDrawUpload(8*1024*1024)); + assert(g_recording_serial==4); // Growth/failure do not begin another list. g_state.reset();assert(!HostDevice::AllocateDrawUpload(8816)); } ''') @@ -127,6 +134,6 @@ def function(signature): subprocess.run([a.compiler,'-std=c++20','-DNOMINMAX',str(h),'-o',str(exe)],check=True,timeout=45) subprocess.run([str(exe.resolve())],check=True,timeout=10) (a.output/'verification.json').write_text(json.dumps({'actual_production_functions':True,'fake_driver':True, - 'checks':['256-byte alignment','550 immutable draws','page growth','independent slots','reset after fence wait','oversize','allocation failure','shutdown'], + 'checks':['256-byte alignment','550 immutable draws','page growth','independent slots','reset after fence wait','recording serial advance and arena reuse stability','oversize','allocation failure','shutdown'], 'passed':True},indent=2)) print('Passed actual allocation/reset bodies: immutable draws, independent fence slots, alignment, growth and failure.') diff --git a/tools/gpu-aot/test_shader_binding_failures.py b/tools/gpu-aot/test_shader_binding_failures.py new file mode 100644 index 0000000..26fcade --- /dev/null +++ b/tools/gpu-aot/test_shader_binding_failures.py @@ -0,0 +1,344 @@ +"""Reject failed nonnull shader binds before cached PSOs, using production bodies.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess + + +def function(source, signature): + start = source.index(signature) + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--compiler', default='clang++') + parser.add_argument('--verify-regression', action='store_true', + help='also prove removing the production guard fails the cache-hit case') + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + shaders = (root / 'rexlego/src/gpu_native/shaders.cpp').read_text() + draw = (root / 'rexlego/src/gpu_native/draw.cpp').read_text() + hooks = (root / 'rexlego/src/gpu_native/hooks_shader.cpp').read_text() + header = (root / 'rexlego/src/gpu_native/shaders.h').read_text() + address_declaration_begin = header.index('u32 BoundShaderAddress(') + address_declaration = header[address_declaration_begin: + header.index(';', address_declaration_begin) + 1] + + # Keep the registry, binding, DXIL resolution and reset implementations + # unchanged. Only guest adoption and the host driver are test boundaries. + registry = shaders[shaders.index('struct ShaderResource {'): + shaders.index('std::shared_ptr FindResource(')] + binding_bodies = '\n'.join(function(shaders, signature) for signature in ( + 'bool BindShader(', 'plume::RenderShader* ResolveBoundShader(', + 'u32 BoundShaderAddress(', 'u64 BoundShaderHash(', + 'void ResetShaderResources(', + )) + setter_bodies = '\n'.join(function(hooks, signature) for signature in ( + 'void SetPixelShaderHook(', 'void SetVertexShaderHook(', + )) + key_types = draw[draw.index('struct PipelineKey {'):draw.index('u32 g_unsupported_draw_logs')] + # The unchanged cache lookup and shader-resolution gates are the behavior + # under test. Pipeline creation below them is supplied by the fake driver. + pipeline_begin = draw.index('plume::RenderPipeline* GetPipeline(') + pipeline_end = draw.index(' std::arraydrawInstanced(count, 1, use_windows ? 0 : start, 0);') + nonindex_begin = draw.rfind(' QueryDrawScope query_scope(commands);', 0, nonindex_draw) + assert nonindex_begin >= 0 + nonindex_end = draw.index('\n }', nonindex_begin) + nonindex_submission = draw[nonindex_begin:nonindex_end] + draw_hooks = '\n'.join(function(draw, signature) for signature in ( + 'u32 DrawVerticesHook(', 'u32 DrawIndexedVerticesHook(', + )) + + harness = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "gpu_native/index_draw.h" +using namespace legodimensions::gpu_native; +using u32=uint32_t;using u64=uint64_t;using i32=int32_t; +constexpr u32 kNativeRenderTargets=4,kNativeVertexStreams=16; +#define REXLOG_WARN(...) ((void)0) +#define REXLOG_ERROR(...) ((void)0) +enum class ShaderStage:u32{kVertex=6,kPixel=7}; +struct ShaderCacheEntry{u32 specConstantsMask=3;}; +struct ShaderBytecode{ + const void* data=nullptr;size_t size=0; + explicit operator bool()const{return data&&size;} +}; +namespace plume { +enum class RenderShaderFormat{DXIL}; +enum class RenderFormat{UNKNOWN,R16_UINT,R32_UINT}; +enum class RenderPrimitiveTopology{TRIANGLE_LIST}; +struct RenderShader{void setName(const char*){}}; +struct RenderPipeline{RenderShader* vs;RenderShader* ps;}; +} +struct FakeDevice{ + u32 shader_creates=0; + auto createShader(const void* data,size_t bytes,const char*,plume::RenderShaderFormat){ + assert(data&&bytes);++shader_creates;return std::make_unique(); + } +} device; +struct HostDevice{ + static inline std::recursive_mutex recording; + static auto LockRecording(){return std::unique_lock(recording);} + static auto Device(){return &device;} +}; +struct Memory{std::vector freed;void SystemHeapFree(u32 address){freed.push_back(address);}} memory; +#define REX_KERNEL_MEMORY() (&memory) +std::vector events; +templatevoid LongProbeEvent(const char* event,bool,Args&&...){events.emplace_back(event);} +''' + registry + r''' +// A guest address may be re-registered/recycled and become unadoptable. +// These fixtures retain exactly the same address and even a zero hash to +// exercise a cache collision with a formerly valid requested shader. +std::unordered_map> adoptable; +u32 adoption_calls=0,dxil_lookups=0; +std::shared_ptr AdoptShader(u32 address,ShaderStage){ + ++adoption_calls;auto it=adoptable.find(address); + return it==adoptable.end()?nullptr:it->second; +} +bool dxil_available=true; +ShaderBytecode FindDxil(u64,u32){ + ++dxil_lookups;static const char bytecode[]="DXIL"; + return dxil_available?ShaderBytecode{bytecode,sizeof(bytecode)}:ShaderBytecode{}; +} +''' + 'void PollPrecompiledShaders() {}\n' + address_declaration + '\n' + binding_bodies + r''' +struct D3DDevice{u32 vertex_shader=0,pixel_shader=0,vertex_declaration=0x4000;}; +''' + setter_bodies + key_types + r''' +struct VertexDeclarationView{u64 content_hash=0x1234;}; +u32 pipeline_creates=0; +''' + pipeline_prefix + r''' + auto pipeline=std::make_unique(); + pipeline->vs=vs;pipeline->ps=ps;++pipeline_creates; + auto [it,inserted]=g_pipelines.emplace(key,std::move(pipeline)); + assert(inserted);return it->second.get(); +} +struct Buffer{Buffer* at(u32 offset){assert(!offset);return this;}} buffer; +struct BufferResourceView{Buffer* buffer=nullptr;u32 length=0,guest_format=0;}; +enum class BufferKind{kIndex}; +u32 buffer_resolves=0; +BufferResourceView ResolveBufferResourceView(u32,BufferKind){++buffer_resolves;return {&buffer,256,1};} +BufferResourceView ResolveBufferResourceWindow(u32,BufferKind,u32,u32){assert(false);return {};} +namespace plume { +struct RenderIndexBufferView{ + RenderIndexBufferView(Buffer* b,u32 bytes,RenderFormat){assert(b&&bytes);} +}; +} +u32 query_begins=0,query_ends=0,query_failures=0,submitted_draws=0; +struct Recorder{ + u32 index_bindings=0; + void setIndexBuffer(const plume::RenderIndexBufferView*){++index_bindings;} + void drawInstanced(u32 count,u32 instances,u32 start,u32 first){ + assert(count==6&&instances==1&&start==5&&!first);++submitted_draws; + } + void drawIndexedInstanced(u32 count,u32 instances,u32 start,i32 base,u32 first){ + assert(count==6&&instances==1&&start==5&&base==-7&&!first);++submitted_draws; + } +} recorder; +struct QueryDrawScope{ + explicit QueryDrawScope(Recorder*){++query_begins;} + ~QueryDrawScope(){++query_ends;} +}; +void FailActiveQueries(){++query_failures;} +bool ConsumeCpuPoolCopyDraw(u32,u32,u32){return false;} +bool DispatchDraw(D3DDevice* device,u32,bool indexed,u32 start,u32 count,i32 base_vertex){ + if(!device||!count)return false; + struct {u32 vertex_declaration;u32 index_buffer;} bindings{device->vertex_declaration,0x3000}; + const VertexDeclarationView declaration; +''' + key_construction + r''' + auto* pipeline=GetPipeline(key,declaration,plume::RenderPrimitiveTopology::TRIANGLE_LIST, + plume::RenderFormat::UNKNOWN); + if(!pipeline)return false; + auto* commands=&recorder; + const bool use_windows=false; + const u32 min_index=0,index_offset=0,index_bytes=0; + ''' + index_branch + r''' else { +''' + nonindex_submission + r''' + } + return true; +} +''' + draw_hooks + r''' +std::shared_ptr add_shader(u32 address,ShaderStage stage,u64 hash, + bool owned=false){ + static const ShaderCacheEntry entry; + auto resource=std::make_shared(); + resource->guest_address=address;resource->stage=stage;resource->hash=hash; + resource->cache_entry=&entry;resource->owns_guest_memory=owned; + g_registry[address]=resource;adoptable[address]=resource;return resource; +} +void reset_counters(){ + query_begins=query_ends=query_failures=submitted_draws=buffer_resolves=0; + recorder={};events.clear(); +} +void rejected(D3DDevice& guest,bool indexed,const char* event="draw_missing_pixel_shader"){ + reset_counters(); + if(indexed)DrawIndexedVerticesHook(&guest,4,u32(-7),5,6); + else DrawVerticesHook(&guest,4,5,6); + assert(submitted_draws==0 && "failed nonnull shader binding reused a cached pipeline"); + assert(query_begins==0&&query_ends==0&&query_failures==1); + assert(!buffer_resolves&&!recorder.index_bindings); + assert(events.size()==1&&events[0]==event); +} +void accepted(D3DDevice& guest,bool indexed){ + reset_counters(); + if(indexed)DrawIndexedVerticesHook(&guest,4,u32(-7),5,6); + else DrawVerticesHook(&guest,4,5,6); + assert(submitted_draws==1&&query_begins==1&&query_ends==1&&!query_failures); + assert(buffer_resolves==u32(indexed)&&recorder.index_bindings==u32(indexed)); + assert(events.empty()); +} +int main(){ + D3DDevice guest; + auto vs=add_shader(0x1000,ShaderStage::kVertex,0); + // A zero hash is valid at the registry boundary; addresses/hashes after a + // failed rebind match the previously cached key if the new guard is removed. + auto ps=add_shader(0x2000,ShaderStage::kPixel,0); + SetVertexShaderHook(&guest,0x1000);SetPixelShaderHook(&guest,0x2000); + assert(guest.vertex_shader==0x1000&&guest.pixel_shader==0x2000); + accepted(guest,false);accepted(guest,true); + assert(g_pipelines.size()==1&&pipeline_creates==1); + assert(adoption_calls==2&&device.shader_creates==2&&dxil_lookups==2); + const auto* cached=g_pipelines.begin()->second.get(); + const u32 original_pipeline_creates=pipeline_creates,original_dxil_lookups=dxil_lookups; + + adoptable.erase(0x2000);g_registry.erase(0x2000); + SetPixelShaderHook(&guest,0x2000); + bool failed=false; + assert(guest.pixel_shader==0x2000); + assert(BoundShaderAddress(ShaderStage::kPixel,&failed)==0x2000&&failed); + assert(!ResolveBoundShader(ShaderStage::kPixel,0)); + rejected(guest,false);rejected(guest,true); + assert(g_pipelines.size()==1&&g_pipelines.begin()->second.get()==cached); + assert(pipeline_creates==original_pipeline_creates&&dxil_lookups==original_dxil_lookups); + + // Successful adoption at the same address immediately recovers; the known + // good cached PSO is still usable, without resolving shader bytecode per draw. + adoptable[0x2000]=ps;g_registry[0x2000]=ps; + SetPixelShaderHook(&guest,0x2000); + assert(BoundShaderAddress(ShaderStage::kPixel,&failed)==0x2000&&!failed); + const auto successful_adoptions=adoption_calls; + accepted(guest,false);accepted(guest,true); + assert(adoption_calls==successful_adoptions&&dxil_lookups==original_dxil_lookups); + assert(pipeline_creates==original_pipeline_creates); + + // The cache-hit bypass applies equally to a failed vertex rebind. Both + // stages must remain unavailable even when the former cache key still exists. + adoptable.erase(0x1000);g_registry.erase(0x1000); + SetVertexShaderHook(&guest,0x1000); + assert(guest.vertex_shader==0x1000); + assert(BoundShaderAddress(ShaderStage::kVertex,&failed)==0x1000&&failed); + rejected(guest,false,"draw_missing_vertex_shader"); + rejected(guest,true,"draw_missing_vertex_shader"); + assert(g_pipelines.size()==1&&pipeline_creates==original_pipeline_creates); + adoptable[0x1000]=vs;g_registry[0x1000]=vs; + SetVertexShaderHook(&guest,0x1000); + accepted(guest,false);accepted(guest,true); + assert(pipeline_creates==original_pipeline_creates&&dxil_lookups==original_dxil_lookups); + + // Genuine null requests intentionally create/cache the depth-only PSO. + SetPixelShaderHook(&guest,0); + assert(guest.pixel_shader==0&&BoundShaderAddress(ShaderStage::kPixel,&failed)==0&&!failed); + accepted(guest,false);accepted(guest,true); + assert(g_pipelines.size()==2&&pipeline_creates==2); + bool found_null=false; + for(const auto& [key,pipeline]:g_pipelines){ + if(!key.pixel_shader){assert(pipeline->vs&&!pipeline->ps);found_null=true;} + } + assert(found_null); + + // A wrong-stage nonnull adoption is also failure, despite a nonnull wrapper. + adoptable[0x2000]=vs;SetPixelShaderHook(&guest,0x2000); + assert(BoundShaderAddress(ShaderStage::kPixel,&failed)==0x2000&&failed); + rejected(guest,false); + SetPixelShaderHook(&guest,0);accepted(guest,false); + + // Non-adoption DXIL failures retain a nonnull address and are rejected by + // the existing GetPipeline guard. They never become a depth-only cache key. + auto unavailable=add_shader(0x2500,ShaderStage::kPixel,0xBAD); + SetPixelShaderHook(&guest,0x2500);dxil_available=false;reset_counters(); + DrawVerticesHook(&guest,4,5,6); + assert(!submitted_draws&&!query_begins&&query_failures==1&&g_pipelines.size()==2); + dxil_available=true;accepted(guest,false);assert(g_pipelines.size()==3); + + // Reset clears both failure and successful binding records, while retaining + // the original ownership rule for freeing guest memory. + add_shader(0x9000,ShaderStage::kPixel,0xDEAD,true); + adoptable.erase(0x2000);SetPixelShaderHook(&guest,0x2000); + ResetShaderResources(); + assert(g_registry.empty()&&memory.freed==std::vector{0x9000}); + assert(!BoundShaderAddress(ShaderStage::kPixel,&failed)&&!failed); + assert(!BoundShaderAddress(ShaderStage::kVertex)&&!ResolveBoundShader(ShaderStage::kVertex,0)); + g_pipelines.clear(); + SetVertexShaderHook(&guest,0x1000);SetPixelShaderHook(&guest,0); + accepted(guest,false); +} +''' + args.output.mkdir(parents=True, exist_ok=True) + cpp = args.output / 'shader-binding-failures.cpp' + cpp.write_text(harness) + executable_suffix = '.exe' if os.name == 'nt' else '' + exe = args.output / ('shader-binding-failures' + executable_suffix) + compile_args = [args.compiler, '-std=c++20', '-O2', '-UNDEBUG', '-pthread', + '-I' + str(root / 'rexlego/src')] + subprocess.run(compile_args + [str(cpp), '-o', str(exe)], check=True, timeout=45) + subprocess.run([str(exe.resolve())], check=True, timeout=10) + regression = None + if args.verify_regression: + for stage, guard in guards.items(): + mutant = args.output / ('shader-binding-without-' + stage + '-guard.cpp') + mutant.write_text(harness.replace(guard, '', 1)) + mutant_exe = mutant.with_suffix(executable_suffix) + subprocess.run(compile_args + [str(mutant), '-o', str(mutant_exe)], + check=True, timeout=45) + result = subprocess.run([str(mutant_exe.resolve())], capture_output=True, + text=True, timeout=10) + assert result.returncode != 0 and ( + 'failed nonnull shader binding reused a cached pipeline' in result.stderr) + regression = 'removing either production stage guard submits its formerly cached failed binding' + report = { + 'result': 'PASS', 'regression_proof': regression, + 'production_sha256': { + 'shader_binding_bodies': hashlib.sha256(binding_bodies.encode()).hexdigest(), + 'pipeline_cache_and_resolution': hashlib.sha256(pipeline_prefix.encode()).hexdigest(), + 'draw_key_construction': hashlib.sha256(key_construction.encode()).hexdigest(), + }, + 'cases': ['PS and VS nonnull failed rebinds with cached same-address/hash-zero PSO', + 'ordinary and indexed draws reject without query scopes or index uploads', + 'same-address recovery reuses cached PSO without per-draw adoption or DXIL', + 'intentional null pixel request retains depth-only pipeline', + 'wrong-stage adoption and uncached DXIL lookup failure', + 'reset clears latest requests and frees owned guest memory only'], + 'boundaries': 'guest adoption, host shader/pipeline construction and driver are fake; ' + 'registry/bind/resolve/reset/hooks, cache gate, key guard and submission are production', + } + (args.output / 'verification.json').write_text(json.dumps(report, indent=2) + '\n') + print('PASS: production shader binding/cache gates; failed nonnull VS/PS reject before ' + 'cached PSO, query scopes and draws; null/recovery/reset remain valid') + + +if __name__ == '__main__': + main() From f397c015f95e8e6f7e8db6b4c5f74c87decc251c Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:03:19 +0200 Subject: [PATCH 28/40] Isolate command state cache experiments from binding diagnostics --- tools/gpu-aot/regression/README.md | 14 ++++++ tools/gpu-aot/regression/compare_samples.py | 8 +++- tools/gpu-aot/regression/execute_run.py | 2 +- tools/gpu-aot/regression/prepare_run.py | 31 ++++++++++++- tools/gpu-aot/regression/sample_cadence.py | 7 ++- tools/gpu-aot/regression/test_regression.py | 51 ++++++++++++++++++++- 6 files changed, 108 insertions(+), 5 deletions(-) diff --git a/tools/gpu-aot/regression/README.md b/tools/gpu-aot/regression/README.md index 498e5e8..5b94303 100644 --- a/tools/gpu-aot/regression/README.md +++ b/tools/gpu-aot/regression/README.md @@ -93,6 +93,20 @@ closure alone does not establish internal ABI compatibility. PM4 cannot use native diagnostic flags. Actual runtime route logs and visible rendering are still required; a launch descriptor does not prove either. +`--state-cache off` (default) or `on` explicitly selects the experimental Plume +command state cache. The native descriptor writes an exact `0` or `1`; inherited +cache settings are removed even when an older descriptor omits its sanitizer. +Compare frozen ON/OFF runs with `--vary state-cache`. Other experiments retain +this setting in their identity checks. Runtime activation still needs separate +evidence from the patched backend; requesting the flag cannot prove activation. + +`--state-calls` requests a finite binding-request capture at private +`logs/state-calls.csv`. Creating `logs/state-calls.trigger` arms its next 120 +present intervals, with at most 4096 accounted guest draws per interval. +Equal adjacent guest requests do not prove redundancy across intervening helper +or external calls. Clean cadence sampling rejects this diagnostic, including +after its budget has been exhausted; prepare a separate run for timing. + ## Fixed cadence intervals Use normal controls to load the checkpoint and required figures, without moving diff --git a/tools/gpu-aot/regression/compare_samples.py b/tools/gpu-aot/regression/compare_samples.py index 061c8d0..5e91f2c 100644 --- a/tools/gpu-aot/regression/compare_samples.py +++ b/tools/gpu-aot/regression/compare_samples.py @@ -12,6 +12,7 @@ EXPERIMENT_FLAGS = {'command-slots': 'LEGO_NATIVE_COMMAND_SLOTS', 'timer-wait': 'REX_TIMER_WAIT_BLOCKING', + 'state-cache': 'PLUME_D3D12_STATE_CACHE', 'gpu-timestamps': 'LEGO_NATIVE_GPU_TIMESTAMPS'} @@ -20,6 +21,8 @@ def experimental_value(sample, varying): return sample['slots'] if varying == 'timer-wait': return sample['timer_wait'] + if varying == 'state-cache': + return 'on' if sample['flags'].get('PLUME_D3D12_STATE_CACHE', '0') == '1' else 'off' return 'on' if 'LEGO_NATIVE_GPU_TIMESTAMPS' in sample['flags'] else 'off' @@ -27,7 +30,8 @@ def comparison_identity(sample, varying='command-slots'): require(varying in EXPERIMENT_FLAGS, 'Unknown experimental variable') scene = sample['scene'] flags = {key: value for key, value in sample['flags'].items() - if key.startswith(('LEGO_NATIVE_', 'LEGO_DUMP_', 'LEGO_GPU_', 'REX_TIMER_'))} + if key.startswith(('LEGO_NATIVE_', 'LEGO_DUMP_', 'LEGO_GPU_', 'REX_TIMER_')) + or key == 'PLUME_D3D12_STATE_CACHE'} for key in ('LEGO_NATIVE_FRAME_METRICS', 'LEGO_DUMP_MISSING_SHADERS', 'LEGO_NATIVE_SHADER_PACK', 'LEGO_NATIVE_SHADER_PACK_TRIGGER'): flags.pop(key, None) # Private paths differ; sampler verifies their ownership. @@ -53,6 +57,7 @@ def compare(paths, varying='command-slots'): require(stats(rows) == sample['metrics'], 'Frozen CSV disagrees with sample metrics') outcomes = [{'source': str(path.resolve()), 'status': sample['status'], 'slots': sample['slots'], 'timer_wait': sample['timer_wait'], + 'state_cache': experimental_value(sample, 'state-cache'), 'gpu_timestamps': experimental_value(sample, 'gpu-timestamps'), 'metrics': sample.get('metrics'), 'rejection_reasons': sample.get('rejection_reasons', [])} for path, sample in zip(paths, samples)] @@ -69,6 +74,7 @@ def compare(paths, varying='command-slots'): means[str(value)] = {'windows': len(group), 'frames': frames, 'recorded_seconds': seconds, 'fps': frames/seconds} baseline, candidate = {'command-slots': ('3', '12'), 'timer-wait': ('spin', 'blocking'), + 'state-cache': ('off', 'on'), 'gpu-timestamps': ('off', 'on')}[varying] gain = (100*(means[candidate]['fps']/means[baseline]['fps']-1) if candidate in means and baseline in means else None) diff --git a/tools/gpu-aot/regression/execute_run.py b/tools/gpu-aot/regression/execute_run.py index ad8dc8c..f4a9d5f 100644 --- a/tools/gpu-aot/regression/execute_run.py +++ b/tools/gpu-aot/regression/execute_run.py @@ -14,7 +14,7 @@ HERE = Path(__file__).resolve().parent INTRINSIC_PREFIXES = ('LEGO_NATIVE_', 'LEGO_GPU_', 'LEGO_DUMP_', 'REX_') INTRINSIC_KEYS = ('XENIA_TOYPAD_PORT', 'REXGLUE_TOYPAD_PORT', 'PROTON_LOG', - 'VKD3D_SHADER_CACHE_PATH', 'DXVK_STATE_CACHE_PATH') + 'VKD3D_SHADER_CACHE_PATH', 'DXVK_STATE_CACHE_PATH', 'PLUME_D3D12_STATE_CACHE') def active_games(): diff --git a/tools/gpu-aot/regression/prepare_run.py b/tools/gpu-aot/regression/prepare_run.py index 4a97d3c..1ada634 100644 --- a/tools/gpu-aot/regression/prepare_run.py +++ b/tools/gpu-aot/regression/prepare_run.py @@ -294,6 +294,11 @@ def prepare(args): cadence = getattr(args, 'cadence_only', False) performance = getattr(args, 'performance', False) or cadence gpu_timestamps = getattr(args, 'gpu_timestamps', False) + state_cache = getattr(args, 'state_cache', 'off') + state_calls = getattr(args, 'state_calls', False) + require(state_cache in ('off', 'on'), 'Unknown command state cache mode') + require(args.renderer == 'native' or (state_cache == 'off' and not state_calls), + 'Command state cache and binding diagnostics require native route') require(not gpu_timestamps or args.renderer == 'native', 'GPU timestamp sidecar requires native route') command_slots = getattr(args, 'command_slots', None) @@ -461,6 +466,7 @@ def prepare(args): overrides['REX_TIMER_WAIT_BLOCKING'] = '1' if args.renderer == 'native': overrides.update({ + 'PLUME_D3D12_STATE_CACHE': '1' if state_cache == 'on' else '0', 'LEGO_NATIVE_STATIC_TEXTURE_WATCH': '1', 'LEGO_NATIVE_BUFFER_WATCH': '1', 'LEGO_NATIVE_TIMING': '1', 'LEGO_NATIVE_STENCIL': '1', 'LEGO_NATIVE_VIEWPORT': '1', 'LEGO_NATIVE_CULL': '1', 'LEGO_NATIVE_FRAME_METRICS': windows(final / 'logs/frames.csv'), @@ -478,6 +484,9 @@ def prepare(args): overrides['LEGO_NATIVE_COMMAND_SLOTS'] = str(command_slots or 3) if gpu_timestamps: overrides['LEGO_NATIVE_GPU_TIMESTAMPS'] = windows(final / 'logs/gpu-timestamps.csv') + if state_calls: + overrides['LEGO_NATIVE_STATE_CALLS'] = windows(final / 'logs/state-calls.csv') + overrides['LEGO_NATIVE_STATE_CALLS_TRIGGER'] = windows(final / 'logs/state-calls.trigger') if args.buffer_windows: overrides['LEGO_NATIVE_BUFFER_WINDOWS'] = '1' if args.snapshots: @@ -501,7 +510,8 @@ def prepare(args): str(final / 'legodimensions.exe')], 'environment_overrides': overrides, 'remove_inherited_prefixes': list(SANITIZED_PREFIXES), 'remove_inherited_keys': ['XENIA_TOYPAD_PORT', 'REXGLUE_TOYPAD_PORT', 'PROTON_LOG', - 'VKD3D_SHADER_CACHE_PATH', 'DXVK_STATE_CACHE_PATH'], + 'VKD3D_SHADER_CACHE_PATH', 'DXVK_STATE_CACHE_PATH', + 'PLUME_D3D12_STATE_CACHE'], 'execution_implemented': False, 'execution_gate': 'Not launched. An operator must refuse any active legodimensions.exe before executing this descriptor; the operator owns launch and desktop. The preparation command does not implement or execute this gate.', } @@ -513,6 +523,7 @@ def prepare(args): 'performance_probe': performance, 'cadence_only_probe': cadence, 'gpu_timestamp_probe': gpu_timestamps, + 'state_cache': state_cache, 'state_call_probe': state_calls, 'command_slots': (command_slots or 3) if cadence else None, 'pm4_plugin': plugin_provenance, 'checkpoint': str(checkpoint), 'checkpoint_files': pinned_saves, @@ -600,6 +611,7 @@ def verify(run, workspace=None): 'Spin timer descriptor must omit the presence-based flag') if renderer == 'native': allowed.update({'LEGO_NATIVE_STATIC_TEXTURE_WATCH', 'LEGO_NATIVE_BUFFER_WATCH', 'LEGO_NATIVE_TIMING', + 'PLUME_D3D12_STATE_CACHE', 'LEGO_NATIVE_STATE_CALLS', 'LEGO_NATIVE_STATE_CALLS_TRIGGER', 'LEGO_NATIVE_STENCIL', 'LEGO_NATIVE_VIEWPORT', 'LEGO_NATIVE_CULL', 'LEGO_NATIVE_FRAME_METRICS', 'LEGO_DUMP_MISSING_SHADERS', 'LEGO_NATIVE_SHADER_PACK', 'LEGO_NATIVE_GPU_TIMESTAMPS', @@ -610,6 +622,8 @@ def verify(run, workspace=None): 1024 <= manifest['toypad_port'] <= 65535 and manifest['toypad_port'] != 9191, 'Unexpected environment override or private Toypad port') path_overrides = {'PROTON_LOG_DIR': run/'logs', 'LEGO_NATIVE_FRAME_METRICS': run/'logs/frames.csv', + 'LEGO_NATIVE_STATE_CALLS': run/'logs/state-calls.csv', + 'LEGO_NATIVE_STATE_CALLS_TRIGGER': run/'logs/state-calls.trigger', 'LEGO_NATIVE_GPU_TIMESTAMPS': run/'logs/gpu-timestamps.csv', 'LEGO_DUMP_MISSING_SHADERS': run/'shader-capture', 'LEGO_NATIVE_SHADER_PACK': run/'hotload/next.pack', @@ -622,6 +636,17 @@ def verify(run, workspace=None): require(env[key] == (str(target) if key == 'PROTON_LOG_DIR' else windows(target)), 'Diagnostic output escaped the private run: '+key) timestamp_probe = manifest.get('gpu_timestamp_probe', False) + cache_mode = manifest.get('state_cache', 'off') + require(cache_mode in ('off', 'on') and + ((renderer == 'native' and env.get('PLUME_D3D12_STATE_CACHE', '0') == + ('1' if cache_mode == 'on' else '0')) or + (renderer == 'pm4' and cache_mode == 'off' and 'PLUME_D3D12_STATE_CACHE' not in env)), + 'Command state cache descriptor disagrees') + state_probe = manifest.get('state_call_probe', False) + require(type(state_probe) is bool and + ('LEGO_NATIVE_STATE_CALLS' in env) is state_probe and + ('LEGO_NATIVE_STATE_CALLS_TRIGGER' in env) is state_probe and + (not state_probe or renderer == 'native'), 'State-call descriptor disagrees') require(type(timestamp_probe) is bool and ('LEGO_NATIVE_GPU_TIMESTAMPS' in env) is timestamp_probe and (not timestamp_probe or renderer == 'native'), @@ -726,6 +751,10 @@ def main(): help='Explicit private cadence experiment slot count (default3)') p.add_argument('--gpu-timestamps', action='store_true', help='Opt-in native per-submission queue timestamps in a separate private CSV') + p.add_argument('--state-cache', choices=('off', 'on'), default='off', + help='Restart-only Plume command state cache experiment; default off') + p.add_argument('--state-calls', action='store_true', + help='Bounded guest binding request capture; private trigger arms next120 frames') p.add_argument('--snapshots', action='store_true', help='Explicitly enable bounded native DDS readbacks; changes timing workload') p.add_argument('--renderer', choices=('native', 'pm4'), default='native') diff --git a/tools/gpu-aot/regression/sample_cadence.py b/tools/gpu-aot/regression/sample_cadence.py index 2507e5a..27e2851 100644 --- a/tools/gpu-aot/regression/sample_cadence.py +++ b/tools/gpu-aot/regression/sample_cadence.py @@ -22,6 +22,7 @@ r'dumped missing shader|loaded additive precompiled pack') FAILURE_EVENTS = re.compile(r'\[(?:error|critical)\]|Native GPU:.*(?:failed|failure|device removed)', re.I) HEAVY_FLAGS = ('LEGO_NATIVE_TRACE_DIR', 'LEGO_NATIVE_DRAW_TRACE_TRIGGER', + 'LEGO_NATIVE_STATE_CALLS', 'LEGO_NATIVE_STATE_CALLS_TRIGGER', 'LEGO_GPU_SNAPSHOT_DIR', 'LEGO_NATIVE_MESH_TRACE', 'LEGO_NATIVE_AUDIT_BUFFER_WATCH', 'LEGO_NATIVE_AUDIT_TEXTURE_WATCH') CRITERIA = {'resource_quiet_seconds': 10, 'cadence_rows': 60, @@ -127,6 +128,10 @@ def context(run, scene_file, workspace): require('LEGO_NATIVE_TIMING' not in env and not any(k in env for k in HEAVY_FLAGS), 'Cadence scope contains detailed/heavy instrumentation') mode = manifest['timer_wait'] + cache_mode = manifest.get('state_cache', 'off') + require(cache_mode in ('off', 'on') and + env.get('PLUME_D3D12_STATE_CACHE', '0') == ('1' if cache_mode == 'on' else '0'), + 'Command state cache mode disagrees') require((mode == 'spin' and 'REX_TIMER_WAIT_BLOCKING' not in env) or (mode == 'blocking' and env.get('REX_TIMER_WAIT_BLOCKING') == '1'), 'Timer mode disagrees') require(env['LEGO_NATIVE_COMMAND_SLOTS'] == str(manifest['command_slots']) and @@ -192,7 +197,7 @@ def context(run, scene_file, workspace): packs.append({'generation': entry['generation'], 'sha256': entry['sha256']}) return {'run': str(run), 'scene': scene, 'scene_confirmation': {'source': str(scene_file.resolve()), **digest(scene_file)}, 'binaries': binaries, 'slots': manifest['command_slots'], 'criteria': dict(CRITERIA), - 'timer_wait': mode, + 'timer_wait': mode, 'state_cache': cache_mode, 'flags': env, 'normalized_config': config, 'shader_packs': packs, 'supporting_state': supporting, 'checkpoint': manifest['checkpoint_files'], diff --git a/tools/gpu-aot/regression/test_regression.py b/tools/gpu-aot/regression/test_regression.py index 3b2b1e1..93fda05 100644 --- a/tools/gpu-aot/regression/test_regression.py +++ b/tools/gpu-aot/regression/test_regression.py @@ -238,6 +238,54 @@ def test_performance_preparation_omits_heavy_diagnostics_and_rejects_tampering(s with self.assertRaisesRegex(Exception, 'without snapshots'): m.prepare(self.args(name='rejected-performance', performance=True, **changes)) + def test_state_cache_and_request_capture_descriptors_are_owned_and_explicit(self): + for mode in ('off', 'on'): + run = Path(m.prepare(self.args('state-'+mode, cadence_only=True, + state_cache=mode, state_calls=True))['run_root']) + manifest = json.loads((run/'regression-manifest.json').read_text()) + env = manifest['launch_plan']['environment_overrides'] + self.assertEqual(manifest['state_cache'], mode) + self.assertEqual(env['PLUME_D3D12_STATE_CACHE'], '1' if mode == 'on' else '0') + self.assertTrue(manifest['state_call_probe']) + m.verify(run) + original = dict(env) + for key, value, message in ( + ('PLUME_D3D12_STATE_CACHE', '0' if mode == 'on' else '1', 'cache descriptor'), + ('LEGO_NATIVE_STATE_CALLS', m.windows(self.root/'escaped.csv'), 'output escaped'), + ('LEGO_NATIVE_STATE_CALLS_TRIGGER', m.windows(self.root/'escaped.trigger'), 'output escaped')): + env[key] = value + (run/'regression-manifest.json').write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, message): m.verify(run) + env.clear(); env.update(original) + del env['LEGO_NATIVE_STATE_CALLS_TRIGGER'] + (run/'regression-manifest.json').write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, 'State-call descriptor'): m.verify(run) + for options in ({'state_cache': 'on'}, {'state_calls': True}): + with self.assertRaisesRegex(ValueError, 'require native route'): + m.prepare(self.args('pm4-state', renderer='pm4', **options)) + + def test_state_cache_comparison_preserves_other_settings_and_all_outcomes(self): + off, _ = self.sample_fixture('cache-off') + on, original = self.sample_fixture('cache-on', state_cache='on') + slow, _ = self.sample_fixture('cache-on-slow', state_cache='on', interval_ms=60) + rejected, _ = self.sample_fixture('cache-bad', state_cache='on', event='dumped missing shader') + result = comparer.compare([off, on, slow, rejected], 'state-cache') + self.assertTrue(result['comparable_identity']) + self.assertEqual(len(result['outcomes']), 4) + self.assertEqual(result['accepted_aggregate']['on']['windows'], 2) + self.assertEqual(result['accepted_aggregate']['off']['windows'], 1) + for flag, value in (('LEGO_NATIVE_CULL', '0'), ('REX_TIMER_WAIT_BLOCKING', '1')): + changed = json.loads(json.dumps(original)); changed['flags'][flag] = value + (on/'sample.json').write_text(json.dumps(changed)) + self.assertFalse(comparer.compare([off, on], 'state-cache')['comparable_identity']) + (on/'sample.json').write_text(json.dumps(original)) + self.assertFalse(comparer.compare([off, on], 'gpu-timestamps')['comparable_identity']) + + def test_state_request_capture_is_rejected_from_clean_cadence_scope(self): + run, scene = self.live_fixture('request-capture', state_calls=True) + with self.assertRaisesRegex(ValueError, 'detailed/heavy instrumentation'): + sampler.context(run, scene, self.root) + def test_actual_preparer_preserves_sources_and_routes_all_writable_paths(self): source_hashes = {p: m.digest(p) for p in self.root.rglob('*') if p.is_file()} with patch.object(subprocess, 'Popen', side_effect=AssertionError('process launched')), \ @@ -443,7 +491,8 @@ def test_executor_intrinsic_sanitizers_survive_an_empty_descriptor_policy(self): 'LEGO_GPU_SNAPSHOT_DIR':'live-readback','LEGO_DUMP_MISSING_SHADERS':'live-capture', 'REX_TIMER_WAIT_BLOCKING':'0','REX_USER_DATA_ROOT':'live-save', 'XENIA_TOYPAD_PORT':'9191','REXGLUE_TOYPAD_PORT':'9191','PROTON_LOG':'1', - 'VKD3D_SHADER_CACHE_PATH':'live-vkd3d','DXVK_STATE_CACHE_PATH':'live-dxvk','PATH':'ordinary-path'} + 'VKD3D_SHADER_CACHE_PATH':'live-vkd3d','DXVK_STATE_CACHE_PATH':'live-dxvk', + 'PLUME_D3D12_STATE_CACHE':'1','PATH':'ordinary-path'} with patch.dict(os.environ,ambient,clear=True):actual=runner.environment(plan) self.assertEqual(actual,{'PATH':'ordinary-path','LEGO_NATIVE_PM4_REFERENCE':'0', 'REXGLUE_TOYPAD_PORT':'19207'}) From 20d89886ddb42759d223c1cd3bbbf3532a501dbc Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:11:10 +0200 Subject: [PATCH 29/40] Add verified opt-in D3D12 pipeline and root signature caching --- rexlego/CMakeLists.txt | 3 + tools/gpu-aot/CMakeLists.txt | 2 +- tools/gpu-aot/patches/plume-state-cache.patch | 174 ++++++++++++ tools/gpu-aot/prepare_plume.py | 40 +-- .../gpu-aot/test_native_build_fingerprint.py | 2 +- tools/gpu-aot/test_plume_state_bindings.py | 268 ++++++++++++++++++ tools/gpu-aot/test_prepare_plume.py | 44 ++- 7 files changed, 514 insertions(+), 19 deletions(-) create mode 100644 tools/gpu-aot/patches/plume-state-cache.patch create mode 100644 tools/gpu-aot/test_plume_state_bindings.py diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt index 3be54f3..464ff3a 100644 --- a/rexlego/CMakeLists.txt +++ b/rexlego/CMakeLists.txt @@ -364,6 +364,9 @@ if(LEGODIMENSIONS_NATIVE_GPU) # from reaching the prelink target that creates that same file. set(LEGO_NATIVE_GPU_FINGERPRINT_INPUTS "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/plume/plume_d3d12.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/plume/plume_d3d12.h" + "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/prepare_plume.py" + "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/patches/plume-state-cache.patch" "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.h" "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/renderer_route.h" diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index 9f352d3..caaf989 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -78,7 +78,7 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING list(APPEND PRODUCTION_CHECKS unmapped_vertex_fetch) endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/plume_d3d12.cpp") - list(APPEND PRODUCTION_CHECKS plume_command_creation gpu_timestamps resize_fences) + list(APPEND PRODUCTION_CHECKS plume_command_creation plume_state_bindings gpu_timestamps resize_fences) endif() if(UNIX) list(APPEND PRODUCTION_CHECKS prelink_publication) diff --git a/tools/gpu-aot/patches/plume-state-cache.patch b/tools/gpu-aot/patches/plume-state-cache.patch new file mode 100644 index 0000000..d4c6f42 --- /dev/null +++ b/tools/gpu-aot/patches/plume-state-cache.patch @@ -0,0 +1,174 @@ +diff --git a/plume_d3d12.h b/plume_d3d12.h +--- a/plume_d3d12.h ++++ b/plume_d3d12.h +@@ -180,6 +180,15 @@ + const D3D12PipelineLayout *activeComputePipelineLayout = nullptr; + const D3D12PipelineLayout *activeGraphicsPipelineLayout = nullptr; + const D3D12GraphicsPipeline *activeGraphicsPipeline = nullptr; ++ // Opt-in emission caches use native identities, not last-graphics ++ // metadata: compute and ray tracing share the active pipeline state. ++ const void *cachedPipelineObject = nullptr; ++ uint32_t cachedPipelineType = 0; ++ bool cachedPipelineValid = false; ++ ID3D12RootSignature *cachedGraphicsRootSignature = nullptr; ++ ID3D12RootSignature *cachedComputeRootSignature = nullptr; ++ bool cachedGraphicsRootValid = false; ++ bool cachedComputeRootValid = false; + bool descriptorHeapsSet = false; + D3D12_PRIMITIVE_TOPOLOGY activeTopology = D3D_PRIMITIVE_TOPOLOGY_UNDEFINED; + uint32_t activeStencilRef = 0; +@@ -226,6 +235,9 @@ + void writeTimestamp(const RenderQueryPool *queryPool, uint32_t queryIndex) override; + void checkDescriptorHeaps(); + void notifyDescriptorHeapWasChangedExternally(); ++ // Invalidate PSO/root-signature emission caches after raw D3D changes. ++ // Rebind the intended pipeline/layout through Plume before draw/dispatch. ++ void notifyCommandStateWasChangedExternally(); + void checkTopology(); + void checkStencilRef(); + void checkFramebufferSamplePositions(); +diff --git a/plume_d3d12.cpp b/plume_d3d12.cpp +--- a/plume_d3d12.cpp ++++ b/plume_d3d12.cpp +@@ -7,6 +7,7 @@ + + #include "plume_d3d12.h" + ++#include + #include + + #include +@@ -44,6 +45,23 @@ + #endif + + namespace plume { ++ static bool D3D12StateCacheEnabled() { ++ static const bool enabled = [] { ++ const char *value = std::getenv("PLUME_D3D12_STATE_CACHE"); ++ return value != nullptr && value[0] == '1' && value[1] == '\0'; ++ }(); ++ return enabled; ++ } ++ ++ static void D3D12ReportStateCacheConfiguration() { ++ static const bool reported = [] { ++ fprintf(stderr, "Plume D3D12 command state cache: %s\n", ++ D3D12StateCacheEnabled() ? "enabled" : "disabled"); ++ return true; ++ }(); ++ (void)reported; ++ } ++ + static const uint32_t ShaderDescriptorHeapSize = 65536; + static const uint32_t SamplerDescriptorHeapSize = 1024; + static const uint32_t TargetDescriptorHeapSize = 16384; +@@ -1896,6 +1914,8 @@ + void D3D12CommandList::begin() { + assert(!open); + ++ D3D12ReportStateCacheConfiguration(); ++ notifyCommandStateWasChangedExternally(); + commandAllocator->Reset(); + d3d->Reset(commandAllocator, nullptr); + open = true; +@@ -1917,6 +1937,7 @@ + activeTopology = D3D_PRIMITIVE_TOPOLOGY_UNDEFINED; + activeStencilRef = 0; + descriptorHeapsSet = false; ++ notifyCommandStateWasChangedExternally(); + } + + void D3D12CommandList::barriers(RenderBarrierStages stages, const RenderBufferBarrier *bufferBarriers, uint32_t bufferBarriersCount, const RenderTextureBarrier *textureBarriers, uint32_t textureBarriersCount) { +@@ -2055,35 +2076,56 @@ + assert(pipeline != nullptr); + + const D3D12Pipeline *interfacePipeline = static_cast(pipeline); ++ const uint32_t pipelineType = uint32_t(interfacePipeline->type); ++ const bool cacheEnabled = D3D12StateCacheEnabled(); + switch (interfacePipeline->type) { + case D3D12Pipeline::Type::Compute: { + const D3D12ComputePipeline *computePipeline = static_cast(interfacePipeline); +- d3d->SetPipelineState(computePipeline->d3d); ++ if (!cacheEnabled || !cachedPipelineValid || cachedPipelineType != pipelineType || ++ cachedPipelineObject != computePipeline->d3d) { ++ d3d->SetPipelineState(computePipeline->d3d); ++ } ++ cachedPipelineObject = computePipeline->d3d; + break; + } + case D3D12Pipeline::Type::Graphics: { + const D3D12GraphicsPipeline *graphicsPipeline = static_cast(interfacePipeline); +- d3d->SetPipelineState(graphicsPipeline->d3d); ++ if (!cacheEnabled || !cachedPipelineValid || cachedPipelineType != pipelineType || ++ cachedPipelineObject != graphicsPipeline->d3d) { ++ d3d->SetPipelineState(graphicsPipeline->d3d); ++ } ++ cachedPipelineObject = graphicsPipeline->d3d; + activeGraphicsPipeline = graphicsPipeline; + break; + } + case D3D12Pipeline::Type::Raytracing: { + assert(d3dV4 != nullptr); + const D3D12RaytracingPipeline *raytracingPipeline = static_cast(interfacePipeline); +- d3dV4->SetPipelineState1(raytracingPipeline->stateObject); ++ if (!cacheEnabled || !cachedPipelineValid || cachedPipelineType != pipelineType || ++ cachedPipelineObject != raytracingPipeline->stateObject) { ++ d3dV4->SetPipelineState1(raytracingPipeline->stateObject); ++ } ++ cachedPipelineObject = raytracingPipeline->stateObject; + break; + } + default: + assert(false && "Unknown pipeline type."); +- break; +- } ++ return; ++ } ++ cachedPipelineType = pipelineType; ++ cachedPipelineValid = true; + } + + void D3D12CommandList::setComputePipelineLayout(const RenderPipelineLayout *pipelineLayout) { + assert(pipelineLayout != nullptr); + + const D3D12PipelineLayout *interfacePipelineLayout = static_cast(pipelineLayout); +- d3d->SetComputeRootSignature(interfacePipelineLayout->rootSignature); ++ if (!D3D12StateCacheEnabled() || !cachedComputeRootValid || ++ cachedComputeRootSignature != interfacePipelineLayout->rootSignature) { ++ d3d->SetComputeRootSignature(interfacePipelineLayout->rootSignature); ++ } ++ cachedComputeRootSignature = interfacePipelineLayout->rootSignature; ++ cachedComputeRootValid = true; + activeComputePipelineLayout = interfacePipelineLayout; + } + +@@ -2109,7 +2151,12 @@ + assert(pipelineLayout != nullptr); + + const D3D12PipelineLayout *interfacePipelineLayout = static_cast(pipelineLayout); +- d3d->SetGraphicsRootSignature(interfacePipelineLayout->rootSignature); ++ if (!D3D12StateCacheEnabled() || !cachedGraphicsRootValid || ++ cachedGraphicsRootSignature != interfacePipelineLayout->rootSignature) { ++ d3d->SetGraphicsRootSignature(interfacePipelineLayout->rootSignature); ++ } ++ cachedGraphicsRootSignature = interfacePipelineLayout->rootSignature; ++ cachedGraphicsRootValid = true; + activeGraphicsPipelineLayout = interfacePipelineLayout; + } + +@@ -2464,6 +2511,15 @@ + + void D3D12CommandList::notifyDescriptorHeapWasChangedExternally() { + descriptorHeapsSet = false; ++ } ++ ++ void D3D12CommandList::notifyCommandStateWasChangedExternally() { ++ cachedPipelineObject = nullptr; ++ cachedPipelineValid = false; ++ cachedGraphicsRootSignature = nullptr; ++ cachedComputeRootSignature = nullptr; ++ cachedGraphicsRootValid = false; ++ cachedComputeRootValid = false; + } + + void D3D12CommandList::checkTopology() { diff --git a/tools/gpu-aot/prepare_plume.py b/tools/gpu-aot/prepare_plume.py index 3e7fa44..d7bfbc5 100644 --- a/tools/gpu-aot/prepare_plume.py +++ b/tools/gpu-aot/prepare_plume.py @@ -67,7 +67,10 @@ def main(): parser.add_argument("--check", action="store_true", help="Verify only; do not modify the checkout") args = parser.parse_args() checkout = args.checkout.resolve() - patch = root / "tools/gpu-aot/patches/plume-lego.patch" + patches = ( + (root / "tools/gpu-aot/patches/plume-lego.patch", "Native Plume fixes"), + (root / "tools/gpu-aot/patches/plume-state-cache.patch", "Native Plume state-cache fixes"), + ) def git(*arguments, check=False): return subprocess.run(["git", "-C", str(checkout), *arguments], @@ -81,25 +84,30 @@ def git(*arguments, check=False): revision = git("rev-parse", "HEAD", check=True).stdout.strip() if revision != PIN: parser.error(f"Expected public Plume {PIN}, got {revision}; select the recorded dependency revision") - if git("apply", "--reverse", "--check", str(patch)).returncode == 0: + applied = False + for patch, description in patches: + if git("apply", "--reverse", "--check", str(patch)).returncode == 0: + try: + verify_patched_source(checkout, patch) + except ValueError as error: + parser.error(str(error)) + continue + if args.check: + parser.error(description + " are missing or incomplete; run prepare_plume.py without --check") + applicability = git("apply", "--check", str(patch)) + if applicability.returncode: + parser.error("Cannot apply " + description.lower() + " without conflicting with existing edits:\n" + applicability.stderr.strip()) + git("apply", str(patch), check=True) + git("apply", "--reverse", "--check", str(patch), check=True) try: verify_patched_source(checkout, patch) except ValueError as error: parser.error(str(error)) - print(f"PASS: public Plume {PIN} with native fence/resize/command-allocation fixes") - return - if args.check: - parser.error("Native Plume fixes are missing or incomplete; run prepare_plume.py without --check") - applicability = git("apply", "--check", str(patch)) - if applicability.returncode: - parser.error("Cannot apply native Plume fixes without conflicting with existing edits:\n" + applicability.stderr.strip()) - git("apply", str(patch), check=True) - git("apply", "--reverse", "--check", str(patch), check=True) - try: - verify_patched_source(checkout, patch) - except ValueError as error: - parser.error(str(error)) - print(f"Applied native fence/resize/command-allocation fixes to public Plume {PIN}") + applied = True + if applied: + print(f"Applied native fence/resize/command-allocation fixes and opt-in state cache to public Plume {PIN}") + else: + print(f"PASS: public Plume {PIN} with native fence/resize/command-allocation fixes and opt-in state cache") if __name__ == "__main__": diff --git a/tools/gpu-aot/test_native_build_fingerprint.py b/tools/gpu-aot/test_native_build_fingerprint.py index d1cbffd..ba93246 100644 --- a/tools/gpu-aot/test_native_build_fingerprint.py +++ b/tools/gpu-aot/test_native_build_fingerprint.py @@ -37,7 +37,7 @@ def main(): if "*" in relative: continue path = (source / relative).resolve() - if path in input_files or path.suffix not in (".cpp", ".h", ".hlsl", ".hlsli", ".in"): + if path in input_files or path.suffix not in (".cpp", ".h", ".hlsl", ".hlsli", ".in", ".py", ".patch"): continue path.parent.mkdir(parents=True, exist_ok=True) path.write_text("// synthetic fingerprint input\n") diff --git a/tools/gpu-aot/test_plume_state_bindings.py b/tools/gpu-aot/test_plume_state_bindings.py new file mode 100644 index 0000000..f86b39b --- /dev/null +++ b/tools/gpu-aot/test_plume_state_bindings.py @@ -0,0 +1,268 @@ +"""Compare actual Plume binding bodies through an ordered fake D3D12 recorder.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess + + +def function(source, signature): + begin = source.index(signature) + end = source.index('{', begin) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[begin:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--compiler', default='clang++') + parser.add_argument('--negative-control', choices=('graphics-key', 'reset', 'external', 'metadata')) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + root = Path(__file__).resolve().parents[2] + source = (root/'thirdparty/plume/plume_d3d12.cpp').read_text() + header = (root/'thirdparty/plume/plume_d3d12.h').read_text() + signatures = ('static bool D3D12StateCacheEnabled(', + 'static void D3D12ReportStateCacheConfiguration(', + 'void D3D12CommandList::begin(', 'void D3D12CommandList::end(', + 'void D3D12CommandList::setPipeline(', + 'void D3D12CommandList::setGraphicsPipelineLayout(', + 'void D3D12CommandList::setComputePipelineLayout(', + 'void D3D12CommandList::setRaytracingPipelineLayout(', + 'void D3D12CommandList::notifyCommandStateWasChangedExternally(', + 'void D3D12CommandList::notifyDescriptorHeapWasChangedExternally(', + 'void D3D12CommandList::checkTopology(', + 'void D3D12CommandList::checkStencilRef(', + 'void D3D12CommandList::setRootDescriptor(', + 'void D3D12CommandList::setGraphicsRootDescriptor(') + bodies = '\n'.join(function(source, signature) for signature in signatures) + if args.negative_control == 'graphics-key': + old = ('!cacheEnabled || !cachedPipelineValid || cachedPipelineType != pipelineType ||\n' + ' cachedPipelineObject != graphicsPipeline->d3d') + assert bodies.count(old) == 1 + bodies = bodies.replace(old, '!cacheEnabled || activeGraphicsPipeline != graphicsPipeline') + elif args.negative_control == 'reset': + old = ' notifyCommandStateWasChangedExternally();' + assert bodies.count(old) == 2 + bodies = bodies.replace(old, '') + elif args.negative_control == 'external': + old = function(source, 'void D3D12CommandList::notifyCommandStateWasChangedExternally(') + bodies = bodies.replace(old, 'void D3D12CommandList::notifyCommandStateWasChangedExternally() {}') + elif args.negative_control == 'metadata': + old = 'activeGraphicsPipelineLayout = interfacePipelineLayout;' + assert bodies.count(old) == 1 + bodies = bodies.replace(old, 'if (!activeGraphicsPipelineLayout || activeGraphicsPipelineLayout->rootSignature != interfacePipelineLayout->rootSignature) '+old) + begin = header.index(' ID3D12GraphicsCommandList *d3d =', header.index('struct D3D12CommandList :')) + end = header.index(' D3D12CommandList(D3D12CommandQueue *queue)', begin) + fields = header[begin:end] + code = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +void Check(bool value,const char* name){if(!value){std::cerr< graphics_args{},compute_args{}; + uint32_t pso_calls=0,graphics_root_calls=0,compute_root_calls=0,cbv_calls=0; + void Reset(void*,void*){pipeline_id=pipeline_kind=0;graphics_root=compute_root=nullptr; + graphics_args={};compute_args={};topology=stencil=0;} + void Close(){} + void SetPipelineState(NativePso* p){++pso_calls;pipeline_id=p?p->id:0;pipeline_kind=p?p->kind:0;} + void SetPipelineState1(ID3D12StateObject* p){++pso_calls;pipeline_id=p?p->id:0;pipeline_kind=3;} + void SetGraphicsRootSignature(ID3D12RootSignature* root){++graphics_root_calls; + if(graphics_root!=root)graphics_args={};graphics_root=root;} + void SetComputeRootSignature(ID3D12RootSignature* root){++compute_root_calls; + if(compute_root!=root)compute_args={};compute_root=root;} + void SetGraphicsRootConstantBufferView(uint32_t index,uint64_t value){++cbv_calls;graphics_args.at(index)=value;} + void SetGraphicsRootShaderResourceView(uint32_t index,uint64_t value){graphics_args.at(index)=value;} + void SetGraphicsRootUnorderedAccessView(uint32_t index,uint64_t value){graphics_args.at(index)=value;} + void SetComputeRootConstantBufferView(uint32_t index,uint64_t value){compute_args.at(index)=value;} + void SetComputeRootShaderResourceView(uint32_t index,uint64_t value){compute_args.at(index)=value;} + void SetComputeRootUnorderedAccessView(uint32_t index,uint64_t value){compute_args.at(index)=value;} + void IASetPrimitiveTopology(uint32_t value){topology=value;} + void OMSetStencilRef(uint32_t value){stencil=value;} +}; +using ID3D12GraphicsCommandList=FakeD3D;using ID3D12GraphicsCommandList1=FakeD3D; +using ID3D12GraphicsCommandList4=FakeD3D; +struct ID3D12CommandAllocator {void Reset(){}}; +struct NativeBuffer {uint64_t GetGPUVirtualAddress()const{return 0x10000;}}; +namespace plume { +struct RenderPipeline{};struct RenderPipelineLayout{};struct RenderCommandList{}; +struct RenderBuffer{};struct RenderBufferReference{RenderBuffer* ref;uint64_t offset;}; +enum class RenderRootDescriptorType {CONSTANT_BUFFER,SHADER_RESOURCE,UNORDERED_ACCESS}; +struct D3D12Pipeline : RenderPipeline {enum class Type:uint32_t{Compute=1,Graphics=2,Raytracing=3};Type type;}; +struct D3D12ComputePipeline:D3D12Pipeline{NativePso* d3d;D3D12ComputePipeline(NativePso* p){type=Type::Compute;d3d=p;}}; +struct D3D12GraphicsPipeline:D3D12Pipeline{NativePso* d3d;uint32_t topology,stencilRef; + D3D12GraphicsPipeline(NativePso* p,uint32_t t,uint32_t s){type=Type::Graphics;d3d=p;topology=t;stencilRef=s;}}; +struct D3D12RaytracingPipeline:D3D12Pipeline{ID3D12StateObject* stateObject; + D3D12RaytracingPipeline(ID3D12StateObject* p){type=Type::Raytracing;stateObject=p;}}; +struct D3D12PipelineLayout:RenderPipelineLayout{ + ID3D12RootSignature* rootSignature;std::vector> rootDescriptorRootIndicesAndTypes; + D3D12PipelineLayout(ID3D12RootSignature* r,uint32_t index=0):rootSignature(r){rootDescriptorRootIndicesAndTypes.push_back({index,RenderRootDescriptorType::CONSTANT_BUFFER});} +}; +struct D3D12Buffer:RenderBuffer{NativeBuffer* d3d;}; +struct D3D12CommandQueue{};struct D3D12Framebuffer{}; +struct D3D12CommandList:RenderCommandList { +''' + fields + r''' + void begin();void end();void setPipeline(const RenderPipeline*); + void setGraphicsPipelineLayout(const RenderPipelineLayout*); + void setComputePipelineLayout(const RenderPipelineLayout*); + void setRaytracingPipelineLayout(const RenderPipelineLayout*); + void notifyCommandStateWasChangedExternally();void notifyDescriptorHeapWasChangedExternally(); + void checkTopology();void checkStencilRef();void resetSamplePositions(){activeSamplePositions=false;} + void setRootDescriptor(const D3D12PipelineLayout*,RenderBufferReference,uint32_t,bool); + void setGraphicsRootDescriptor(RenderBufferReference,uint32_t); +}; +''' + bodies + r''' +} +int main(int argc,char** argv){ + Check(argc==2,"expected enabled/disabled mode");const bool enabled=std::string(argv[1])=="enabled"; + using namespace plume; + FakeD3D d3d;ID3D12CommandAllocator allocator;D3D12CommandList list; + list.d3d=list.d3dV1=list.d3dV4=&d3d;list.commandAllocator=&allocator; + NativePso gn{101,2},cn{201,1},external_native{999,2};ID3D12StateObject rn{301}; + D3D12GraphicsPipeline graphics(&gn,7,2),graphics_alias(&gn,9,3); + D3D12ComputePipeline compute(&cn);D3D12RaytracingPipeline ray(&rn); + ID3D12RootSignature gr{11},cr{21},other{31}; + D3D12PipelineLayout gl(&gr),cl(&cr),other_layout(&other),gl_alias(&gr,2); + NativeBuffer native_buffer;D3D12Buffer buffer;buffer.d3d=&native_buffer; + std::vector states; + auto snapshot=[&](const char* label){std::ostringstream out; + out<id:0)<<':'<<(d3d.compute_root?d3d.compute_root->id:0) + <<':'< pipelines{&graphics,&compute,&ray}; + const std::array ids{101,201,301},kinds{2,1,3}; + for(uint32_t i=0;i<3;++i)for(uint32_t j=0;j<3;++j){ + list.setPipeline(pipelines[i]);list.setPipeline(pipelines[j]); + Check(d3d.pipeline_id==ids[j]&&d3d.pipeline_kind==kinds[j],"pipeline_transition_state"); + list.setPipeline(&graphics); + Check(d3d.pipeline_id==101&&d3d.pipeline_kind==2,"pipeline_transition_state"); + snapshot("transition"); + } + const auto before_alias=d3d.pso_calls; + list.setPipeline(&graphics_alias);list.checkTopology();list.checkStencilRef(); + Check(list.activeGraphicsPipeline==&graphics_alias&&d3d.topology==9&&d3d.stencil==3,"graphics_alias_metadata"); + Check(d3d.pso_calls==before_alias+uint32_t(!enabled),"underlying_handle_alias_redundant"); + snapshot("alias"); + // External heap notification has no effect on root signature/PSO identity. + const auto heap_pso=d3d.pso_calls,heap_root=d3d.graphics_root_calls; + list.descriptorHeapsSet=true;list.notifyDescriptorHeapWasChangedExternally(); + Check(!list.descriptorHeapsSet,"heap_notification_preserved"); + list.setPipeline(&graphics_alias);list.setGraphicsPipelineLayout(&gl); + Check(d3d.pso_calls==heap_pso+uint32_t(!enabled)&& + d3d.graphics_root_calls==heap_root+uint32_t(!enabled),"heap_notification_orthogonal"); + // Raw native changes are allowed only with explicit invalidation before rebinding. + d3d.SetPipelineState(&external_native);d3d.SetGraphicsRootSignature(&other); + d3d.SetComputeRootSignature(&other);list.notifyCommandStateWasChangedExternally(); + list.setPipeline(&graphics_alias);list.setGraphicsPipelineLayout(&gl);list.setComputePipelineLayout(&cl); + Check(d3d.pipeline_id==101&&d3d.graphics_root==&gr&&d3d.compute_root==&cr,"external_mutation_state"); + snapshot("external"); + // End/begin Reset reuses the same object and all native handles. + list.end();list.begin();list.setPipeline(&graphics_alias); + list.setGraphicsPipelineLayout(&gl);list.setComputePipelineLayout(&cl); + Check(d3d.pipeline_id==101&&d3d.graphics_root==&gr&&d3d.compute_root==&cr,"reset_rebind_state"); + list.checkTopology();list.checkStencilRef();snapshot("reset");list.end(); + // Null native signatures remain distinguishable from unknown initial state. + D3D12PipelineLayout null_layout(nullptr); + list.begin();const auto null_before=d3d.graphics_root_calls; + list.setGraphicsPipelineLayout(&null_layout);list.setGraphicsPipelineLayout(&null_layout); + Check(d3d.graphics_root_calls==null_before+(enabled?1u:2u),"unknown_null_signature_first_emit");list.end(); + std::cout<<"{\"pso_calls\":"< Date: Wed, 7 Oct 2026 08:43:15 +0200 Subject: [PATCH 30/40] Bound memory-watch validation to the requested physical pages Replace allocation-wide region enumeration with the SDK's existing locked readability check. Preserve physical mapping, overflow checks, lock order, and invalidation behavior. Production-body equivalence tests cover page holes, protection flags, aliases, endpoints, and 2,000 random spans; the old implementation fails the counted-work regression. --- rexlego/src/gpu_native/memory_watch.cpp | 14 +- tools/gpu-aot/CMakeLists.txt | 3 + tools/gpu-aot/test_memory_watch.py | 3 + tools/gpu-aot/test_memory_watch_bounds.py | 210 ++++++++++++++++++++++ 4 files changed, 220 insertions(+), 10 deletions(-) create mode 100644 tools/gpu-aot/test_memory_watch_bounds.py diff --git a/rexlego/src/gpu_native/memory_watch.cpp b/rexlego/src/gpu_native/memory_watch.cpp index 431e976..c6e66f4 100644 --- a/rexlego/src/gpu_native/memory_watch.cpp +++ b/rexlego/src/gpu_native/memory_watch.cpp @@ -78,16 +78,10 @@ bool PhysicalSpan(rex::memory::Memory* memory, CpuMemorySpan input, CpuMemorySpa if (first == UINT32_MAX || uint64_t(first) + input.length > kPhysicalSize || physical(input.address + input.length - 1) != first + input.length - 1) return false; auto* heap = memory->GetPhysicalHeap(); - for (uint64_t at = first, end = uint64_t(first) + input.length; at < end;) { - rex::memory::HeapAllocationInfo info{}; - if (!heap->QueryRegionInfo(uint32_t(at), &info) || - !(info.state & rex::memory::kMemoryAllocationCommit) || - !(info.protect & rex::memory::kMemoryProtectRead)) return false; - const uint64_t page = heap->heap_base() + ((at - heap->heap_base()) / heap->page_size()) * heap->page_size(); - const uint64_t next = page + info.region_size; - if (next <= at) return false; - at = std::min(next, end); - } + // WatchCpuMemory holds the SDK global critical region before this heap lock. + // Validate only the requested pages; QueryRegionInfo scans to the allocation + // end even when a draw uses a small window within a large shared pool. + if (!heap->IsRangeCommittedReadable(first, input.length)) return false; output = {first, input.length}; return true; } diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index caaf989..b8a7f30 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -67,6 +67,9 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING list(APPEND PRODUCTION_CHECKS precompiled_pack) endif() endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/system/xmemory.cpp") + list(APPEND PRODUCTION_CHECKS memory_watch_bounds) + endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/shaders/bytecode/d3d12_5_1/texture_load_depth_unorm_cs.h" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/d3d12/texture_cache.cpp" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../rexglue-sdk/src/graphics/pipeline/texture/util.cpp" diff --git a/tools/gpu-aot/test_memory_watch.py b/tools/gpu-aot/test_memory_watch.py index 4a29f4d..45027f1 100644 --- a/tools/gpu-aot/test_memory_watch.py +++ b/tools/gpu-aot/test_memory_watch.py @@ -36,6 +36,9 @@ bool QueryRegionInfo(uint32_t at,HeapAllocationInfo* info) { if(at>=0x100000) return false; *info={}; return true; } + bool IsRangeCommittedReadable(uint32_t at,uint32_t size) { + return uint64_t(at)+size<=0x100000; + } uint32_t heap_base() { return 0; } uint32_t page_size() { return 4096; } }; struct Memory { diff --git a/tools/gpu-aot/test_memory_watch_bounds.py b/tools/gpu-aot/test_memory_watch_bounds.py new file mode 100644 index 0000000..457ef6a --- /dev/null +++ b/tools/gpu-aot/test_memory_watch_bounds.py @@ -0,0 +1,210 @@ +"""Execute actual PhysicalSpan and SDK range-query bodies against counted pages.""" +import argparse +import hashlib +import json +from pathlib import Path +import resource +import subprocess + + +def extract(source, signature): + start = source.index(signature) + begin = source.index('{', start) + depth = 1 + end = begin + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +OLD_VALIDATION = r''' for (uint64_t at = first, end = uint64_t(first) + input.length; at < end;) { + rex::memory::HeapAllocationInfo info{}; + if (!heap->QueryRegionInfo(uint32_t(at), &info) || + !(info.state & rex::memory::kMemoryAllocationCommit) || + !(info.protect & rex::memory::kMemoryProtectRead)) return false; + const uint64_t page = heap->heap_base() + ((at - heap->heap_base()) / heap->page_size()) * heap->page_size(); + const uint64_t next = page + info.region_size; + if (next <= at) return false; + at = std::min(next, end); + }''' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('output', type=Path) + ap.add_argument('--compiler', default='clang++') + ap.add_argument('--verify-regression', action='store_true') + args = ap.parse_args() + root = Path(__file__).resolve().parents[2] + watch_path = root/'rexlego/src/gpu_native/memory_watch.cpp' + sdk_path = root/'rexglue-sdk/src/system/xmemory.cpp' + watch, sdk = watch_path.read_text(), sdk_path.read_text() + span = extract(watch, 'bool PhysicalSpan(') + query = extract(sdk, 'bool BaseHeap::QueryRegionInfo(') + bounded = extract(sdk, 'bool BaseHeap::IsRangeCommittedReadable(') + replacement = ' if (!heap->IsRangeCommittedReadable(first, input.length)) return false;' + assert span.count(replacement) == 1, 'Expected the bounded production validation' + legacy = span.replace(replacement, OLD_VALIDATION).replace('bool PhysicalSpan(', 'bool LegacyPhysicalSpan(', 1) + # Instrument only query call counts; page reads are counted by the model's + # operator[], so the production loop and all its decisions remain intact. + query = query.replace('{', '{\n ++queries;', 1) + bounded = bounded.replace('{', '{\n ++bounded_calls;', 1) + model = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#define REXSYS_ERROR(...) ((void)0) +namespace rex::memory { +constexpr uint32_t kMemoryAllocationCommit=2, kMemoryProtectRead=1; +struct HeapAllocationInfo { + uint32_t base_address,allocation_base,allocation_protect,allocation_size, + region_size,state,protect; +}; +struct Entry { uint32_t base_address,allocation_protect,region_page_count,state,current_protect; }; +struct Table { + std::vector entries=std::vector(131072); + uint64_t reads=0; + size_t size() const { return entries.size(); } + Entry operator[](size_t index) { ++reads; return entries.at(index); } +}; +struct BaseHeap { + uint32_t heap_base_=0,heap_size_=0x20000000,page_size_=4096,page_size_shift_=12; + std::recursive_mutex heap_mutex_; + Table page_table_; + uint64_t queries=0,bounded_calls=0; + uint32_t heap_base() const { return heap_base_; } + uint32_t page_size() const { return page_size_; } + bool QueryRegionInfo(uint32_t,HeapAllocationInfo*); + bool IsRangeCommittedReadable(uint32_t,uint32_t); + void Fill(uint32_t state=2,uint32_t protection=1) { + for(auto& entry:page_table_.entries) entry={0,protection,131072,state,protection}; + } + void ResetCount() { page_table_.reads=queries=bounded_calls=0; } +}; +struct Memory { + BaseHeap heap; + bool discontinuous=false; + uint32_t bad_endpoint=UINT32_MAX; + uint32_t GetPhysicalAddress(uint32_t address) { + if(bad_endpoint!=UINT32_MAX && address==bad_endpoint) return UINT32_MAX; + if(address>=0xA0000000) { + auto result=address&0x1fffffff; + return discontinuous && (result&0xfff) ? result+4096 : result; + } + return UINT32_MAX; + } + BaseHeap* GetPhysicalHeap() { return &heap; } +}; +} +namespace legodimensions::gpu_native { +constexpr uint32_t kPhysicalSize=0x20000000; +struct CpuMemorySpan { uint32_t address=0,length=0; bool operator==(const CpuMemorySpan&) const=default; }; +''' + checks = r''' +using namespace legodimensions::gpu_native; +void Same(rex::memory::Memory& memory,uint32_t address,uint32_t length,bool expected) { + CpuMemorySpan input{address,length},old_output{0xdddddddd,0xeeeeeeee},new_output=old_output; + bool old=LegacyPhysicalSpan(&memory,input,old_output); + bool next=PhysicalSpan(&memory,input,new_output); + if(old!=next || next!=expected || !(old_output==new_output)) + std::cerr << "span " << std::hex << address << " length " << length + << " legacy " << old << " bounded " << next << " expected " << expected << "\n"; + assert(old==next && next==expected && old_output==new_output); + if(next) assert(new_output.length==length); +} +int main() { + rex::memory::Memory memory; + memory.heap.Fill(); + for(uint32_t alias:{0u,0xA0000000u,0xC0000000u,0xE0000000u}) { + Same(memory,alias+4097,8192,true); // Unaligned, three pages. + Same(memory,alias+0x1ffffff0,16,true); // Last physical byte, no overflow. + } + Same(memory,0,0,false); + Same(memory,0x70001000,4,false); // Not physically mapped. + Same(memory,0xfffffff0,32,false); // Guest address wraps. + Same(memory,0x1ffffff0,32,false); // Physical span wraps out of backing. + memory.discontinuous=true; + Same(memory,0xA0001000,4096,false); // Endpoint mapping must be contiguous. + memory.discontinuous=false; + memory.bad_endpoint=0xA0002fff; + Same(memory,0xA0001000,8192,false); + memory.bad_endpoint=UINT32_MAX; + for(uint32_t state:{0u,1u,2u,3u}) { + for(uint32_t protect:{0u,1u,2u,3u,4u,5u,8u,9u}) { + // Noaccess/write-only/guard-bit variants follow the actual SDK's bit + // predicate; guard plus read remains readable, rather than guessing. + memory.heap.Fill(); + auto& hole=memory.heap.page_table_.entries[2]; + hole.state=state;hole.current_protect=protect; + Same(memory,0xA0001000,8192,(state&2)&&(protect&1)); + Same(memory,0xA0001000,4096,true); // Hole outside requested span. + } + } + memory.heap.Fill(); + // Adjacent allocations/protection regions remain valid if all used pages read. + for(uint32_t i=8;i<16;++i) memory.heap.page_table_.entries[i]={8,3,8,3,3}; + Same(memory,0xC0007001,8*4096,true); + std::mt19937 random(0x624); + for(unsigned round=0;round<50;++round) { + memory.heap.Fill(); + for(unsigned j=0;j<20;++j) { + auto& e=memory.heap.page_table_.entries[random()%128]; + e.state=random()%4;e.current_protect=random()%16; + } + for(unsigned j=0;j<40;++j) { + uint32_t offset=random()%(128*4096),length=1+random()%32768; + bool expected=true; + for(uint32_t page=offset/4096;page<=(offset+length-1)/4096;++page) { + auto e=memory.heap.page_table_.entries.at(page); + if(!(e.state&2)||!(e.current_protect&1)) expected=false; + } + Same(memory,0xA0000000+offset,length,expected); + } + } + memory.heap.Fill(); + CpuMemorySpan output; + memory.heap.ResetCount(); + assert(LegacyPhysicalSpan(&memory,{0xA0001000,16},output)); + auto legacy_reads=memory.heap.page_table_.reads; + assert(memory.heap.queries==1 && legacy_reads==131072); + memory.heap.ResetCount(); + assert(PhysicalSpan(&memory,{0xA0001000,16},output)); + assert(memory.heap.queries==0 && memory.heap.bounded_calls==1 && memory.heap.page_table_.reads==1); + std::cout << "PASS equivalence: holes/protections/aliases/overflow/random; page_reads " + << legacy_reads << " -> " << memory.heap.page_table_.reads << "\n"; +} +''' + source = model + span + '\n' + legacy + '\n}\nnamespace rex::memory {\n' + query + '\n' + bounded + '\n}\n' + checks + args.output.mkdir(parents=True, exist_ok=True) + cpp, exe = args.output/'memory-watch-bounds.cpp', args.output/'memory-watch-bounds.exe' + cpp.write_text(source) + subprocess.run([args.compiler,'-std=c++20','-O0','-pthread',str(cpp),'-o',str(exe)],check=True,timeout=45) + def no_core(): resource.setrlimit(resource.RLIMIT_CORE,(0,0)) + result=subprocess.run([str(exe.resolve())],capture_output=True,text=True,check=True,timeout=20,preexec_fn=no_core) + print(result.stdout.strip()) + negative=False + if args.verify_regression: + control = source.replace(replacement,OLD_VALIDATION,1) + negative_cpp=args.output/'memory-watch-bounds-original.cpp' + negative_exe=args.output/'memory-watch-bounds-original.exe' + negative_cpp.write_text(control) + subprocess.run([args.compiler,'-std=c++20','-O0','-pthread',str(negative_cpp),'-o',str(negative_exe)],check=True,timeout=45) + bad=subprocess.run([str(negative_exe.resolve())],capture_output=True,text=True,timeout=20,preexec_fn=no_core) + assert bad.returncode != 0 and 'memory.heap.queries==0' in bad.stderr,bad.stderr + negative=True + (args.output/'verification.json').write_text(json.dumps(dict(status='passed', + actual_production_physical_span=True,actual_sdk_query_and_bounded_bodies=True, + modeled_page_table_and_mapping=True,negative_original_validation_rejected=negative, + legacy_page_reads=131072,bounded_page_reads=1, + source_sha256={str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in (watch_path,sdk_path)}, + scope='Asset-free predicate/scaling proof; actual callback/write invalidation covered separately. No runtime speed or driver claim.'),indent=2)+'\n') + + +if __name__ == '__main__': + main() From 62473118c11bf44b0e2da3c5a1eca6f715e709c7 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:50:45 +0200 Subject: [PATCH 31/40] Pin tested SDK correctness fixes and explicit interior body owners Use the published SDK runtime, codegen, platform, and verified SIMDe preparation changes. Map five callable interior addresses to their actual body owners so generated entry points preserve backward branches, live registers, and stack epilogues. Fresh codegen and consumer builds remain the next integration gate. --- rexglue-sdk | 2 +- rexlego/legodimensions_config.toml | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/rexglue-sdk b/rexglue-sdk index 0541b80..b71efb4 160000 --- a/rexglue-sdk +++ b/rexglue-sdk @@ -1 +1 @@ -Subproject commit 0541b80c9bbc3b96a4dd6f5783638eea19917e1b +Subproject commit b71efb4009e516eba2712ccc149a13ac78227bfb diff --git a/rexlego/legodimensions_config.toml b/rexlego/legodimensions_config.toml index 46de221..7db51af 100644 --- a/rexlego/legodimensions_config.toml +++ b/rexlego/legodimensions_config.toml @@ -22,17 +22,19 @@ # Same fix as the analysis-time unresolved calls: declare it as an entry point. [functions] +# These addresses are callable interior entries. Their body owners preserve +# backward branches, live registers, and the owner's stack epilogue. 0x82D73A10 = {} -0x8307CA78 = {} +0x8307CA78 = { body = 0x8307C528 } 0x83803F18 = {} 0x82888768 = {} -0x838052D8 = {} +0x838052D8 = { body = 0x838051D0 } 0x8381BB80 = {} -0x839AAE60 = {} +0x839AAE60 = { body = 0x839AACC0 } 0x82B68438 = {} 0x83283438 = {} 0x82B6B878 = {} -0x82FD1520 = {} +0x82FD1520 = { body = 0x82FD1470 } 0x82B63298 = {} 0x8361E6E8 = {} 0x837FE708 = {} @@ -40,7 +42,7 @@ 0x8379EEF8 = {} 0x8379EF10 = {} 0x8379E7D0 = {} -0x82D41F28 = {} +0x82D41F28 = { body = 0x82D416A0 } 0x832A2F88 = {} 0x8361E6F8 = {} 0x8361E6D8 = {} From 7148d99107a087b1b64af60442c336f5601de6ff Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:53:02 +0200 Subject: [PATCH 32/40] Record controlled renderer evidence and robust viewport source validation Locate actual generated viewport definitions across codegen repartitioning, keep assertions active, and record source hashes. Correct the historical TU23 alpha-reference identification and preserve dated renderer experiment outcomes, SDK publication limits, and the new measured-development approach. --- tools/gpu-aot/COLLABORATION.md | 491 +++++++++++++++++++++++- tools/gpu-aot/test_viewport_state.py | 32 +- tools/gpu-aot/tiling-answer-20260930.md | 34 +- 3 files changed, 535 insertions(+), 22 deletions(-) diff --git a/tools/gpu-aot/COLLABORATION.md b/tools/gpu-aot/COLLABORATION.md index 19948c6..d7ea9ce 100644 --- a/tools/gpu-aot/COLLABORATION.md +++ b/tools/gpu-aot/COLLABORATION.md @@ -8,8 +8,19 @@ development handoff, not a completed renderer or a release. - `rexlego/CMakeLists.txt` explicitly rejects `LEGODIMENSIONS_NATIVE_GPU=ON` outside Windows. The native implementation includes Win32/D3D12 APIs. No - Linux native renderer, Wine, or cross-compilation run was validated here. -- The current compiler-preparation CMake uses Windows DXC `.lib`/`.dll` files, + Linux native renderer exists. On October6, Clang22 cross-compilation with an + xwin Windows SDK linked the modified SDK runtime and native prelink tools; + the complete game build subsequently linked with direct COFF shader-bank + embedding. The first local native launch selected that renderer, initialized + D3D12 and recorded draw/present activity, but its window remained black. + Later runs verified Vorton arrival, save reload and character actions, as + recorded below. Full campaign validation remains outstanding. + The published v0.1.34 build ran through Proton with visible first-run setup, + then reproduced the reported owner-thread-0x17 GPU lock wait. Dump analysis + found that owner polling GPU completion while a pipeline worker was inside + `vkCreateGraphicsPipelines`; the command pointer continued advancing. This + does not establish a leaked lock or justify bypassing synchronization. +- The tracked compiler-preparation CMake uses Windows DXC `.lib`/`.dll` files, MSVC runtime options, and Windows dependency output names. The runtime probe scripts are PowerShell/Windows tools. They are not Linux build instructions. - A clean clone lacks `rexlego/generated/rexglue.cmake`, translated game C++, @@ -31,10 +42,94 @@ ctest --test-dir native-checks --output-on-failure ``` Requirements for this target: Git, CMake >=3.25 and a C++20 compiler. -It registers 19 tests. Release assertions stay enabled. The workflow is +It registers 20 C++ helper tests. With Python3 and a GNU-style compiler driver +(GCC or clang++), it also registers 27 production-body fixtures, for 47 +checks total. These compile real renderer functions against synthetic guest +heaps and fences. They cover shader read bounds, texture residency, submission +wait failures, transactional frame-ring creation, indexed draw bounds, texture +upload failure propagation, query retirement, completion callbacks and failed +native pipeline handles. Failed resolve pipelines are discarded so later calls +can retry; rejected handles cannot reach binding or draw operations. The build +fixture verifies fresh shader-bank generation and build fingerprint dependencies. +Python fixtures use +the compiler selected by CMake; generated fixtures stay in the build directory. +With Plume initialized, an additional production factory fixture checks command +allocation failures. Unix hosts with the SDK template also register a real +cross-build producer fixture. With dotnet available, an optional extractor +path-containment fixture checks 27 archive paths. With Clang and the host LLVM +tools, an additional `binary_cache_build` fixture checks the actual CMake +COFF-bank route, archive consumption and rebuild dependencies. This configured +Linux checkout therefore has 51 checks. Run the preparer below before those checks. +Release assertions stay enabled. The workflow is `.github/workflows/native-gpu-checks.yml`; consult its actual run result before reporting Linux validation. +The October6 safety follow-up passed all 29 then-current checks on Fedora +with Clang22.1.8 and GCC16.2.1. The expanded configured Clang suite +subsequently passed all 33 checks, then all 34 after adding the pipeline failure +fixture. Its texture residency and submission-wait regressions rejected +the original renderer paths; the original shader parser crossed an inaccessible +guard page in the bounded fixture. The fixes validate shader reads against +guest allocation/protection boundaries, reject unreadable 2D/cube texture +sources, and require successful D3D12 fence completion before retiring resources +or reusing a frame allocator. It also validates indexed draw intervals before +recording commands, aborts draws after failed texture uploads, and stages new +frame rings before publishing any replacement resources. These are host safety +checks, not evidence that +the reported in-game hangs or rendering artifacts are fixed. A full Windows +SDK runtime link and prelink-tool build subsequently passed. All 35 remaining +host/native translation units also passed isolated COFF compilation with the +actual Ninja Windows flags after fixing public ImGui header propagation and +the native RenderDoc header dependency. This does not establish a complete +game link or a playable scene. + +The complete linked bank exposed a compiler memory problem: its 84 MB numeric +byte initializer exceeded 8 GB of Clang memory and drove this 16 GB machine +into heavy swapping; the build was interrupted. +The prelink emitter now uses adjacent escaped byte strings for Clang and GCC, +with the prior numeric representation retained for MSVC's string-size limit. +The explicit compressed size excludes the string initializer's terminal NUL. +Both representations passed exact byte checks and miniz round trips; an actual +small Windows COFF compile also passed strict overlength warnings. At full +bank size, Clang still expanded the string initializer into per-byte constant +values and failed safely under a 3 GB address-space limit. Escaped literals +therefore did not solve the complete-bank compilation problem. + +`tools/gpu-aot/embed_linked_bank.py` now streams the unchanged compressed +payload into a binary file and keeps the original shader tables and size +constants in a small C++ file. Host `llvm-mc` embeds the binary directly into +a read-only Windows COFF section with the existing MSVC C++ array symbol; +the C++ compiler no longer sees the payload initializer. The optional +`LEGO_NATIVE_BINARY_CACHE` route supports Windows Clang with the MSVC ABI on +x64 or ARM64, requires Python3 and host `llvm-mc`, and defaults on when those +requirements are met. Other configurations retain C++ embedding. Small +x64/ARM64 checks passed. The full actual x64 native game build then passed +with one worker and compiler/linker address-space caps of 3/4 GiB. The +measured peak RSS of the build's process tree was 710 MiB. This is build +memory use, not a runtime memory or performance measurement. +Streaming comparison also verified that the complete bank's numeric and +escaped representations have identical payload bytes, tables and metadata. +The focused emission and worker fixtures passed after the earlier complete +34-check run. The complete game link passed; native gameplay remains unverified. +The expanded configured suite subsequently passed all 36 checks in 33.26 +seconds, including `binary_cache_build`. + +The Plume dependency uses public commit +`e0c8871b930dc1a6544bf457f2348c456b9d7e13`; the previously recorded +`ceaeeee0` could not be fetched from its configured public remote. Before a +native renderer build, apply the tracked compatibility patch: + +```sh +git submodule update --init thirdparty/plume +python tools/gpu-aot/prepare_plume.py +python tools/gpu-aot/prepare_plume.py --check +``` + +This preserves the target-value fence wait, failed-resize recovery and command +allocation failure propagation needed +by this renderer. The pin-checked preparer refuses conflicting local edits. +The patch lives at `tools/gpu-aot/patches/plume-lego.patch`. + The SDK gitlink is `17e3675767067339581018c7e722e1d8d65de201`, published on the fork's `codex/native-renderer` branch. Its dependency draft is https://github.com/NeverCookFirst/rexglue-sdk/pull/1. Use the gitlink with @@ -60,7 +155,80 @@ viewport and gradient preparations in that order. It requires a fresh output directory, does not modify the compiler checkout, and does not install banks. The resulting ten C++/header files were compared with the October4 installed compiler source and matched after newline normalization. Shared constants -remain 624 bytes. This source comparison is not a Linux compiler-build test. +remain 624 bytes. The reconstructed compiler subsequently built on Linux against official DXC +1.9.2609. All 9,601 unique shaders extracted from the complete base disc and +TU23 compiled to DXIL; this is offline shader coverage, not rendering proof. +Prelink DXC must match the compiler version: the older pinned DXC failed to +link these newer libraries. Matching Windows DXC then prelinked 19,064 +variants from 9,532 specialization shaders, with 19,124 microcode records. +The local runtime bootstrap contains one unchanged duplicate static shader +and adds no captured-runtime coverage. Native launch must capture genuine +missing runtime shaders before that coverage can be claimed. + +The complete disc, TU23 and all 30 DLC packages subsequently produced an +expanded candidate with 12,141 unique compiled shaders, 12,064 specialization +shaders, 24,128 linked variants and 24,194 microcode records. Every identity +from the earlier bank was retained. All 12,141 HLSL dumps were verified and +the production prelinker's deflate round trip passed. One DLC archive has no +shader candidates; its 8,422 named entries were still parsed successfully. +These are offline validation results; native gameplay remains unverified. + +The first local native launch captured 33 complete missing shader containers. +Their section and microcode bounds were validated separately from the raw +placement fragments. One runtime vertex container has valid flags +`0x102A1111`; its low-byte flags are accepted by the actual compiler, so the +capture helper was adjusted to match that parser. The compiler rejected this +container's memexport operation. The renderer already handles the observed +copy through CPU emulation; the compiler rejection alone does not establish +that a general GPU memexport path is needed. A fresh diagnostic candidate +explicitly excluded that named container with its hash and reason, compiled +all 32 included containers, and prelinked 44 variants. The next native run +showed a purple brick landscape and cloudy sky after confirm input, then +returned to black output after scene progression. That run captured 36 +additional complete containers. The expanded candidate compiled all 68 +included shaders and prelinked 94 variants from 47 specialization shaders; +its union index contains 24,316 records. Every prior runtime shader hash and +index row was retained. Those source artifacts were installed with rollback +copies, and the rebuilt native app reached the opening cinematic and title. +Seven further menu captures produced a strict 75-shader candidate with 106 +linked variants. Its rebuilt app advanced through safe-zone settings, DLC +management and New Game, then wrote fresh Game 1 save files. Save reload and +character movement have not yet been verified. Missing controller icons and +broken-looking dark geometry remain visible; cinematic output is not renderer +correctness or playable-game proof. + +The menu compiler also encountered four valid containers without an optional +CTAB reflection table. The tracked `xenosrecomp-empty-reflection.patch` selects +a static empty table instead of aliasing the container header or failing an +assertion. Four absent-table fixtures and one present-table fixture compiled +successfully, with byte-identical outputs to the previous release compiler. +`prepare_native_compiler.py` applies this patch to the reconstructed source. + +An audited native run completed 3.3 million unchanged-texture checks and +2.98 million unchanged-buffer checks with zero mismatches before shutdown. +These checks still hash every audited cache hit and do not measure the fast +path's performance. That run captured three more complete shaders during New +Game progression. All 78 included shaders compiled and prelinked successfully; +the installed source candidate retains every previous identity and the same +explicit startup-copy exclusion. Its native rebuild passed at 759 MiB peak +build-tree RSS. The rebuilt game loaded Game 1 and accepted Batman, Gandalf +and Wyldstyle on the requested pad regions, progressing through the Gotham, +Middle-earth and LEGO Movie cinematics toward Vorton. Missing scene shaders, +dark output and high frame times remain; character movement is unverified. +No later save checkpoint was observed. That capture run produced 111 complete +containers. All 110 included shaders compiled, with 172 linked variants and +24,395 union-index records; the native rebuild passed at 754 MiB. The next +controlled run enables the existing stencil candidate and captures late UI +draws. Coverage remains partial. The failed 33-container candidate is preserved. + +CPU-only draw diagnostics can be triggered with +`LEGO_NATIVE_DRAW_TRACE_TRIGGER`, without GPU readback. An empty trigger selects +the first 64 draw attempts in the next frame; `skip=160` selects later draws. +Output is limited to one frame and 1 MiB. Reads of device fields, declarations, +indices and vertex samples validate committed readable guest spans. An optional +`LEGO_DUMP_TEXTURE_UPLOADS_PIXEL_SHADER` accepts exactly 16 hexadecimal digits +and restricts the existing bounded texture-upload dump to that shader, allowing +UI atlases to be inspected without filling the quota with background textures. For Windows compiler builds, `prepare_alpha_candidate.py` currently expects the pinned compiler's dependencies already built in `out/build/lego-release`: @@ -77,6 +245,15 @@ extractor/index/prelink commands and input layouts are described in README.md. ## Local inputs and collaboration +The detached native app currently has no `OnCreateImmediateDrawer` override. +The SDK consequently does not initialize host overlays in this route: settings, +console, achievements, mods, cheats, and the host Credits/Quit prompts are +unavailable. Use `main_menu_extras = false` to keep those prompt-dependent entries +out of the guest main menu. Guest title/story rendering and controller input +remain separate; Alt+Enter and the window close action still work. +SDK message boxes automatically choose the focused button without a drawer, +and SDK keyboard prompts return default text instead of displaying a prompt. + Git contains project-owned source, tools, patches and investigation notes. It does not contain game executables/data, generated game C++, shader containers/DXIL/HLSL banks, saves, captures, or the Avengers reference install. @@ -115,3 +292,309 @@ images still showed blank portraits and rendering artifacts. Visual checks of the current compiler/bias combination, actual internal1920x1080, stable60FPS, three story levels/cutscenes and AMD hardware remain outstanding. The latest saved owner configuration has depth of field disabled. + + +### October 7: TU23 alpha-reference repair, before live verification + +Title-menu controller glyphs use an intact BC1 atlas, but the native renderer +read the unrelated float shadow at device `+10620` (default `1.0`) as ALPHAREF. +The actual patched TU23 image proves render-state index 25 uses setter +`83FB8260`, default integer `0`, and stores a normalized float at `+10500`. Its +constant at `82005D78` is `3B808081`, float32 `1/255`; descriptor index 59 and +setter `83FB92B8` belong to the separate `+10620` state. Both shader-constant +and pipeline reads now use `+10500`, and native initialization invokes the +verified integer-reference setter. Mesh and bounded draw diagnostics agree. + +The headless loader read only selected words, without guest execution. Actual +XEX/XEXP, loader, selected-word and generated-source provenance is recorded in +`.local-testing/reports/TU23-alpha-reference-byte-proof.json`. The production +regression executes original leaf setters, native default initialization, +original gameplay alpha setup, both renderer read expressions, all compares +and opaque/transparent icon cases. Correct reads pass; an isolated fixture +using the original wrong offset compiles and fails the intended reference +assertion. Evidence: `.local-testing/alpha-reference-state/verification.json`. +The portable GPU suite passes 38/38 in 34.06 seconds. Live icon validation, +dark geometry, performance, controlled character movement and campaign +completion remain outstanding. The pre-fix viewport run reached the Gandalf +cinematic and is preserved in `native-runtime110-viewport-evidence`; it does +not establish a new story checkpoint. + +Live follow-up: the bounded native rebuild passed at 752.8 MiB peak owned-tree +RSS. After staging and relaunching with the same watch/stencil/viewport flags, +`native110-alpha-select.png` shows the previously missing green A and red B +title-menu glyphs. This verifies that specific repair. Dark planet debris +remains; the first 64 scene draws were captured separately for diagnosis. + + +### October 7: controlled culling, timing and resource profiling + +The next native build passed with 814.7 MiB peak owned-tree RSS. The +`LEGO_NATIVE_CULL` candidate honors TU23 low-three-bit face/cull state, includes +it in pipeline identity, preserves draw/query submission with empty coverage +when both polygon faces are culled, and selects the surviving stencil masks and +polygon-offset face. Actual setter/Plume/production-body fixtures pass. Live +title icons remain correct, but culling has not established a repair for dark +title debris. Five opt-in constants timing phases distinguish shadow/shared, +upload, descriptor/root binding, textures and framebuffer restoration. + +An owned five-second kernel-perf profile found framebuffer invalidation and +vertex upload costs on the renderer thread. It is confounded by CTest overlap +and verbose Proton exception/unwind tracing; it does not explain the entire +frame gap. The launch helper no longer forces verbose Proton tracing: one run +produced an 808 MiB trace. Native logs remain enabled, and +`LEGO_NATIVE_PROTON_LOG=1` restores that explicit diagnostic. + +Framebuffer invalidation now maintains a weak index of current framebuffer +owners instead of locking every sampled texture. Recording serialization +prevents creation/invalidation races; attachment matching and fence retirement +are preserved. The actual production fixture passes, including 10,000 +non-owners, all color/depth dependencies, stale generations, failure, reset, +retirement and concurrent creation. Removing the invalidation recording lock +fails the intended race assertion. The GPU suite passes 41/41 in 51.17 seconds. + +SDK TimerQueue has a private restart-only blocking-wait candidate, enabled by +`REX_TIMER_WAIT_BLOCKING` presence. Default spinning remains unchanged. The real +TimerQueue/WaitItem/Disruptor fixture covers publication races, deadlines, empty +shutdown, recurring timers, backpressure and cancellation/callback lifetimes. +The SDK production regression suite passes 10/10 in 36.18 seconds. Wakeup +lateness and actual installed candidate behavior remain unverified. + +The Vorton placement gate was resolved by moving Batman to the center, +Gandalf to the left and Wyldstyle to the right. The arrival cinematic advanced, +the player HUD appeared, and bounded keyboard input moved Batman forward. +The real save files changed immediately after placement; persistence through +a later reload remains unverified. This is the first verified native player +control, not campaign completion. The scene produced 13 additional complete +shader captures. The strict candidate compiled all 123 included containers +from 124 observed files, preserving prior coverage, with 196 linked variants +and 24,421 union-index entries. Only the previously proven CPU pool-copy +container is excluded. The candidate was promoted while the owned game and +Ninja were idle; the next app build and live staging remain pending. + +The owners/timer native build passed with 944.5 MiB peak owned-tree RSS. The +current Vorton run still used the older staged binaries. Its exact 600-frame +scope includes arrival and gameplay, with 52.79 ms median intervals and zero +resource synchronization time. That scope does not justify an ASYNC speedup +claim; buffer-window benefits need a matched gameplay measurement. Full +campaign, visual correctness and performance goals remain outstanding. + +### October 7: Vorton rebuild and failed shader bindings + +The 123-container native build passed with 744.0 MiB peak owned-tree RSS and +was staged with the newly built SDK runtime. The focused culling, diagnostic, +pipeline-failure and new shader-binding production fixtures pass (4/4). +Failed nonzero vertex or pixel bindings now preserve the requested address and +reject draws before pipeline-cache lookup; genuinely null pixel bindings still +permit depth-only passes. Removing either guard reproduces stale cached +pipeline submission in the fixture. Live Vorton reload remains pending. + +The actual TimerQueue latency fixture measured 1,000 one-millisecond recurring +callbacks per mode on both Linux and Windows/Proton. Linux blocking waits used +1.1% timer-thread CPU versus 65.0% spinning, with lower observed lateness. +Windows/Proton used 10 ms process CPU versus 540 ms for spinning, but blocking +p99 lateness was 1.15 ms versus 1.06 ms, and maximum 2.24 ms versus 1.21 ms. +No callback ran early. Windows measurements overlapped game startup/menu load; +matching MSVC/UCRT DLL hashes were verified. Zero reported blocking thread CPU +reflects accounting precision, not literal zero work. This supports testing the +opt-in, not a frame-rate or default-on claim. + +### October 7: tutorial coverage, first-use depth and capture reuse + +Native123 restored the Vorton arrival save and verified more tutorial actions: +Batman used the grapple and Batarang, Gandalf raised the stairs and removed the +crystal, and Wyldstyle climbed the bar wall and broke the object. These actions +and their screenshots are recorded in +`.local-testing/reports/campaign-e2e-progress.json`. The gateway has not been +rebuilt, no subsequent checkpoint has been verified as persisted, and Wizard +of Oz has not been entered. Campaign completion, crash-free play, visual +correctness and acceptable performance remain unverified. + +The buffer-window trial is not a valid performance comparison. Its startup +skipped the introduction earlier, and the depth atlas at `AC585174` had never +received a host resolve. The baseline had four earlier host resolves. In the +selected 300-frame scopes, the trial rejected 27,522 texture uploads at slot15; +the baseline rejected none. Attempted draw counts consequently hide different +submitted work. This establishes different depth authority and startup order, +not a speedup or regression caused by `LEGO_NATIVE_BUFFER_WINDOWS`. Evidence: +`.local-testing/reports/native123-depth-atlas-first-use-investigation.json`. + +Previously, a borrowed guest depth texture could only be sampled after a host +resolve. The renderer now uploads supported first-use `k_24_8` guest depth into +an R32_FLOAT sampling mirror, using the SDK's endian conversion, UNORM24 unpack +and intrinsic RRRR mapping. It refreshes the descriptor after successful upload, +retains host-resolve precedence, and refreshes guest content after invalidation. +Unsupported depth formats remain rejected. The production-body fixture covers +the actual 960x3840 tiled rectangle, generation changes, source bounds, +allocation/map retries and descriptor refresh. Its host retirement and +invalidation notifications are simulated boundaries; it does not prove live +image correctness. + +Missing placement-shader captures now deduplicate the validated complete +container and stage instead of the guest object address. Reusing an address +for different valid shader bytes therefore produces a new capture. Unchanged +contents remain deduplicated, malformed containers do not reserve a slot, and +the process-wide placement budget remains 256 containers. The production-body +fixture checks exact section bytes and archive markers, stage separation, +concurrent duplicate requests and the budget. Restoring address-based dedup +fails the recycled-address regression. Dump I/O is simulated in this fixture; +file persistence is not its claim. + +The Vorton tutorial candidate includes all 136 selected runtime containers: +57 vertex and 79 pixel shaders, with 111 specialization shaders, 222 linked +variants and 24,447 union-index records. It preserves every included identity +from the prior 123 candidate and adds 13. The only excluded observed container +remains the proven startup CPU pool copy; the snapshot contains 137 observed +containers. This is partial captured coverage, not complete campaign coverage. +Candidate provenance is +`.local-testing/game-launch/native-compiler/runtime-candidates/vorton-tutorial-136/provenance.json`. + +The complete native build with the depth and capture fixes passed with one +worker, 3/4 GiB compiler/linker address-space caps and 734.0 MiB peak owned-tree RSS. +Five focused production fixtures passed in 6.79 seconds: buffer allocation +diagnostics, shader binding failures, capture dedup, texture binding failures +and depth upload. Staging with `--no-seed-content` also passed, preserving the +existing game content and saves. Build evidence is +`.local-testing/reports/native-runtime136-depth-capture-build.json`; staged +binary hashes and current bank hashes are recorded in +`.local-testing/reports/native-aot-complete-manifest.json`. + +The 136 retry was launched with texture/buffer watches, timing, stencil, +viewport and culling enabled; asynchronous CPU resources, buffer windows, +blocking timer waits and verbose Proton logging are disabled. Its buffer +allocation diagnostic selects guest `849A4090`. Live136 Vorton validation +remains pending at this handoff. The successful build and staging do not +establish that the depth repair is visually correct or that later missing +campaign shaders are covered. + +### October 7: staged 152 and bounded missing-shader identities + +The 136 fast-skip run proved the first-use depth path in the running game. Guest +`AC585174` uploaded 14,745,600 bytes of tiled depth and the following slot15 +fetch used its new descriptor, with zero prior atlas resolves. The bounded +proof prefix contained no texture or submission failures; see +`.local-testing/reports/native136-depth-fastskip-live-proof.json`. Later tutorial +scenes still requested shaders absent from that bank, so this result establishes +the depth fix, rather than complete scene or campaign coverage. + +The staged 152 runtime candidate contains 65 vertex and 87 pixel containers, +125 specialization shaders, 250 linked variants and 24,479 union microcode +records. Its 153 observed containers include one explicitly excluded startup +CPU-copy shader. All earlier candidate identities and index rows remain present. +The complete candidate provenance is +`.local-testing/game-launch/native-compiler/runtime-candidates/vorton-batmobile-after136-20261007/provenance.json`. +Missing placement captures now emit their exact container and physical hashes +once per newly validated stage/container identity, before an existing dump file +can suppress writing. Hashes describe the same byte snapshot. The 256-container +budget remains in force. + +This build also includes the opt-in nonindexed buffer-window extension and the +diagnostic texture-authority signature cache. The latter preserves selected +views and emitted event payloads while skipping repeated global dedup checks. +These tests establish bounded behavior, not a gameplay speedup. The earlier +window trial used different depth authority after a different intro sequence; +it does not establish a causal performance or visual effect of the window flag. + +Four focused production fixtures passed: nonindexed buffer windows, shader +capture deduplication, first-use depth upload and texture-authority probes. +The configured portable target now has 48 checks; this focused run is not a full +48-check result. The native152 bounded build passed at 1,245,569,024 bytes peak +owned-process RSS, with three build workers and one prelink worker. Its report +is `.local-testing/reports/native152-renderer-build.json`. The staged EXE SHA256 +is `ef7f4e3b7415ffc1897cfa0b5945a5875c2780b5d03df1fe5614acbf36dab678`. + +Root launched the fresh 152 baseline with windows, asynchronous uploads and the +blocking timer adapter disabled. Its live validation is pending at this record. +Tutorial progress already includes the Batman, Gandalf and Wyldstyle subtasks +recorded in `campaign-e2e-progress.json`; neither a completed gateway checkpoint +nor the campaign is claimed here. + +### October 7: source-only epoch cache and development shader packs + +The memory-watch epoch cache passed its actual SDK-body modes and independent +review. Its evidence is `.local-testing/reports/native-memory-watch-epoch-cache.json`. +It was not part of the staged 152 performance run, so no live gain is claimed. +The original 48 portable checks passed with two compatibility adaptations; the +subsequent memory-watch, precompiled-pack and QuadList registrations bring this +checkout to 51 checks. Those additions have focused results, rather than a new +full 51-suite result. + +The optional additive shader-pack loader is disabled by default. Its actual +archive and recovery fixture passed 24 malformed packs, 672 truncated inputs, +64-pack lifetime and file-size bounds, concurrent pointer-lifetime checks, +unchanged skip-list generation refresh, failed placement/owned-resource recovery +and intentional-null bindings. A provenance-bypass negative control failed as +intended. An offline exporter compiled real generated tables and its 16-shader +152-minus136 delta passed the production parser, including real prelinked DXIL. +That smoke artifact adds no coverage to native152, which already contains those +shaders. The implementation and exporter passed independent review; no live +hot-load result is claimed. See `.local-testing/reports/native-precompiled-hotload.json`. + +The actual 152 window run later captured three new pixel containers at frame 20920. +Each exact emitted container/physical identity recomputes from its captured bytes +and matches one physical-section row in the strict155 candidate. The identity +proof is `.local-testing/reports/native152-misses-to155-index.json`. Root retains +152 as the compiled baseline for the first additive-load test. Source review and +offline compilation do not establish correct rendering in those later scenes +or completion of the gateway, first Oz checkpoint or campaign. + +### October 7: controlled experiments and reproducible SDK publication + +The current approach freezes executable/runtime/compiler identities, loads the +same private checkpoint through normal menus, and separates diagnostics from +cadence sampling. All 18 ordered additive packs passed real export/parser +checks and were accepted by both the `00dc0ee8b130f5c1` and +`8c30655312455dbb` live native builds. Acceptance is not a visual or campaign +correctness claim. + +The bounded `00dc` guest-state ledger recorded 67,842 draws over 120 contiguous +present intervals, with no dropped draws. Adjacent layout/table requests were +equal 98.70% of the time; pipeline requests were equal 34.33%. Helpers, +compute/ray work, and external command mutations are outside that ledger, so +these counts alone cannot justify suppressing backend calls. + +Plume now has an opt-in, actual-handle pipeline/root-signature cache. Tests +cover graphics/compute/ray transitions, alias metadata, recording resets, and +explicit external-state invalidation. Tables, constant addresses, barriers, +fences, and retirement are not suppressed. The public dependency pin remains +unchanged; the tracked additive patch and verified preparation step define the +overlay. Cache activation requires exactly `PLUME_D3D12_STATE_CACHE=1` and the +backend reports its actual mode once per process. + +Same-binary `8c` testing retained all four windows, including two rejected +cache-off windows with texture streaming. The accepted off/on windows measured +15.84/16.59 FPS, but mean draws also differed (578/545). That 4.75% cadence +difference is not an attributable cache gain. Defaults remain off. Evidence: +`.local-testing/reports/native-8c-state-cache-comparison.json` and +`native-8c-state-cache-decision.json`. + +The memory-watch validator now uses the SDK's existing bounded readability +predicate instead of enumerating the remainder of an allocation. Actual-body +tests match the old validity result for aliases, holes, protections, endpoints, +overflow, and 2,000 random spans. A 16-byte validation inside a 512 MiB +allocation reads one page-table entry instead of 131,072. Existing write-watch +invalidation tests also pass. This establishes less validation work, not an +observed gameplay FPS gain. + +SDK changes are published as separate local commits through `b71efb4`: atomic +GPU interrupt publication/counters, shader-cache bounds, failed-dispatch TLS, +prelaunch quit handling, vblank-before-processor shutdown, explicit interior +body entries, graph indexing, verified SIMDe preparation, platform/include +integration, and final-worker singleton lifetime. Concurrent legacy-order +controls detect the two shutdown defects. Arbitrary guest-thread stragglers +and full runtime teardown remain outside those fixtures. + +The five game `body` mappings require the new codegen feature. The old native +CLI predates its later static-library/producer rebuild, so it cannot establish +current-source generation. A fresh isolated CLI and generated-code producer, +followed by genuine game codegen and matched consumer/plugin builds, are the +next integration gate. Expected vendor overlays must remain explicit in +provenance; old frozen binaries do not become clean builds when source commits +are created. + +Native Linux/Vulkan infrastructure exists, but this native renderer still needs +window/surface handling, genuine SPIR-V archives, three-address push constants, +resource copies, dynamic state, and real query/readback support. An asset-free +GPU ABI prototype precedes any full backend port. A matched SDK PM4 route is +the planned local visual reference; it must preserve CPU-visible writes and +settle shader compilation. Neither route has completed the campaign. No level +completion or reloadable Oz checkpoint is established by this record. diff --git a/tools/gpu-aot/test_viewport_state.py b/tools/gpu-aot/test_viewport_state.py index ae5ef4a..65a426b 100644 --- a/tools/gpu-aot/test_viewport_state.py +++ b/tools/gpu-aot/test_viewport_state.py @@ -1,5 +1,6 @@ """Run TU23 viewport setter and emitted HLSL math offline, without a game.""" import argparse +import hashlib import json from pathlib import Path import subprocess @@ -15,14 +16,29 @@ def body(s, signature): return s[start:end] +def generated_definition(directory, signature): + # Code generation repartitions translation units as the function graph + # changes. Locate the actual definition, never a guessed TU number or call. + matches = [] + for path in sorted(directory.glob('*.cpp')): + with path.open() as source: + if any(line.startswith(signature + ' {') for line in source): + matches.append(path) + if len(matches) != 1: + raise RuntimeError(f'Expected one {signature} in {directory}; found {matches}') + return matches[0] + + p = argparse.ArgumentParser(description=__doc__) p.add_argument('common', type=Path) p.add_argument('output', type=Path) +p.add_argument('--compiler', default='clang++') a = p.parse_args() root = Path(__file__).resolve().parents[2] a.output.mkdir(parents=True, exist_ok=True) -original = body((root/'rexlego/generated/default/legodimensions_recomp.70.cpp').read_text(), - 'DEFINE_REX_FUNC(sub_83FB8FD0)') +signature = 'DEFINE_REX_FUNC(sub_83FB8FD0)' +setter_path = generated_definition(root/'rexlego/generated/default', signature) +original = body(setter_path.read_text(), signature) helper = body(a.common.read_text(), 'float4 LegoViewportPosition(') draw = (root/'rexlego/src/gpu_native/draw.cpp').read_text() key = body(draw, 'struct PipelineKey {')+';\n'+body(draw, 'struct PipelineKeyHash {')+';\n' @@ -33,7 +49,7 @@ def body(s, signature): #include #include #include -using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using uint=uint32_t; +using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using uint=uint32_t;using i32=int32_t; constexpr u32 kNativeRenderTargets=4,kNativeVertexStreams=16; union PPCRegister {u64 u64;int64_t s64;u32 u32;int32_t s32;}; struct CR {bool eq=false;templatevoid compare(T a,T b,int){eq=a==b;}}; @@ -81,16 +97,22 @@ def body(s, signature): assert(unchanged.xy.x==input.xy.x&&unchanged.xy.y==input.xy.y); } PipelineKey a,b;b.clip_disable=1; + assert(!(a==b));assert(PipelineKeyHash{}(a)!=PipelineKeyHash{}(b)); std::unordered_map cache; cache[a]=1;cache[b]=2;assert(cache.size()==2&&cache[a]==1&&cache[b]==2); } ''') exe = a.output/'test.exe' -subprocess.run(['clang++','-std=c++20','-O2',str(cpp),'-o',str(exe)],check=True) -subprocess.run([str(exe)],check=True) +subprocess.run([a.compiler,'-std=c++20','-O2','-UNDEBUG',str(cpp),'-o',str(exe)], + check=True,timeout=45) +subprocess.run([str(exe)],check=True,timeout=10) proof = {'passed': True, 'original_setter_cases': 16, 'pixel_mapping_cases': 900, 'modes': ['TU23 0x43F', 'TU23 0x400'], 'shared_bytes': 752 if 'fetch_lod_bias[32]' in draw else 624, 'depth_clip_pipeline_cache': True, 'game_launched': False, + 'setter_source': str(setter_path), + 'setter_body_sha256': hashlib.sha256(original.encode()).hexdigest(), + 'common_source': str(a.common.resolve()), + 'viewport_helper_sha256': hashlib.sha256(helper.encode()).hexdigest(), 'scope': 'CPU setter, exact emitted HLSL arithmetic, host viewport mapping and PSO identity; no pixel/character repair claim'} (a.output/'verification.json').write_text(json.dumps(proof,indent=2)+'\n') print('PASS: TU23 viewport setter, 900 screen coordinates, unchanged normal mode and distinct clip PSOs') diff --git a/tools/gpu-aot/tiling-answer-20260930.md b/tools/gpu-aot/tiling-answer-20260930.md index e0a1bca..9c892d3 100644 --- a/tools/gpu-aot/tiling-answer-20260930.md +++ b/tools/gpu-aot/tiling-answer-20260930.md @@ -843,15 +843,23 @@ The device shadows the register block starting at 0x2200 in order, so the offset Don't wire either of them into alpha test. ## 2. The real alpha-test leaves (RB_COLORCONTROL at +10556, dirty[16] |= 0x200) + +**Historical correction (2026-10-07):** this section originally misidentified +`83FB92B8 / +10620` as ALPHA_REF. That was an unrelated float state (descriptor +index 59), and reading its default `1.0` discarded the native title-menu button +icons. The actual TU23 alpha reference is descriptor index 25, +`83FB8260 / +10500`. The mapped, patched TU23 image verifies its setter words, +descriptor and normalization constant; see +[`TU23-alpha-reference-byte-proof.json`](../../.local-testing/reports/TU23-alpha-reference-byte-proof.json). + | Setter / getter | Word | Field | |---|---|---| | **83FB82C0 / 83FB82E0** | `rlwimi r4,r11,0,0,28` -> bits 0..2 = r4 | **ALPHA_FUNC** | | 83FB7E00 / 83FB7E28 | bit 3 = r4&1 (also dirty[16] \|= 0x40000) | **ALPHA_TEST_ENABLE** | | 83FB93B8 / 83FB93D8 | bit 4 = r4&1 | ALPHA_TO_MASK_ENABLE | | 83FB93E8 / (none found) | bits 24..31 = r4 | ALPHA_TO_MASK_OFFSET0..3 (2 bits each) | -| **83FB92B8 / 83FB92E0** | `stw r4,tmp; lfs f0,tmp; stfs f0,+10620`; dirty[24] \|= 1<<48 | **ALPHA_REF (float)** | -(The neighbouring pair 83FB9280/83FB92A8 writes the float at +10616 with dirty bit 47. It's a -separate float state, which I did not identify.) +| **83FB8260 / 83FB8298** | unsigned r4 -> float; multiply by float at `0x82005D78`; `stfs f0,+10500`; dirty[16] \|= `0x08000000` | **ALPHA_REF (integer / 255)** | +| 83FB92B8 / 83FB92E0 | `stw r4,tmp; lfs f0,tmp; stfs f0,+10620`; dirty[24] \|= 1<<48 | Unrelated float state, descriptor index 59 | Leaf contract: - The function value goes into bits 0..2 **raw**, so the enum is the hardware CompareFunction: @@ -859,21 +867,21 @@ Leaf contract: (360 D3DCMP_* use these values, not PC's 1..8). Emit all 8, not only `>=`. Xenos semantics: the pixel passes if `alpha FUNC ref` holds, so GEQUAL keeps `oC0.w >= ref` (clip when `oC0.w < ref`, which matches your current code). NEVER/ALWAYS = discard all / no-op. -- The ref leaf does **no conversion**: it copies r4's 32 bits straight into a float slot. - So what reaches the leaf is already float bits, and +10620 is the RB_ALPHA_REF shadow value itself. - Normalisation (if any) happens in the caller, not here. - **Unleashed's value/256 does not apply to this leaf.** Interpreting an integer 0..255 as float - bits would give a denormal ~0, i.e. "alpha test never discards". - Runtime check to settle it: log r4 in 83FB92B8. Plausible floats (0..1, e.g. 0x3F000000) confirm this reading. - Small ints mean a caller path I missed. +- The actual reference leaf `83FB8260` receives an **unsigned integer**, converts it to + float and multiplies by the TU23 constant at `0x82005D78`: bytes `3B 80 80 81`, + float32 `1/255`. The normalized float at `+10500` is ready for the shader comparison; + do not normalize it again. The gameplay alpha helper `82BCF720` calls this setter + with integer references. The previous float-bits/no-conversion claim applied to + `83FB92B8`, which is not the alpha-reference setter. - Compare in the shader against the value **before** the EDRAM/exp scale (`g_ColorOutputScale`). That's where your clip already sits. Xenos tests the shader's oC0.w against RB_ALPHA_REF before the RB format conversion. - The alpha-to-mask bits are a separate feature (the A2C spec bit). If bit 4 is set and native ignores it, foliage/hair-style cutouts render as solid or vanish depending on blend. -Defaults: I did not read them out of 83FC9450's descriptor table. D3D/Xenos reset values are -enable=0, func=ALWAYS(7), ref=0.0, alpha-to-mask=0. Because native skips the init, store these in -your shadow at CreateDevice. They are harmless: enable=0 means nothing is discarded until the game sets it. +Verified TU23 descriptor defaults: index 24 is enable=0, index 25 is integer ref=0, +and index 26 is func=ALWAYS(7). Native CreateDevice initializes these through the +actual leaf setters, including `83FB8260`, after validating the descriptor addresses. +This does not establish the separate alpha-to-mask state or repair other visual defects. ## 3. Tonemap PS 3A47E5DDE66B42C6: how sharp, "mip" and slot 3 combine Slots from the descriptor table: 0 `fullColor_tex`, **3 `blur_tex`**, 4 `mipColor1_tex`, 6 `noiseTex`, From 050349b3fa6e875310861175a94ca0f34d77d4b2 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:18:22 +0200 Subject: [PATCH 33/40] Pin SDK with portable NT calendar conversions and corrected unit assertions --- rexglue-sdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rexglue-sdk b/rexglue-sdk index b71efb4..5dc986f 160000 --- a/rexglue-sdk +++ b/rexglue-sdk @@ -1 +1 @@ -Subproject commit b71efb4009e516eba2712ccc149a13ac78227bfb +Subproject commit 5dc986f69c454fb474bc9472481625a04a500789 From 6815e9cc79b258e76a9ef9a54e52968d458cc206 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:36:00 +0200 Subject: [PATCH 34/40] Preserve TU23 viewport and scissor semantics and repair Vulkan readback --- .gitattributes | 3 + rexlego/CMakeLists.txt | 1 + rexlego/src/gpu_native/d3d.h | 14 +- rexlego/src/gpu_native/draw.cpp | 49 +-- rexlego/src/gpu_native/hooks_device.cpp | 4 +- rexlego/src/gpu_native/hooks_state.cpp | 52 ++- rexlego/src/gpu_native/scissor_state.h | 52 +++ tools/gpu-aot/CMakeLists.txt | 9 +- tools/gpu-aot/COLLABORATION.md | 43 ++- .../patches/plume-vulkan-readback.patch | 140 ++++++++ tools/gpu-aot/prepare_plume.py | 5 +- tools/gpu-aot/test_draw_diagnostic.py | 4 +- tools/gpu-aot/test_prepare_plume.py | 47 +++ tools/gpu-aot/test_scissor_state.py | 227 +++++++++++++ tools/gpu-aot/test_viewport_shadow.py | 307 ++++++++++++++++++ tools/gpu-aot/test_vulkan_texture_copy.py | 211 ++++++++++++ 16 files changed, 1133 insertions(+), 35 deletions(-) create mode 100644 rexlego/src/gpu_native/scissor_state.h create mode 100644 tools/gpu-aot/patches/plume-vulkan-readback.patch create mode 100644 tools/gpu-aot/test_scissor_state.py create mode 100644 tools/gpu-aot/test_viewport_shadow.py create mode 100644 tools/gpu-aot/test_vulkan_texture_copy.py diff --git a/.gitattributes b/.gitattributes index dbfab37..0955a1a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,6 @@ *.wav binary *.ttf binary *.bin binary + +# Empty unified-diff context lines contain their required leading space. +tools/gpu-aot/patches/*.patch whitespace=-blank-at-eol diff --git a/rexlego/CMakeLists.txt b/rexlego/CMakeLists.txt index 464ff3a..7eb7017 100644 --- a/rexlego/CMakeLists.txt +++ b/rexlego/CMakeLists.txt @@ -367,6 +367,7 @@ if(LEGODIMENSIONS_NATIVE_GPU) "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/plume/plume_d3d12.h" "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/prepare_plume.py" "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/patches/plume-state-cache.patch" + "${CMAKE_CURRENT_SOURCE_DIR}/../tools/gpu-aot/patches/plume-vulkan-readback.patch" "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/device.h" "${CMAKE_CURRENT_SOURCE_DIR}/src/gpu_native/renderer_route.h" diff --git a/rexlego/src/gpu_native/d3d.h b/rexlego/src/gpu_native/d3d.h index d2dcf36..14ae213 100644 --- a/rexlego/src/gpu_native/d3d.h +++ b/rexlego/src/gpu_native/d3d.h @@ -24,6 +24,18 @@ struct D3DViewport9 { }; static_assert(sizeof(D3DViewport9) == 0x18); +// TU23 83FBA710 stores numeric floats in the private device shadow. The +// public D3DVIEWPORT9 argument above keeps its unsigned integer ABI. +struct D3DViewportState { + be_f32 x; + be_f32 y; + be_f32 width; + be_f32 height; + be_f32 min_z; + be_f32 max_z; +}; +static_assert(sizeof(D3DViewportState) == 0x18); + struct D3DRect { be_i32 left; be_i32 top; @@ -134,7 +146,7 @@ struct D3DDevice { be_u32 vertex_stream_buffers[18]; u8 unknown_326C[0x3278 - 0x326C]; be_u32 textures[kNativeTextureSlots]; - D3DViewport9 viewport; + D3DViewportState viewport; be_u32 viewport_reserved; D3DRect scissor; be_u32 pixel_shader; diff --git a/rexlego/src/gpu_native/draw.cpp b/rexlego/src/gpu_native/draw.cpp index 8a94e96..058b911 100644 --- a/rexlego/src/gpu_native/draw.cpp +++ b/rexlego/src/gpu_native/draw.cpp @@ -38,6 +38,7 @@ #include "gpu_native/depth_state.h" #include "gpu_native/polygon_offset.h" #include "gpu_native/stencil_state.h" +#include "gpu_native/scissor_state.h" #include "gpu_native/device.h" #include "gpu_native/shaders.h" #include "gpu_native/state.h" @@ -662,8 +663,8 @@ struct CpuDrawDiagnostic { {"stencil", 10496}, {"alpha", 10556}, {"alpha_ref_bits", 10500}, {"legacy_float10620_bits", 10620}, {"blend", 10552}, {"color_mask", 12292}, {"clip_control", 10564}, - {"viewport_control", 10572}, {"viewport_x", 13024}, {"viewport_y", 13028}, - {"viewport_width", 13032}, {"viewport_height", 13036}, + {"viewport_control", 10572}, {"viewport_x_bits", 13024}, {"viewport_y_bits", 13028}, + {"viewport_width_bits", 13032}, {"viewport_height_bits", 13036}, {"viewport_min_z_bits", 13040}, {"viewport_max_z_bits", 13044}, {"scissor_left", 13052}, {"scissor_top", 13056}, {"scissor_right", 13060}, {"scissor_bottom", 13064}}; @@ -920,6 +921,11 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, if (!device || !count) { return false; } + if (!NativeViewportValid(device->viewport.x, device->viewport.y, + device->viewport.width, device->viewport.height, + device->viewport.min_z, device->viewport.max_z)) return false; + if (float(device->viewport.width) == 0.0f || + float(device->viewport.height) == 0.0f) return true; // Trace attempted bindings before shader/declaration/pipeline early exits. // A mask on a rejected draw is diagnostic evidence, not a submitted draw. if (PortraitProbeEnabled()) { @@ -1163,24 +1169,29 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, guest_blend_factor[2], guest_blend_factor[3]}; static_cast(commands)->d3d->OMSetBlendFactor(blend_factor); - const float viewport_width = static_cast(device->viewport.width); - const float viewport_height = static_cast(device->viewport.height); + const float viewport_width = device->viewport.width; + const float viewport_height = device->viewport.height; const bool screen_space = viewport_candidate && u32(*reinterpret_cast( reinterpret_cast(device) + 10572)) == 0x400; commands->setViewports(plume::RenderViewport( - screen_space ? 0 : static_cast(device->viewport.x), - screen_space ? 0 : static_cast(device->viewport.y), - screen_space ? float(color.width) : viewport_width > 0 ? viewport_width : float(color.width), - screen_space ? float(color.height) : viewport_height > 0 ? viewport_height : float(color.height), + screen_space ? 0.0f : float(device->viewport.x), + screen_space ? 0.0f : float(device->viewport.y), + screen_space ? float(color.width) : viewport_width, + screen_space ? float(color.height) : viewport_height, screen_space ? 0.0f : static_cast(device->viewport.min_z), screen_space ? 1.0f : static_cast(device->viewport.max_z))); - const i32 scissor_right = static_cast(device->scissor.right); - const i32 scissor_bottom = static_cast(device->scissor.bottom); + const auto effective_scissor = DecodeNativeScissor( + float(device->viewport.x), float(device->viewport.y), + float(device->viewport.width), float(device->viewport.height), + {i32(device->scissor.left), i32(device->scissor.top), + i32(device->scissor.right), i32(device->scissor.bottom)}, + u32(*reinterpret_cast(state_bytes + + kNativeScissorEnableOffset)) != 0, color.width, color.height); + const i32 scissor_right = effective_scissor.right; + const i32 scissor_bottom = effective_scissor.bottom; commands->setScissors(CullDrawScissor(plume::RenderRect( - static_cast(device->scissor.left), - static_cast(device->scissor.top), - scissor_right > 0 ? scissor_right : static_cast(color.width), - scissor_bottom > 0 ? scissor_bottom : static_cast(color.height)), + effective_scissor.left, effective_scissor.top, + scissor_right, scissor_bottom), key.raster_control, polygonal)); std::array views; @@ -1462,9 +1473,9 @@ bool DispatchDraw(D3DDevice* device, u32 primitive_type, bool indexed, "view={},{},{},{} scissor={},{},{},{} tex={:08X},{:08X},{:08X},{:08X} " "tex4-7={:08X},{:08X},{:08X},{:08X} mask={:X} blend={:08X}", key.vertex_shader, key.pixel_shader, bindings.render_targets[0], key.render_target_formats[0], - bindings.depth_stencil, key.depth_control, u32(device->viewport.x), u32(device->viewport.y), - u32(device->viewport.width), u32(device->viewport.height), i32(device->scissor.left), - i32(device->scissor.top), scissor_right, scissor_bottom, + bindings.depth_stencil, key.depth_control, float(device->viewport.x), float(device->viewport.y), + float(device->viewport.width), float(device->viewport.height), effective_scissor.left, + effective_scissor.top, scissor_right, scissor_bottom, bindings.textures[0], bindings.textures[1], bindings.textures[2], bindings.textures[3], bindings.textures[4], bindings.textures[5], bindings.textures[6], bindings.textures[7], key.color_write_mask, u32(*reinterpret_cast( @@ -1721,8 +1732,8 @@ u32 BeginTilingHook(D3DDevice* device, u32 flags, u32 count, static std::atomic logs{0}; if (logs.fetch_add(1) < 8) REXLOG_INFO("Native GPU: BeginTiling flags={:X} tiles={} host={}x{} viewport={}x{} guest_count={}", - flags, count, extent.width, extent.height, u32(device->viewport.width), - u32(device->viewport.height), u32(*reinterpret_cast( + flags, count, extent.width, extent.height, float(device->viewport.width), + float(device->viewport.height), u32(*reinterpret_cast( reinterpret_cast(device) + 13124))); // The TU23 implementation branches past its initial clear when bit 0 is set. if (flags & 1) return 0; diff --git a/rexlego/src/gpu_native/hooks_device.cpp b/rexlego/src/gpu_native/hooks_device.cpp index b1acb73..dd9ad63 100644 --- a/rexlego/src/gpu_native/hooks_device.cpp +++ b/rexlego/src/gpu_native/hooks_device.cpp @@ -154,8 +154,8 @@ u32 Direct3DCreateDeviceHook(u32 /*adapter*/, u32 /*device_type*/, u32 /*focus_w memory->SystemHeapFree(guest_address); return 0x80004005u; } - device->viewport.width = 1280; - device->viewport.height = 720; + device->viewport.width = 1280.0f; + device->viewport.height = 720.0f; device->viewport.max_z = 1.0f; device->scissor.right = 1280; device->scissor.bottom = 720; diff --git a/rexlego/src/gpu_native/hooks_state.cpp b/rexlego/src/gpu_native/hooks_state.cpp index efcdeb2..b4900c1 100644 --- a/rexlego/src/gpu_native/hooks_state.cpp +++ b/rexlego/src/gpu_native/hooks_state.cpp @@ -1,5 +1,6 @@ #include "gpu_native/renderer_route.h" #include +#include #include #include "gpu_native/d3d.h" @@ -8,6 +9,7 @@ #include "gpu_native/vertex_declarations.h" #include "gpu_native/sampler_state.h" #include "gpu_native/depth_state.h" +#include "gpu_native/scissor_state.h" #include "gpu_native/device.h" #include @@ -100,10 +102,35 @@ u32 GetDepthStencilHook(D3DDevice* /*device*/) { return surface; } -void SetViewportHook(D3DDevice* device, const D3DViewport9* viewport) { - if (device && viewport) { - device->viewport = *viewport; +bool NativeViewportRange(u32 address, u32 length, bool write) { + auto* memory = REX_KERNEL_MEMORY(); + if (!memory || !address || !length || u64(address) + length > 0x100000000ull) + return false; + auto* heap = memory->LookupHeap(address); + if (!heap || !heap->IsRangeCommittedReadable(address, length)) return false; + if (!write) return true; + const u32 page_size = heap->page_size(); + if (!page_size) return false; + for (u64 at = address; at < u64(address) + length;) { + u32 protect = 0; + if (!heap->QueryProtect(u32(at), &protect) || + !(protect & (rex::memory::kMemoryProtectWrite | + rex::memory::kMemoryProtectWriteCombine))) return false; + const u64 next = (at / page_size + 1) * page_size; + if (next <= at) return false; + at = next; } + return true; +} + +bool NativeViewportArguments(u32 device_address, u32 viewport_address, u32 stack) { + if (!NativeViewportRange(device_address, kGuestDeviceSize, true) || + !NativeViewportRange(viewport_address, sizeof(D3DViewport9), false) || + stack < 512 || !NativeViewportRange(stack - 512, 512, true)) return false; + const auto* device = REX_KERNEL_MEMORY()->TranslateVirtual(device_address); + const u32 surface = device->render_targets[0] ? u32(device->render_targets[0]) + : u32(device->depth_stencil); + return !surface || NativeViewportRange(surface, sizeof(D3DSurface), false); } void SetScissorRectHook(D3DDevice* device, const D3DRect* rect) { @@ -112,6 +139,15 @@ void SetScissorRectHook(D3DDevice* device, const D3DRect* rect) { } } +void SetScissorEnableHook(D3DDevice* device, u32 enabled) { + if (device) { + // Keep the requested rectangle intact across disable/re-enable. Native + // draws derive the effective intersection; no Xbox packet is emitted. + *reinterpret_cast(reinterpret_cast(device) + + kNativeScissorEnableOffset) = enabled; + } +} + // The native backend does not execute Xbox query predicates yet. Entering the // original XDK routine is unsafe because it dereferences private command-buffer // arrays that do not exist in our deliberately minimal guest D3DDevice. @@ -151,7 +187,15 @@ REX_HOOK_RAW(sub_83FBB110) { } REX_HOOK(sub_83FBA398, legodimensions::gpu_native::GetRenderTargetHook); REX_HOOK(sub_83FBA3E0, legodimensions::gpu_native::GetDepthStencilHook); -REX_HOOK(sub_83FBA978, legodimensions::gpu_native::SetViewportHook); +REX_HOOK_RAW(sub_83FBA978) { + auto recording = legodimensions::gpu_native::HostDevice::LockRecording(); + if (!legodimensions::gpu_native::NativeViewportArguments( + ctx.r3.u32, ctx.r4.u32, ctx.r1.u32)) return; + // Actual CPU-only TU23 conversion, surface/tile clipping and RB_VPORT + // shadows. Its scissor call takes the native hook, without Xbox packets. + __imp__sub_83FBA978(ctx, base); +} REX_HOOK(sub_83FBA068, legodimensions::gpu_native::SetScissorRectHook); +REX_HOOK(sub_83FBA968, legodimensions::gpu_native::SetScissorEnableHook); REX_HOOK(sub_83FBCBB8, legodimensions::gpu_native::SetPredicationHook); REX_HOOK(sub_83FAF5D8, legodimensions::gpu_native::SetupFormatHook); diff --git a/rexlego/src/gpu_native/scissor_state.h b/rexlego/src/gpu_native/scissor_state.h new file mode 100644 index 0000000..fc97548 --- /dev/null +++ b/rexlego/src/gpu_native/scissor_state.h @@ -0,0 +1,52 @@ +#pragma once +#include +#include +#include + +namespace legodimensions::gpu_native { +constexpr uint32_t kNativeScissorEnableOffset = 12288; +struct NativeScissorRect { + int32_t left, top, right, bottom; + bool operator==(const NativeScissorRect&) const = default; +}; + +inline bool NativeViewportValid(float x, float y, float width, float height, + float min_z, float max_z) { + // Empty TU23 clips retain their zero dimensions but submit no host draw. + return std::isfinite(x) && std::isfinite(y) && std::isfinite(width) && + std::isfinite(height) && std::isfinite(min_z) && std::isfinite(max_z) && + width >= 0.0f && height >= 0.0f && min_z >= 0.0f && min_z <= 1.0f && + max_z >= 0.0f && max_z <= 1.0f; +} + +// TU23 83FBA068 intersects the requested rectangle with the viewport only +// when 83FBA968's requested enable at +12288 is nonzero. Clamp the effective +// rectangle to host storage and collapse reversed bounds, preserving empties. +inline int32_t NativeViewportInteger(float value) { + // Match TU23 fctiwz without an out-of-range C++ float-to-integer cast. + if (std::isnan(value) || double(value) < double(INT32_MIN)) return INT32_MIN; + if (double(value) >= double(INT32_MAX)) return INT32_MAX; + return static_cast(value); +} + +inline NativeScissorRect DecodeNativeScissor(float x, float y, + float width, float height, NativeScissorRect requested, bool enabled, + uint32_t target_width, uint32_t target_height) { + int64_t left = NativeViewportInteger(x), top = NativeViewportInteger(y), + right = left + NativeViewportInteger(width), + bottom = top + NativeViewportInteger(height); + if (enabled) { + left = std::max(left, int64_t(requested.left)); + top = std::max(top, int64_t(requested.top)); + right = std::min(right, int64_t(requested.right)); + bottom = std::min(bottom, int64_t(requested.bottom)); + } + const int64_t max_x = std::min(int64_t(target_width), int64_t(INT32_MAX)); + const int64_t max_y = std::min(int64_t(target_height), int64_t(INT32_MAX)); + left = std::clamp(left, int64_t(0), max_x); + top = std::clamp(top, int64_t(0), max_y); + right = std::clamp(right, left, max_x); + bottom = std::clamp(bottom, top, max_y); + return {int32_t(left), int32_t(top), int32_t(right), int32_t(bottom)}; +} +} // namespace legodimensions::gpu_native diff --git a/tools/gpu-aot/CMakeLists.txt b/tools/gpu-aot/CMakeLists.txt index b8a7f30..eeba4eb 100644 --- a/tools/gpu-aot/CMakeLists.txt +++ b/tools/gpu-aot/CMakeLists.txt @@ -45,7 +45,7 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") if(Python3_Interpreter_FOUND) set(PRODUCTION_CHECKS draw_device_view state_call_diagnostic buffer_alloc_diagnostic - command_slots completion_callbacks constant_timing cpu_resource_wait cull_state + command_slots completion_callbacks constant_timing cpu_resource_wait cull_state scissor_state viewport_shadow draw_diagnostic draw_index_bounds draw_upload frame_cadence frame_ring_transaction framebuffer_owners memory_watch nonindexed_buffer_windows quad_buffer_windows pipeline_failures shader_binding_failures prelink_emission prelink_workers @@ -83,6 +83,13 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/plume_d3d12.cpp") list(APPEND PRODUCTION_CHECKS plume_command_creation plume_state_bindings gpu_timestamps resize_fences) endif() + # This fixture records the real Vulkan copy body without loading a + # driver. It needs the public Vulkan headers and POSIX process limits. + if(UNIX + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/plume_vulkan.cpp" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/plume/contrib/Vulkan-Headers/include/vulkan/vulkan_core.h") + list(APPEND PRODUCTION_CHECKS vulkan_texture_copy) + endif() if(UNIX) list(APPEND PRODUCTION_CHECKS prelink_publication) endif() diff --git a/tools/gpu-aot/COLLABORATION.md b/tools/gpu-aot/COLLABORATION.md index d7ea9ce..5a97a7e 100644 --- a/tools/gpu-aot/COLLABORATION.md +++ b/tools/gpu-aot/COLLABORATION.md @@ -4,6 +4,41 @@ This branch contains an experimental C++ native D3D12 renderer and its AOT translation tools. `LEGODIMENSIONS_NATIVE_GPU` defaults to `OFF`. This is a development handoff, not a completed renderer or a release. +## Local verification on October 7, 2026 + +The current source approach preserves exact TU23 CPU setters wherever they do +not emit GPU packets, then translates their private state at the draw boundary. +Public API structures and private device structures must not share a type +unless the actual guest writer proves their representations match. The +viewport audit found integer API bytes copied into a six-float private shadow; +bulk reset uses a 65,535-pixel sentinel that the original setter clips to the +active surface. The correction and scissor enable/empty-rectangle semantics +passed the full 68-check renderer suite and independent review. Older frozen +builds below predate them. + +An isolated native SDK build through local commit `5dc986f` passed its 225 +selected unit and production regressions. Four existing BitStream cases remain +skipped; the PPC execution target was not built and is explicitly excluded. +The Linux NT calendar overflow now has real kernel-body regressions and exact +negative controls. Fresh CLI SHA256 +`41c932ae475a5bf910b0b1693cc459ea530de8b920f263f41e48e1c49ed8de4f` +forced genuine game codegen with all five explicit body mappings and reproduced +identical generated bytes. CLI version text alone is not the build identity. + +The native game still uses Windows D3D12 through Proton on this machine. +A separate real Intel Vulkan prototype checked 4,096 pixels for the current +24-byte three-address binding ABI and 624-byte shared constants. It exposed +Plume's missing image-to-buffer copy branch; the fixed stock Plume API now +produces byte-identical output. This establishes the tested ABI and readback +path, not a complete Vulkan game renderer or a performance gain. Plume stays +on its public pin with an additional verified `plume-vulkan-readback.patch`. + +The last accepted native checkpoint samples remain 15.84/16.59 FPS with unequal +draw workloads; no cache speedup is established and the cache stays off. The matching Windows runtime and PM4 plugin have built. The full game +build exposed a separate 165,000-call generated registration function that +exceeds the 3 GiB compiler cap; compact registration output is now the build +gate before the native/PM4 scene comparison. No level or full campaign completion has been verified. + ## Linux limitations - `rexlego/CMakeLists.txt` explicitly rejects `LEGODIMENSIONS_NATIVE_GPU=ON` @@ -575,7 +610,7 @@ allocation reads one page-table entry instead of 131,072. Existing write-watch invalidation tests also pass. This establishes less validation work, not an observed gameplay FPS gain. -SDK changes are published as separate local commits through `b71efb4`: atomic +SDK changes are recorded as separate local commits through `5dc986f`: atomic GPU interrupt publication/counters, shader-cache bounds, failed-dispatch TLS, prelaunch quit handling, vblank-before-processor shutdown, explicit interior body entries, graph indexing, verified SIMDe preparation, platform/include @@ -585,9 +620,9 @@ and full runtime teardown remain outside those fixtures. The five game `body` mappings require the new codegen feature. The old native CLI predates its later static-library/producer rebuild, so it cannot establish -current-source generation. A fresh isolated CLI and generated-code producer, -followed by genuine game codegen and matched consumer/plugin builds, are the -next integration gate. Expected vendor overlays must remain explicit in +current-source generation. A fresh isolated CLI, generated-code producer +and genuine game codegen have now passed with identical generated bytes. +Matched consumer/plugin builds are the next integration gate. Expected vendor overlays must remain explicit in provenance; old frozen binaries do not become clean builds when source commits are created. diff --git a/tools/gpu-aot/patches/plume-vulkan-readback.patch b/tools/gpu-aot/patches/plume-vulkan-readback.patch new file mode 100644 index 0000000..a4a755d --- /dev/null +++ b/tools/gpu-aot/patches/plume-vulkan-readback.patch @@ -0,0 +1,140 @@ +diff --git a/plume_vulkan.cpp b/plume_vulkan.cpp +index 11abda8..93e34ea 100644 +--- a/plume_vulkan.cpp ++++ b/plume_vulkan.cpp +@@ -3250,7 +3250,124 @@ namespace plume { + imageCopy.imageExtent.depth = srcLocation.placedFootprint.depth; + vkCmdCopyBufferToImage(vk, srcBuffer->vk, dstTexture->vk, toImageLayout(dstTexture->textureLayout), 1, &imageCopy); + } +- else { ++ else if ((dstLocation.type == RenderTextureCopyType::PLACED_FOOTPRINT) && (srcLocation.type == RenderTextureCopyType::SUBRESOURCE)) { ++ if ((dstBuffer == nullptr) || (srcTexture == nullptr)) { ++ assert(false && "Image-to-buffer copy requires a source texture and destination buffer."); ++ return; ++ } ++ ++ const auto &footprint = dstLocation.placedFootprint; ++ if ((queue == nullptr) || (queue->type == RenderCommandListType::UNKNOWN) || ++ (srcTexture->desc.format <= RenderFormat::UNKNOWN) || (srcTexture->desc.format >= RenderFormat::MAX) || ++ (footprint.format <= RenderFormat::UNKNOWN) || (footprint.format >= RenderFormat::MAX) || ++ (srcTexture->desc.multisampling.sampleCount != RenderSampleCount::COUNT_1)) { ++ assert(false && "Image-to-buffer copy requires a valid format, queue and single-sampled source."); ++ return; ++ } ++ if ((srcLocation.subresource.mipLevel >= srcTexture->desc.mipLevels) || ++ (srcLocation.subresource.arrayIndex >= srcTexture->desc.arraySize)) { ++ assert(false && "Image-to-buffer source subresource is outside its texture."); ++ return; ++ } ++ const uint32_t blockWidth = RenderFormatBlockWidth(srcTexture->desc.format); ++ const uint32_t blockSize = RenderFormatSize(srcTexture->desc.format); ++ if ((blockWidth == 0) || (blockSize == 0) || ((blockWidth == 4) && (blockSize != 8) && (blockSize != 16))) { ++ assert(false && "Image-to-buffer copy requires a supported texel block size."); ++ return; ++ } ++ const VkImageAspectFlags aspect = toAspectFlags(srcTexture->desc.format, srcTexture->desc.flags); ++ // Buffer copies select exactly one aspect. The public copy location ++ // has no plane selector, so combined depth/stencil cannot be copied. ++ if ((aspect == 0) || ((aspect & (aspect - 1)) != 0) || ++ (RenderFormatBlockWidth(footprint.format) != blockWidth) || ++ (RenderFormatSize(footprint.format) != blockSize) || ++ (RenderFormatIsDepth(srcTexture->desc.format) != ((aspect & VK_IMAGE_ASPECT_DEPTH_BIT) != 0)) || ++ (((aspect & VK_IMAGE_ASPECT_DEPTH_BIT) != 0) && (queue->type != RenderCommandListType::DIRECT))) { ++ assert(false && "Image-to-buffer copy requires one aspect and a compatible footprint."); ++ return; ++ } ++ ++ VkBufferImageCopy imageCopy = {}; ++ imageCopy.imageSubresource.aspectMask = aspect; ++ imageCopy.imageSubresource.baseArrayLayer = srcLocation.subresource.arrayIndex; ++ imageCopy.imageSubresource.layerCount = 1; ++ imageCopy.imageSubresource.mipLevel = srcLocation.subresource.mipLevel; ++ const uint32_t mip = srcLocation.subresource.mipLevel; ++ const uint32_t width = mip < 32 ? std::max(1U, srcTexture->desc.width >> mip) : 1U; ++ const uint32_t height = mip < 32 ? std::max(1U, srcTexture->desc.height >> mip) : 1U; ++ const uint32_t depth = mip < 32 ? std::max(1U, srcTexture->desc.depth >> mip) : 1U; ++ if (srcBox != nullptr) { ++ if ((srcBox->left < 0) || (srcBox->top < 0) || (srcBox->front < 0) || ++ (srcBox->left >= srcBox->right) || (srcBox->top >= srcBox->bottom) || ++ (srcBox->front >= srcBox->back) || (srcBox->right > width) || ++ (srcBox->bottom > height) || (srcBox->back > depth)) { ++ assert(false && "Image-to-buffer source box is outside its mip."); ++ return; ++ } ++ imageCopy.imageOffset = { int32_t(srcBox->left), int32_t(srcBox->top), int32_t(srcBox->front) }; ++ imageCopy.imageExtent = { uint32_t(srcBox->right - srcBox->left), uint32_t(srcBox->bottom - srcBox->top), uint32_t(srcBox->back - srcBox->front) }; ++ } ++ else { ++ imageCopy.imageExtent = { width, height, depth }; ++ } ++ ++ // rowWidth is in texels, as in the upload path and D3D12's ++ // footprint conversion. Buffer offsets are in compressed blocks. ++ const uint64_t rowBlocks = (uint64_t(footprint.rowWidth) + blockWidth - 1) / blockWidth; ++ const uint64_t imageRows = (uint64_t(footprint.height) + blockWidth - 1) / blockWidth; ++ if ((footprint.width == 0) || (footprint.height == 0) || (footprint.depth == 0) || ++ (footprint.rowWidth < footprint.width) || (rowBlocks * blockWidth > UINT32_MAX) || ++ (imageRows * blockWidth > UINT32_MAX) || ++ (uint64_t(dstX) + imageCopy.imageExtent.width > footprint.width) || ++ (uint64_t(dstY) + imageCopy.imageExtent.height > footprint.height) || ++ (uint64_t(dstZ) + imageCopy.imageExtent.depth > footprint.depth) || ++ ((dstX % blockWidth) != 0) || ((dstY % blockWidth) != 0) || ++ ((imageCopy.imageOffset.x % blockWidth) != 0) || ((imageCopy.imageOffset.y % blockWidth) != 0) || ++ (((imageCopy.imageExtent.width % blockWidth) != 0) && (uint64_t(imageCopy.imageOffset.x) + imageCopy.imageExtent.width != width)) || ++ (((imageCopy.imageExtent.height % blockWidth) != 0) && (uint64_t(imageCopy.imageOffset.y) + imageCopy.imageExtent.height != height))) { ++ assert(false && "Image-to-buffer copy does not fit its footprint or block alignment."); ++ return; ++ } ++ const uint64_t rowPitch = rowBlocks * blockSize; ++ if (rowPitch > INT32_MAX) { ++ assert(false && "Image-to-buffer row pitch exceeds Vulkan's signed 31-bit limit."); ++ return; ++ } ++ // Padding after the final copied row is not accessed. Check stride ++ // arithmetic first, then the exact last accessed byte below. ++ if ((footprint.offset > dstBuffer->desc.size) || (imageRows > UINT64_MAX / rowPitch)) { ++ assert(false && "Image-to-buffer footprint exceeds its buffer."); ++ return; ++ } ++ const uint64_t slicePitch = rowPitch * imageRows; ++ const uint64_t remaining = dstBuffer->desc.size - footprint.offset; ++ const uint64_t lastSlice = uint64_t(dstZ) + imageCopy.imageExtent.depth - 1; ++ if (lastSlice > remaining / slicePitch) { ++ assert(false && "Image-to-buffer slices exceed its buffer."); ++ return; ++ } ++ const uint64_t sliceEnd = lastSlice * slicePitch; ++ const uint64_t lastRow = uint64_t(dstY / blockWidth) + (uint64_t(imageCopy.imageExtent.height) + blockWidth - 1) / blockWidth - 1; ++ const uint64_t rowEnd = lastRow * rowPitch; ++ const uint64_t copiedRow = ((uint64_t(dstX) + imageCopy.imageExtent.width + blockWidth - 1) / blockWidth) * blockSize; ++ if ((rowEnd > remaining - sliceEnd) || (copiedRow > remaining - sliceEnd - rowEnd)) { ++ assert(false && "Image-to-buffer region exceeds its buffer."); ++ return; ++ } ++ imageCopy.bufferOffset = footprint.offset + uint64_t(dstZ) * slicePitch + uint64_t(dstY / blockWidth) * rowPitch + uint64_t(dstX / blockWidth) * blockSize; ++ // VUIDs07975/07978/07746 apply to the final offset, including ++ // destination coordinates. COPY is conservative if its selected ++ // queue family also happens to support graphics or compute. ++ const uint32_t offsetAlignment = ((aspect & VK_IMAGE_ASPECT_DEPTH_BIT) != 0 || queue->type == RenderCommandListType::COPY) ? std::max(4U, blockSize) : blockSize; ++ if ((imageCopy.bufferOffset % offsetAlignment) != 0) { ++ assert(false && "Image-to-buffer final offset is not aligned for its aspect or queue."); ++ return; ++ } ++ imageCopy.bufferRowLength = uint32_t(rowBlocks * blockWidth); ++ imageCopy.bufferImageHeight = uint32_t(imageRows * blockWidth); ++ vkCmdCopyImageToBuffer(vk, srcTexture->vk, toImageLayout(srcTexture->textureLayout), dstBuffer->vk, 1, &imageCopy); ++ } ++ else if ((dstLocation.type == RenderTextureCopyType::SUBRESOURCE) && (srcLocation.type == RenderTextureCopyType::SUBRESOURCE)) { + VkImageCopy imageCopy = {}; + imageCopy.srcSubresource.aspectMask = toAspectFlags(srcTexture->desc.format, srcTexture->desc.flags); + imageCopy.srcSubresource.baseArrayLayer = srcLocation.subresource.arrayIndex; +@@ -3283,6 +3400,9 @@ namespace plume { + + vkCmdCopyImage(vk, srcTexture->vk, toImageLayout(srcTexture->textureLayout), dstTexture->vk, toImageLayout(dstTexture->textureLayout), 1, &imageCopy); + } ++ else { ++ assert(false && "Unsupported texture copy location combination."); ++ } + } + + void VulkanCommandList::copyBuffer(const RenderBuffer *dstBuffer, const RenderBuffer *srcBuffer) { diff --git a/tools/gpu-aot/prepare_plume.py b/tools/gpu-aot/prepare_plume.py index d7bfbc5..eb4eeed 100644 --- a/tools/gpu-aot/prepare_plume.py +++ b/tools/gpu-aot/prepare_plume.py @@ -70,6 +70,7 @@ def main(): patches = ( (root / "tools/gpu-aot/patches/plume-lego.patch", "Native Plume fixes"), (root / "tools/gpu-aot/patches/plume-state-cache.patch", "Native Plume state-cache fixes"), + (root / "tools/gpu-aot/patches/plume-vulkan-readback.patch", "Native Plume Vulkan readback fixes"), ) def git(*arguments, check=False): @@ -105,9 +106,9 @@ def git(*arguments, check=False): parser.error(str(error)) applied = True if applied: - print(f"Applied native fence/resize/command-allocation fixes and opt-in state cache to public Plume {PIN}") + print(f"Applied native fence/resize/command-allocation fixes, opt-in state cache and Vulkan readback to public Plume {PIN}") else: - print(f"PASS: public Plume {PIN} with native fence/resize/command-allocation fixes and opt-in state cache") + print(f"PASS: public Plume {PIN} with native fence/resize/command-allocation fixes, opt-in state cache and Vulkan readback") if __name__ == "__main__": diff --git a/tools/gpu-aot/test_draw_diagnostic.py b/tools/gpu-aot/test_draw_diagnostic.py index de2d6bd..388ec89 100644 --- a/tools/gpu-aot/test_draw_diagnostic.py +++ b/tools/gpu-aot/test_draw_diagnostic.py @@ -176,7 +176,7 @@ def main(): fixture_memory.Word(0x2000+offsetof(D3DDevice,vertex_declaration),0x10080); fixture_memory.Word(0x2000+10564,0x90000);fixture_memory.Word(0x2000+10572,0x400); fixture_memory.Word(0x2000+10500,0x3b808081);fixture_memory.Word(0x2000+10620,0x3f800000); - fixture_memory.Word(0x2000+13032,1280);fixture_memory.Word(0x2000+13036,720); + fixture_memory.Word(0x2000+13032,0x44A00000);fixture_memory.Word(0x2000+13036,0x44340000); fixture_memory.Word(0x2000+0x780+48*16,0x3f800000); fixture_memory.Word(0x2000+0x1780+4*16+12,0x3f000000); fixture_memory.Word(0x2000+0x1780+45*16,0x3f800000); @@ -197,7 +197,7 @@ def main(): auto text=Read(trace.output()); for(std::string_view expected:{"attempted_ordinal=1","VS=1234 PS=abcd","raw_count=2", "raw_decl_bytes=000102030405060708090a0b0c0d0e0f1011121314151617", - "clip_control=90000 viewport_control=400", "viewport_width=500 viewport_height=2d0", + "clip_control=90000 viewport_control=400", "viewport_width_bits=44a00000 viewport_height_bits=44340000", "alpha_ref_bits=3b808081 legacy_float10620_bits=3f800000", "VS_C48=3f800000,00000000,00000000,00000000,", "PS_C4=00000000,00000000,00000000,3f000000,", diff --git a/tools/gpu-aot/test_prepare_plume.py b/tools/gpu-aot/test_prepare_plume.py index 012d992..85da43a 100644 --- a/tools/gpu-aot/test_prepare_plume.py +++ b/tools/gpu-aot/test_prepare_plume.py @@ -32,6 +32,7 @@ def main(): preparer = args.preparer.resolve() patch = root / "tools/gpu-aot/patches/plume-lego.patch" cache_patch = root / "tools/gpu-aot/patches/plume-state-cache.patch" + readback_patch = root / "tools/gpu-aot/patches/plume-vulkan-readback.patch" git_executable = shutil.which("git") if git_executable is None: parser.error("Git is required for the offline public-source fixture") @@ -53,6 +54,7 @@ def git(checkout, *arguments): "status": git(source, "status", "--porcelain=v1", "--untracked-files=all").stdout, "source_sha256": digest(source / "plume_d3d12.cpp"), "header_sha256": digest(source / "plume_d3d12.h"), + "vulkan_sha256": digest(source / "plume_vulkan.cpp"), } git(source, "cat-file", "-e", f"{PIN}^{{commit}}") public_blob = subprocess.run([git_executable, "-C", str(source), "show", f"{PIN}:plume_d3d12.cpp"], @@ -82,6 +84,9 @@ def prepare(checkout, *options, expect=0, oracle=None): git(expected, "apply", str(patch)) git(expected, "apply", "--check", str(cache_patch)) git(expected, "apply", str(cache_patch)) + git(expected, "apply", "--check", str(readback_patch)) + git(expected, "apply", str(readback_patch)) + expected_vulkan_sha = digest(expected / "plume_vulkan.cpp") expected_sha = digest(expected / "plume_d3d12.cpp") expected_header_sha = digest(expected / "plume_d3d12.h") checks.append("actual pinned public source and independently Git-applied patch") @@ -110,6 +115,8 @@ def prepare(checkout, *options, expect=0, oracle=None): prepare(checkout, oracle="PASS: public Plume") if digest(checkout / "plume_d3d12.cpp") != expected_sha: raise AssertionError("Repeated preparation changed the patched source") + if digest(checkout / "plume_vulkan.cpp") != expected_vulkan_sha: + raise AssertionError("Vulkan readback differs from independent real patch") checks.append("apply/check/repeat from unrelated cwd preserve exact patch result") old = clone("old-mandatory-prepared-checkout") @@ -124,6 +131,29 @@ def prepare(checkout, *options, expect=0, oracle=None): raise AssertionError("Old prepared upgrade differs from fresh complete preparation") checks.append("old mandatory-prepared checkout upgrades without reverting existing fixes") + readback_old = clone("state-cache-prepared-missing-readback") + git(readback_old, "apply", str(patch)) + git(readback_old, "apply", str(cache_patch)) + readback_before = digest(readback_old / "plume_vulkan.cpp") + prepare(readback_old, "--check", expect=2, oracle="Native Plume Vulkan readback fixes are missing or incomplete") + if digest(readback_old / "plume_vulkan.cpp") != readback_before: + raise AssertionError("Missing readback --check modified source") + prepare(readback_old) + prepare(readback_old, "--check", oracle="PASS: public Plume") + if digest(readback_old / "plume_vulkan.cpp") != expected_vulkan_sha: + raise AssertionError("Readback upgrade differs from independent real patch") + checks.append("state-cache-prepared checkout upgrades with exact Vulkan readback patch") + + partial_readback = clone("partial-vulkan-readback") + prepare(partial_readback) + text = (partial_readback / "plume_vulkan.cpp").read_text() + guard = 'assert(false && "Image-to-buffer copy requires a source texture and destination buffer.");' + if text.count(guard) != 1: + raise AssertionError("Readback guard was not found exactly once") + (partial_readback / "plume_vulkan.cpp").write_text(text.replace(guard, 'assert(false && "fixture partial readback");')) + prepare(partial_readback, "--check", expect=2, oracle="Native Plume Vulkan readback fixes are missing or incomplete") + checks.append("partial Vulkan readback patch rejected") + cache_partial = clone("partial-state-cache-header") prepare(cache_partial) header = (cache_partial / "plume_d3d12.h").read_text() @@ -200,11 +230,26 @@ def fake_run(argv, *args, **kwargs): raise AssertionError("State-cache no-op fixture modified source") checks.append("state-cache false Git success rejected independently of mandatory stage") + readback_noop = clone("successful-no-op-readback-git") + git(readback_noop, "apply", str(patch)) + git(readback_noop, "apply", str(cache_patch)) + readback_before = digest(readback_noop / "plume_vulkan.cpp") + readback_wrapper = wrapper.replace("and 'apply' in argv:", + "and 'apply' in argv and any('plume-vulkan-readback.patch' in str(v) for v in argv):") + for mode in ("already", "apply"): + command([sys.executable, "-c", readback_wrapper, mode, str(preparer), str(readback_noop)], unrelated, + expect=2, oracle="Native Plume source verification failed: missing patched hunk") + if digest(readback_noop / "plume_vulkan.cpp") != readback_before: + raise AssertionError("Readback no-op fixture modified source") + checks.append("readback false Git success rejected independently of both earlier stages") + + after = { "head": git(source, "rev-parse", "HEAD").stdout, "status": git(source, "status", "--porcelain=v1", "--untracked-files=all").stdout, "source_sha256": digest(source / "plume_d3d12.cpp"), "header_sha256": digest(source / "plume_d3d12.h"), + "vulkan_sha256": digest(source / "plume_vulkan.cpp"), } if original != after: raise AssertionError("Original Plume checkout changed during the offline fixture") @@ -212,6 +257,8 @@ def fake_run(argv, *args, **kwargs): report = {"passed": True, "pin": PIN, "actual_public_source": True, "offline": True, "preparer_sha256": digest(preparer), "patch_sha256": digest(patch), "state_cache_patch_sha256": digest(cache_patch), + "vulkan_readback_patch_sha256": digest(readback_patch), + "patched_vulkan_sha256": expected_vulkan_sha, "public_source_sha256": hashlib.sha256(public_blob).hexdigest(), "patched_source_sha256": expected_sha, "patched_header_sha256": expected_header_sha, "checks": checks} diff --git a/tools/gpu-aot/test_scissor_state.py b/tools/gpu-aot/test_scissor_state.py new file mode 100644 index 0000000..6c2a140 --- /dev/null +++ b/tools/gpu-aot/test_scissor_state.py @@ -0,0 +1,227 @@ +"""Execute native setters/draw scissor code; optionally compare actual TU23 bodies. + +The base fixture needs no SDK, generated game code, assets or graphics device. +When the two generated TU23 files exist, execute their unchanged original +SetScissorRect and enable bodies with bounded big-endian memory as an oracle. +""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('output', type=Path) +p.add_argument('--compiler', default='g++') +p.add_argument('--no-oracle', action='store_true', help='Run the asset-free base fixture only') +a = p.parse_args() +a.output.mkdir(parents=True, exist_ok=True) +root = Path(__file__).resolve().parents[2] + + +def body(source, signature): + start = source.index(signature) + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +hooks = (root / 'rexlego/src/gpu_native/hooks_state.cpp').read_text() +draw = (root / 'rexlego/src/gpu_native/draw.cpp').read_text() +native_setters = '\n'.join(body(hooks, sig) for sig in + ['void SetScissorRectHook(', 'void SetScissorEnableHook(']) +start = draw.index(' const auto effective_scissor = DecodeNativeScissor(') +end = draw.index('\n std::array +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "gpu_native/scissor_state.h" +#include "gpu_native/cull_state.h" +using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using i32=int32_t; +struct be_u32 { + u32 raw=0;operator u32()const{return __builtin_bswap32(raw);} + be_u32& operator=(u32 v){raw=__builtin_bswap32(v);return *this;} +}; +struct be_i32 { + u32 raw=0;operator i32()const{return i32(__builtin_bswap32(raw));} + be_i32& operator=(i32 v){raw=__builtin_bswap32(u32(v));return *this;} +}; +struct D3DViewport9 {be_u32 x,y,width,height,min_z,max_z;}; +struct be_f32 { + u32 raw=0;operator float()const{return std::bit_cast(__builtin_bswap32(raw));} + be_f32& operator=(float v){raw=__builtin_bswap32(std::bit_cast(v));return *this;} +}; +struct D3DViewportState {be_f32 x,y,width,height,min_z,max_z;}; +struct D3DRect {be_i32 left,top,right,bottom;}; +struct D3DDevice {u8 prefix[13024]{};D3DViewportState viewport;be_u32 reserved;D3DRect scissor;}; +static_assert(offsetof(D3DDevice,scissor)==13052); +using namespace legodimensions::gpu_native; +namespace plume {using RenderRect=NativeScissorRect;} +struct Commands {NativeScissorRect result;void setScissors(NativeScissorRect r){result=r;}}; +''' + native_setters + '\n' + body(draw, 'plume::RenderRect CullDrawScissor(') + r''' +NativeScissorRect NativeDraw(D3DDevice* device,u32 width,u32 height, + u32 raster_control=0,bool polygonal=true){ + Commands recorded;auto* commands=&recorded; + const auto* state_bytes=reinterpret_cast(device); + struct {u32 width,height;}color{width,height}; + struct {u32 raster_control;}key{raster_control}; +''' + native_draw + r''' + return recorded.result; +} +// Exact removed draw expression, preserved as the deliberate negative control. +NativeScissorRect LegacyDraw(D3DDevice* device,u32 width,u32 height){ + const i32 scissor_right=static_cast(device->scissor.right); + const i32 scissor_bottom=static_cast(device->scissor.bottom); + return {static_cast(device->scissor.left),static_cast(device->scissor.top), + scissor_right>0?scissor_right:static_cast(width), + scissor_bottom>0?scissor_bottom:static_cast(height)}; +} +D3DRect Rect(i32 l,i32 t,i32 r,i32 b){D3DRect v;v.left=l;v.top=t;v.right=r;v.bottom=b;return v;} +D3DViewport9 View(u32 x,u32 y,u32 w,u32 h){D3DViewport9 v;v.x=x;v.y=y;v.width=w;v.height=h;return v;} +// This fixture supplies API inputs numerically. The separate viewport fixture +// executes the production RAW routing and the exact original TU23 setters. +void SetViewportFixture(D3DDevice* d,const D3DViewport9* v){if(!d||!v)return; + d->viewport.x=float(u32(v->x));d->viewport.y=float(u32(v->y)); + d->viewport.width=float(u32(v->width));d->viewport.height=float(u32(v->height));} +void Expect(D3DDevice& d,NativeScissorRect r){assert(NativeDraw(&d,64,48)==r);} +''' +if has_oracle: + source += r''' +union PPCRegister {uint64_t u64;int64_t s64;uint32_t u32;int32_t s32;float f32;}; +union FRegister {double f64;uint64_t u64;int64_t s64;}; +struct CR {bool eq=false,gt=false,lt=false;templatevoid compare(T a,T b,int){eq=a==b;gt=a>b;lt=a ram{}; +u32 Load(u32 at){assert(at+4ull<=ram.size());u32 v;std::memcpy(&v,ram.data()+at,4);return __builtin_bswap32(v);} +void Store(u32 at,u32 v){assert(at+4ull<=ram.size());v=__builtin_bswap32(v);std::memcpy(ram.data()+at,&v,4);} +u64 Load64(u32 at){return (u64(Load(at))<<32)|Load(at+4);} +void Store64(u32 at,u64 v){Store(at,u32(v>>32));Store(at+4,u32(v));} +u64 Rotate(u64 x,int n){return std::rotl(x,n);} +double simde_mm_load_sd(const double* x){return *x;} +i32 simde_mm_cvttsd_si32(double x){return std::isfinite(x)&&x>=INT_MIN&&x<=INT_MAX?i32(x):INT_MIN;} +#define __builtin_rotateleft64 Rotate +#define REX_LOAD_U32(p) Load(u32(p)) +#define REX_STORE_U32(p,v) Store(u32(p),u32(v)) +#define REX_LOAD_U64(p) Load64(u32(p)) +#define REX_STORE_U64(p,v) Store64(u32(p),u64(v)) +#define REX_FUNC_PROLOGUE() ((void)0) +#define DEFINE_REX_FUNC(n) void n(PPCContext& ctx,u8* base) +void sub_83FB7AB8(PPCContext&,u8*){} // Packet emission has no shadow effect. +''' + oracle + r''' +NativeScissorRect Oracle(D3DDevice& d){ + constexpr u32 address=0x1000; + // Native and original TU23 now share the same floating point device shadow. + Store(address+13024,std::bit_cast(float(d.viewport.x))); + Store(address+13028,std::bit_cast(float(d.viewport.y))); + Store(address+13032,std::bit_cast(float(d.viewport.width))); + Store(address+13036,std::bit_cast(float(d.viewport.height))); + Store(address+13052,u32(i32(d.scissor.left)));Store(address+13056,u32(i32(d.scissor.top))); + Store(address+13060,u32(i32(d.scissor.right)));Store(address+13064,u32(i32(d.scissor.bottom))); + PPCContext c{};c.r1.u32=0x1F000;c.r3.u32=address; + c.r4.u32=u32(*reinterpret_cast(reinterpret_cast(&d)+12288)); + sub_83FBA968(c,ram.data()); + auto tl=Load(address+10436),br=Load(address+10440); + i32 l=std::min(i32(tl&32767),64),t=std::min(i32((tl>>16)&32767),48); + return {l,t,std::max(l,std::min(i32(br&32767),64)), + std::max(t,std::min(i32((br>>16)&32767),48))}; +} +''' +source += r''' +int main(){ + D3DDevice d{}; + // Detached-device zero initialization resets enable; no global cache survives. + auto view=View(0,0,64,48);SetViewportFixture(&d,&view); + auto requested=Rect(8,7,20,19);SetScissorRectHook(&d,&requested); + Expect(d,{0,0,64,48});assert(LegacyDraw(&d,64,48)!=NativeDraw(&d,64,48)); + SetScissorEnableHook(&d,1);Expect(d,{8,7,20,19}); + SetScissorEnableHook(&d,0);Expect(d,{0,0,64,48}); + SetScissorEnableHook(&d,2);Expect(d,{8,7,20,19}); // All nonzero enables. + assert(i32(d.scissor.left)==8&&i32(d.scissor.right)==20); + requested=Rect(0,0,0,0);SetScissorRectHook(&d,&requested); + Expect(d,{0,0,0,0});assert(LegacyDraw(&d,64,48)!=NativeDraw(&d,64,48)); + requested=Rect(0,0,32,0);SetScissorRectHook(&d,&requested);Expect(d,{0,0,32,0}); + requested=Rect(40,30,10,20);SetScissorRectHook(&d,&requested);Expect(d,{40,30,40,30}); + requested=Rect(-10,-20,100,80);SetScissorRectHook(&d,&requested);Expect(d,{0,0,64,48}); + view=View(4,5,20,16);SetViewportFixture(&d,&view);Expect(d,{4,5,24,21}); + requested=Rect(8,9,30,40);SetScissorRectHook(&d,&requested);Expect(d,{8,9,24,21}); + SetScissorEnableHook(&d,0);Expect(d,{4,5,24,21}); + view=View(60,40,UINT32_MAX,UINT32_MAX);SetViewportFixture(&d,&view);Expect(d,{60,40,64,48}); + view=View(UINT32_MAX,UINT32_MAX,1,1);SetViewportFixture(&d,&view);Expect(d,{64,48,64,48}); + view=View(4,5,0,0);SetViewportFixture(&d,&view);Expect(d,{4,5,4,5}); + auto saved=d;SetViewportFixture(nullptr,&view);SetViewportFixture(&d,nullptr); + SetScissorRectHook(nullptr,&requested);SetScissorRectHook(&d,nullptr); + SetScissorEnableHook(nullptr,1);assert(std::memcmp(&d,&saved,sizeof(d))==0); + d={};view=View(0,0,64,48);SetViewportFixture(&d,&view);Expect(d,{0,0,64,48}); + assert((NativeDraw(&d,64,48,kNativeCullCandidate|3)==NativeScissorRect{0,0,0,0})); +''' +if has_oracle: + source += r''' + std::mt19937 random(0x83FBA068); + for(u32 n=0;n<2000;++n){ + view=View(random()%64,random()%48,random()%65,random()%49); + SetViewportFixture(&d,&view); + requested=Rect(random()%80,random()%60,random()%80,random()%60); + SetScissorRectHook(&d,&requested); + SetScissorEnableHook(&d,n%3);assert(NativeDraw(&d,64,48)==Oracle(d)); + const auto before=d.scissor; + SetScissorEnableHook(&d,0);assert(NativeDraw(&d,64,48)==Oracle(d)); + SetScissorEnableHook(&d,1);assert(NativeDraw(&d,64,48)==Oracle(d)); + assert(std::memcmp(&before,&d.scissor,sizeof(before))==0); + } +''' +source += '\n}\n' +cpp = a.output / 'scissor-state.cpp' +exe = a.output / 'scissor-state.exe' +cpp.write_text(source) +subprocess.run([a.compiler, '-std=c++20', '-O0', '-UNDEBUG', + '-I', str(root / 'rexlego/src'), str(cpp), '-o', str(exe)], + check=True, timeout=45) +subprocess.run([str(exe.resolve())], check=True, timeout=20) +report = {'passed': True, 'actual_native_setters_and_draw_section': True, + 'actual_tu23_oracle': has_oracle, 'oracle_cases': 6000 if has_oracle else 0, + 'original_bodies': provenance, + 'legacy_negative_cases': ['disabled stale rect', 'enabled empty rect'], + 'limits': 'CPU recording only; no game or GPU'} +(a.output / 'verification.json').write_text(json.dumps(report, indent=2) + '\n') +print(json.dumps(report)) diff --git a/tools/gpu-aot/test_viewport_shadow.py b/tools/gpu-aot/test_viewport_shadow.py new file mode 100644 index 0000000..cfb72b1 --- /dev/null +++ b/tools/gpu-aot/test_viewport_shadow.py @@ -0,0 +1,307 @@ +"""Exercise native viewport routing/layout and optional unchanged TU23 setters. + +Base tests are asset-free. Initialized generated sources add the exact unsigned +API setter, float restore setter, shared writer, AA18 reset and complete bulk +attachment setter. Reset bytes were read from the actual patched TU23 image; +provenance: .local-testing/reports/TU23-viewport-reference-byte-proof.json. +""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('output', type=Path) +p.add_argument('--compiler', default='g++') +p.add_argument('--no-oracle', action='store_true') +a = p.parse_args() +a.output.mkdir(parents=True, exist_ok=True) +root = Path(__file__).resolve().parents[2] + + +def body(source, signature): + start = source.index(signature) + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +hooks = (root / 'rexlego/src/gpu_native/hooks_state.cpp').read_text() +draw = (root / 'rexlego/src/gpu_native/draw.cpp').read_text() +native = '\n'.join(body(hooks, signature) for signature in + ['bool NativeViewportRange(', 'bool NativeViewportArguments(', + 'void SetScissorRectHook(', 'void SetScissorEnableHook(']) +raw = body(hooks, 'REX_HOOK_RAW(sub_83FBA978)') +raw = raw.replace('REX_HOOK_RAW(sub_83FBA978)', 'void sub_83FBA978(PPCContext& ctx,u8* base)', 1) +begin = draw.index(' const float viewport_width =') +end = draw.index('\n std::array +#include +using u8=uint8_t;using u32=uint32_t;using u64=uint64_t;using i32=int32_t; +templatestruct BigEndian { + uint32_t raw=0; + operator T()const{auto v=__builtin_bswap32(raw);return std::bit_cast(v);} + BigEndian& operator=(T v){raw=__builtin_bswap32(std::bit_cast(v));return *this;} +}; +using be_u32=BigEndian;using be_i32=BigEndian;using be_f32=BigEndian; +''') +source = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "gpu_native/d3d.h" +#include "gpu_native/scissor_state.h" +#include "gpu_native/cull_state.h" +using namespace legodimensions::gpu_native; +union PPCRegister {uint64_t u64;int64_t s64;uint32_t u32;int32_t s32;uint8_t u8;float f32;}; +union FRegister {double f64;uint64_t u64;int64_t s64;}; +struct XER {u8 ca=0;}; +struct CR {bool eq=false,gt=false,lt=false;templatevoid compare(T a,T b,XER){eq=a==b;gt=a>b;lt=a ram{}; +std::map constants; +u32 Load(u32 at){if(at>=ram.size()){if(!constants.contains(at))std::fprintf(stderr,"unmapped oracle load %08X\n",at);assert(constants.contains(at));return constants.at(at);} + assert(at+4ull<=ram.size());u32 v;std::memcpy(&v,ram.data()+at,4);return __builtin_bswap32(v);} +void Store(u32 at,u32 v){assert(at+4ull<=ram.size());v=__builtin_bswap32(v);std::memcpy(ram.data()+at,&v,4);} +u64 Load64(u32 at){return (u64(Load(at))<<32)|Load(at+4);} +void Store64(u32 at,u64 v){Store(at,u32(v>>32));Store(at+4,u32(v));} +void Float(u32 at,float value){Store(at,std::bit_cast(value));} +u64 Rotate(u64 v,int n){return std::rotl(v,n);} +u32 Rotate32(u32 v,int n){return std::rotl(v,n);} +double simde_mm_load_sd(const double* v){return *v;} +i32 simde_mm_cvttsd_si32(double v){return std::isfinite(v)&&v>=INT_MIN&&v<=INT_MAX?i32(v):INT_MIN;} +#define __builtin_rotateleft64 Rotate +#define __builtin_rotateleft32 Rotate32 +#define REX_LOAD_U32(p) Load(u32(p)) +#define REX_STORE_U32(p,v) Store(u32(p),u32(v)) +#define REX_LOAD_U64(p) Load64(u32(p)) +#define REX_STORE_U64(p,v) Store64(u32(p),u64(v)) +#define REX_LOAD_U8(p) (ram.at(u32(p))) +#define REX_STORE_U8(p,v) (ram.at(u32(p))=u8(v)) +#define REX_LOAD_U16(p) ((u32(ram.at(u32(p)))<<8)|ram.at(u32(p)+1)) +#define REX_FUNC_PROLOGUE() ((void)0) +#define DEFINE_REX_FUNC(n) void n(PPCContext& ctx,u8* base) +namespace rex::memory { +constexpr u32 kMemoryProtectRead=1,kMemoryProtectWrite=2,kMemoryProtectWriteCombine=8; +struct HeapAllocationInfo {u32 protect=3;}; +} +struct Heap { + std::set unreadable,readonly;u32 page_size()const{return 4096;} + bool IsRangeCommittedReadable(u32 at,u32 bytes){ + if(!bytes||u64(at)+bytes>0x100000000ull)return false; + if(at>=ram.size()){ + for(u64 x=at;xram.size())return false; + for(u64 page=at/4096;page<=(u64(at)+bytes-1)/4096;++page) + if(unreadable.contains(u32(page)))return false; + return true; + } + bool QueryProtect(u32 at,u32* protect){ + *protect=at>=ram.size()||readonly.contains(at/4096)?1:3;return true; + } +}heap; +struct Memory { + Heap* LookupHeap(u32){return &heap;} + templateT TranslateVirtual(u32 at){assert(at(ram.data()+at);} +}memory; +#define REX_KERNEL_MEMORY() (&::memory) +namespace legodimensions::gpu_native { +struct HostDevice {static auto LockRecording(){return std::unique_lock(recording);}static inline std::recursive_mutex recording;}; +''' + native + r''' +} +void sub_83FBA710(PPCContext&,u8*);void sub_83FBA978(PPCContext&,u8*); +void __imp__sub_83FBA978(PPCContext&,u8*); +void sub_83FBA068(PPCContext& c,u8*){ + auto* d=memory.TranslateVirtual(c.r3.u32); + SetScissorRectHook(d,memory.TranslateVirtual(c.r4.u32)); +} +''' + raw + '\n' + '\n'.join(originals) +if not has_oracle: + source += 'u32 original_calls=0;void __imp__sub_83FBA978(PPCContext&,u8*){++original_calls;}\n' +source += r''' +namespace plume { +using RenderRect=NativeScissorRect; +struct RenderViewport {float x,y,w,h,min_z,max_z;RenderViewport(float x,float y,float w,float h,float z,float m):x(x),y(y),w(w),h(h),min_z(z),max_z(m){}}; +} +struct Commands {plume::RenderViewport viewport{0,0,0,0,0,0};NativeScissorRect scissor{}; + void setViewports(plume::RenderViewport v){viewport=v;}void setScissors(NativeScissorRect v){scissor=v;}}; +''' + body(draw, 'plume::RenderRect CullDrawScissor(') + r''' +u32 host_records=0; +bool GuardedDraw(D3DDevice* device){ +''' + draw_guard + r''' + ++host_records;return true; +} +bool query_failed=false; +void AttemptDraw(D3DDevice* device){if(!GuardedDraw(device))query_failed=true;} +Commands Record(D3DDevice* device){ + Commands output;auto* commands=&output;const auto* state_bytes=reinterpret_cast(device); + const bool viewport_candidate=false;struct{u32 width=64,height=48;}color;struct{u32 raster_control=0;}key;bool polygonal=true; +''' + record + r''' + return output; +} +constexpr u32 device=0x1000,argument=0x9000,color_surface=0xA000,depth_surface=0xB000,stack=0x1F000; +D3DDevice& Device(){return *memory.TranslateVirtual(device);} +PPCContext Context(){PPCContext c{};c.r1.u32=stack;c.r3.u32=device;c.r4.u32=argument;return c;} +void Api(float x,float y,u32 w,u32 h){auto* a=memory.TranslateVirtual(argument); + a->x=u32(x);a->y=u32(y);a->width=w;a->height=h;a->min_z=0.25f;a->max_z=0.75f; + auto c=Context();sub_83FBA978(c,ram.data());} +void Reset(){ram.fill(0);heap.unreadable.clear();heap.readonly.clear(); + Device().render_targets[0]=color_surface;Device().depth_stencil=depth_surface; + auto* color=memory.TranslateVirtual(color_surface);color->size_bits=((64-1)<<18)|((48-1)<<3); + auto* depth=memory.TranslateVirtual(depth_surface);depth->size_bits=((32-1)<<18)|((24-1)<<3); +} +int main(){ + Reset();assert(NativeViewportArguments(device,argument,stack)); + assert(!NativeViewportArguments(0,argument,stack));assert(!NativeViewportArguments(device,0,stack)); + assert(!NativeViewportArguments(device,argument,128)); + assert(!NativeViewportArguments(0xFFFFFFF0,argument,stack)); + heap.unreadable.insert(2);assert(!NativeViewportArguments(device,argument,stack));heap.unreadable.clear(); + heap.readonly.insert(3);assert(!NativeViewportArguments(device,argument,stack));heap.readonly.clear(); + heap.readonly.insert((stack-512)/4096);assert(!NativeViewportArguments(device,argument,stack));heap.readonly.clear(); + Device().render_targets[0]=0xFFFFFFF0;assert(!NativeViewportArguments(device,argument,stack));Reset(); + auto c=Context();c.r4.u32=0;auto before=ram;sub_83FBA978(c,ram.data());assert(ram==before); + assert(NativeViewportInteger(12.75f)==12);assert(NativeViewportInteger(-12.75f)==-12); + assert(NativeViewportInteger(INFINITY)==INT_MAX);assert(NativeViewportInteger(-INFINITY)==INT_MIN); + assert(NativeViewportInteger(NAN)==INT_MIN); + Device().viewport.x=3.0f;Device().viewport.y=4.0f;Device().viewport.width=20.0f;Device().viewport.height=16.0f; + auto recorded=Record(&Device());assert(recorded.viewport.x==3&&recorded.viewport.w==20); + assert((recorded.scissor==NativeScissorRect{3,4,23,20})); + Device().viewport.width=0.0f;assert(Record(&Device()).viewport.w==0); // No fabricated full-target viewport. + assert(NativeViewportValid(3,4,20,16,0.25f,0.75f)); + assert(NativeViewportValid(3,4,0,16,0,1));assert(!NativeViewportValid(3,4,-1,16,0,1)); + for(float bad:{NAN,INFINITY,-INFINITY}){ + assert(!NativeViewportValid(bad,4,20,16,0,1));assert(!NativeViewportValid(3,bad,20,16,0,1)); + assert(!NativeViewportValid(3,4,bad,16,0,1));assert(!NativeViewportValid(3,4,20,bad,0,1)); + assert(!NativeViewportValid(3,4,20,16,bad,1));assert(!NativeViewportValid(3,4,20,16,0,bad)); + } + assert(!NativeViewportValid(3,4,20,16,-1,1));assert(!NativeViewportValid(3,4,20,16,0,2)); + assert(NativeViewportValid(3,4,20,16,0.75f,0.25f)); + Device().viewport.min_z=0;Device().viewport.max_z=1; + AttemptDraw(&Device());assert(!query_failed&&host_records==0&&float(Device().viewport.width)==0); + Device().viewport.width=20;AttemptDraw(&Device());assert(!query_failed&&host_records==1); + Device().viewport.width=NAN;AttemptDraw(&Device());assert(query_failed&&host_records==1); +''' +if has_oracle: + source += r''' + Reset();Api(3,4,40,30); + assert(float(Device().viewport.x)==3&&float(Device().viewport.y)==4); + assert(float(Device().viewport.width)==40&&float(Device().viewport.height)==30); + assert(float(Device().viewport.min_z)==0.25f&&float(Device().viewport.max_z)==0.75f); + assert(Load(device+10504)==std::bit_cast(20.0f)); // X scale + assert(Load(device+10508)==std::bit_cast(23.0f)); // X offset + assert(Load(device+10512)==std::bit_cast(-15.0f)); // Y scale + assert(Load(device+10516)==std::bit_cast(19.0f)); + assert(Load(device+10520)==std::bit_cast(0.5f));assert(Load(device+10524)==std::bit_cast(0.25f)); + assert(Load(device+13048)==0&&(Load64(device+16)&0x07E00000)==0x07E00000); + recorded=Record(&Device());assert(recorded.viewport.w==40&&recorded.viewport.h==30); + // Real original private save28 -> float setter restore, same memory ABI. + std::memcpy(ram.data()+argument,reinterpret_cast(&Device().viewport),28); + const auto saved=Device().viewport;Device().viewport={};c=Context();sub_83FBA9F8(c,ram.data()); + assert(std::memcmp(&saved,&Device().viewport,sizeof(saved))==0); + // Deliberate former native memcpy makes internal float width subnormal. + auto api=memory.TranslateVirtual(argument);api->x=3;api->y=4;api->width=40;api->height=30; + api->min_z=0.25f;api->max_z=0.75f; + std::memcpy(&Device().viewport,api,sizeof(*api));assert(float(Device().viewport.width)!=40); + std::memcpy(ram.data()+argument,reinterpret_cast(&Device().viewport),28); + c=Context();sub_83FBA9F8(c,ram.data());assert(float(Device().viewport.width)==0); + // Surface bounds, depth fallback, no attachment early return, reversed bounds. + Reset();Api(60,40,100,100);assert(float(Device().viewport.width)==4&&float(Device().viewport.height)==8); + Device().render_targets[0]=0;Api(3,4,100,100);assert(float(Device().viewport.width)==29&&float(Device().viewport.height)==20); + const auto no_target=Device().viewport;Device().depth_stencil=0;Api(0,0,12,13); + assert(std::memcmp(&no_target,&Device().viewport,sizeof(no_target))==0); + Reset();Api(70,0,10,10);assert(float(Device().viewport.width)==0&&float(Device().viewport.height)==0); + // Original tiled extent selection with exactly matching shadow attachments. + Reset();ram[device+11068]=0x10;Store(device+13556,80);Store(device+13560,60); + Api(0,0,100,100);assert(float(Device().viewport.width)==80&&float(Device().viewport.height)==60); + // Actual patched TU23 default API bytes, observed by mapped-image probe. + Reset();constants[0x824CAF44]=0;constants[0x824CAF48]=0; + constants[0x824CAF4C]=65535;constants[0x824CAF50]=65535; + constants[0x824CAF54]=0;constants[0x824CAF58]=0x3F800000; + constants[0x824CAF5C]=0;constants[0x824CAF60]=0;constants[0x824CAF64]=65535;constants[0x824CAF68]=65535; + c=Context();c.r4.u32=color_surface;sub_83FBAA18(c,ram.data()); + assert(float(Device().viewport.width)==64&&float(Device().viewport.height)==48); + assert(float(Device().viewport.max_z)==1&&i32(Device().scissor.right)==65535); + // Complete exact original bulk setter, including reset and native API route. + auto* attachments=memory.TranslateVirtual(argument); + attachments[0]=depth_surface;attachments[1]=color_surface; + attachments[2]=attachments[3]=attachments[4]=0; + c=Context();c.r5.u32=0;sub_83FBB110(c,ram.data()); + assert(float(Device().viewport.width)==64&&float(Device().viewport.height)==48); + recorded=Record(&Device());assert(recorded.viewport.w==64&&recorded.viewport.h==48); + // Repeated bulk binds/reset cannot resurrect the preceding small viewport. + Api(3,4,5,6);attachments[0]=depth_surface;attachments[1]=color_surface; + attachments[2]=attachments[3]=attachments[4]=0; + c=Context();c.r5.u32=0;sub_83FBB110(c,ram.data()); + assert(float(Device().viewport.x)==0&&float(Device().viewport.width)==64); +''' +else: + source += 'Reset();Api(3,4,40,30);assert(original_calls==1);\n' +source += '\n}\n' +if has_oracle: + # Shared writer uses the actual image constant proved by the same probe. + source = source.replace('Reset();assert(NativeViewportArguments', + 'constants[0x82005D7C]=0x3F000000;Reset();assert(NativeViewportArguments', 1) +cpp = a.output / 'viewport-shadow.cpp' +exe = a.output / 'viewport-shadow.exe' +cpp.write_text(source) +subprocess.run([a.compiler, '-std=c++20', '-O0', '-UNDEBUG', '-I', str(a.output/'include'), + '-I', str(root/'rexlego/src'), str(cpp), '-o', str(exe)], check=True, timeout=45) +subprocess.run([str(exe.resolve())], check=True, timeout=20) +report = {'passed': True, 'actual_native_raw_guard_and_recording': True, + 'actual_d3d_layout': True, 'actual_original_tu23_setters_and_bulk': has_oracle, + 'legacy_uint_memcpy_negative': has_oracle, + 'original_bodies': provenance, + 'image_constants_provenance': '.local-testing/reports/TU23-viewport-reference-byte-proof.json', + 'limits': 'Bounded CPU fixture; no game or GPU execution'} +(a.output/'verification.json').write_text(json.dumps(report, indent=2)+'\n') +print(json.dumps(report)) diff --git a/tools/gpu-aot/test_vulkan_texture_copy.py b/tools/gpu-aot/test_vulkan_texture_copy.py new file mode 100644 index 0000000..bf23521 --- /dev/null +++ b/tools/gpu-aot/test_vulkan_texture_copy.py @@ -0,0 +1,211 @@ +"""Exercise the actual Vulkan texture-copy body and public footprint types without a GPU. + +The Vulkan dispatch is recorded, not emulated. Real pixel/coherence coverage is +provided separately by the bounded headless Vulkan ABI prototype. +""" +import argparse +import hashlib +import json +from pathlib import Path +import resource +import signal +import subprocess + + +def function(source, signature): + start = source.index(signature) + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--compiler', default='clang++') + parser.add_argument('--source', type=Path) + parser.add_argument('--vulkan-include', type=Path, + help='Vulkan header include root (defaults to public Plume contrib/Vulkan-Headers/include)') + parser.add_argument('--verify-regression', action='store_true') + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + plume = root / 'thirdparty/plume' + vulkan_include = args.vulkan_include or plume / 'contrib/Vulkan-Headers/include' + if not (vulkan_include / 'vulkan/vulkan_core.h').is_file(): + parser.error('Vulkan headers unavailable; initialize Plume contrib headers or set --vulkan-include') + path = args.source or plume / 'plume_vulkan.cpp' + production = path.read_text() + helpers = '\n'.join(function(production, sig) for sig in ( + 'static VkImageLayout toImageLayout(', 'static VkImageAspectFlags toViewAspectFlags(', + 'static VkImageAspectFlags toAspectFlags(')) + signature = 'void VulkanCommandList::copyTextureRegion(' + actual = function(production, signature) + harness = r''' +#include +#include +#include +#include +#include +#define VK_NO_PROTOTYPES +#include +#include "plume_render_interface_types.h" +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr,"copy_oracle line %d: %s\n",__LINE__,#c); std::exit(17); } } while (0) +namespace plume { +struct RenderBuffer { virtual ~RenderBuffer() = default; }; +struct RenderTexture { virtual ~RenderTexture() = default; }; +struct VulkanBuffer: RenderBuffer { VkBuffer vk = (VkBuffer)uintptr_t(0x111); RenderBufferDesc desc; }; +struct VulkanTexture: RenderTexture { VkImage vk = (VkImage)uintptr_t(0x222); RenderTextureDesc desc; RenderTextureLayout textureLayout=RenderTextureLayout::COPY_SOURCE; }; +struct VulkanCommandQueue { RenderCommandListType type=RenderCommandListType::DIRECT; }; +struct Record { int kind; VkBufferImageCopy buffer{}; VkImageCopy image{}; VkImage src{},dst{}; VkBuffer srcBuffer{},dstBuffer{}; VkImageLayout srcLayout{},dstLayout{}; }; +std::vector records; +void vkCmdCopyBufferToImage(VkCommandBuffer, VkBuffer b, VkImage i, VkImageLayout layout, uint32_t n, const VkBufferImageCopy* c) { CHECK(n==1); Record r{};r.kind=1;r.buffer=*c;r.dst=i;r.srcBuffer=b;r.dstLayout=layout;records.push_back(r); } +void vkCmdCopyImageToBuffer(VkCommandBuffer, VkImage i, VkImageLayout layout, VkBuffer b, uint32_t n, const VkBufferImageCopy* c) { CHECK(n==1); Record r{};r.kind=2;r.buffer=*c;r.src=i;r.dstBuffer=b;r.srcLayout=layout;records.push_back(r); } +void vkCmdCopyImage(VkCommandBuffer,VkImage s,VkImageLayout sl,VkImage d,VkImageLayout dl,uint32_t n,const VkImageCopy* c) { CHECK(n==1);Record r{};r.kind=3;r.image=*c;r.src=s;r.dst=d;r.srcLayout=sl;r.dstLayout=dl;records.push_back(r); } +struct VulkanCommandList { VkCommandBuffer vk{};VulkanCommandQueue ownedQueue;VulkanCommandQueue* queue=&ownedQueue;int closures=0;void endActiveRenderPass(){++closures;}void copyTextureRegion(const RenderTextureCopyLocation&,const RenderTextureCopyLocation&,uint32_t=0,uint32_t=0,uint32_t=0,const RenderBox*=nullptr); }; +@HELPERS@ +@BODY@ +} +using namespace plume; +int main() { + VulkanCommandList list;VulkanTexture image,destination;VulkanBuffer buffer; + image.desc=RenderTextureDesc::Texture(RenderTextureDimension::TEXTURE_2D,32,16,1,3,4,RenderFormat::R8G8B8A8_UNORM); + destination.desc=image.desc;destination.textureLayout=RenderTextureLayout::COPY_DEST;destination.vk=(VkImage)uintptr_t(0x333); + buffer.desc=RenderBufferDesc::ReadbackBuffer(65536); + auto src=RenderTextureCopyLocation::Subresource(&image,1,2); + auto dst=RenderTextureCopyLocation::PlacedFootprint(&buffer,RenderFormat::R8G8B8A8_UNORM,24,16,3,32,512); + // Existing upload and image-copy routes retain their original semantics. + auto upload=RenderTextureCopyLocation::PlacedFootprint(&buffer,RenderFormat::R8G8B8A8_UNORM,7,5,1,10,256); + list.copyTextureRegion(RenderTextureCopyLocation::Subresource(&destination,1,3),upload,3,2,0); + CHECK(records.size()==1&&records.back().kind==1);auto u=records.back(); + CHECK(u.buffer.bufferOffset==256&&u.buffer.bufferRowLength==10&&u.buffer.bufferImageHeight==5); + CHECK(u.buffer.imageExtent.width==7&&u.buffer.imageExtent.height==5&&u.buffer.imageExtent.depth==1); + CHECK(u.buffer.imageOffset.x==3&&u.buffer.imageOffset.y==2&&u.buffer.imageSubresource.mipLevel==1&&u.buffer.imageSubresource.baseArrayLayer==3); + CHECK(u.srcBuffer==buffer.vk&&u.dst==destination.vk&&u.dstLayout==VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + RenderBox box(2,1,10,5,0,1); + list.copyTextureRegion(RenderTextureCopyLocation::Subresource(&destination,0,1),src,7,8,0,&box); + CHECK(records.size()==2&&records.back().kind==3);auto t=records.back(); + CHECK(t.image.srcOffset.x==2&&t.image.srcOffset.y==1&&t.image.extent.width==8&&t.image.extent.height==4); + CHECK(t.image.srcSubresource.mipLevel==1&&t.image.srcSubresource.baseArrayLayer==2&&t.image.dstSubresource.mipLevel==0&&t.image.dstSubresource.baseArrayLayer==1); + CHECK(t.image.dstOffset.x==7&&t.image.dstOffset.y==8&&t.srcLayout==VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL&&t.dstLayout==VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + // Missing stock route: full selected mip and array layer, padded texel pitch. + std::puts("BEGIN stock readback null-image regression");std::fflush(stdout); + list.copyTextureRegion(dst,src); + CHECK(records.size()==3&&records.back().kind==2);auto r=records.back(); + CHECK(r.buffer.bufferOffset==512&&r.buffer.bufferRowLength==32&&r.buffer.bufferImageHeight==16); + CHECK(r.buffer.imageExtent.width==16&&r.buffer.imageExtent.height==8&&r.buffer.imageExtent.depth==1); + CHECK(r.buffer.imageSubresource.mipLevel==1&&r.buffer.imageSubresource.baseArrayLayer==2&&r.buffer.imageSubresource.layerCount==1); + CHECK(r.src==image.vk&&r.dstBuffer==buffer.vk&&r.srcLayout==VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + // Source box, destination XYZ and independently calculated row/slice offset. + image.desc=RenderTextureDesc::Texture3D(32,16,8,3,RenderFormat::R8G8B8A8_UNORM);src=RenderTextureCopyLocation::Subresource(&image,1); + box=RenderBox(2,1,10,5,0,2);list.copyTextureRegion(dst,src,4,2,1,&box);r=records.back(); + CHECK(r.kind==2&&r.buffer.bufferOffset==2832&&r.buffer.imageExtent.depth==2); + CHECK(r.buffer.imageOffset.x==2&&r.buffer.imageOffset.y==1&&r.buffer.imageOffset.z==0); + // BC1: pitches are texels rounded to4; byte offset counts8-byte blocks. + image.desc=RenderTextureDesc::Texture2D(10,7,1,RenderFormat::BC1_UNORM);src=RenderTextureCopyLocation::Subresource(&image); + dst=RenderTextureCopyLocation::PlacedFootprint(&buffer,RenderFormat::BC1_UNORM,18,15,1,21,1024); + list.copyTextureRegion(dst,src,4,4);r=records.back(); + CHECK(r.buffer.bufferOffset==1080&&r.buffer.bufferRowLength==24&&r.buffer.bufferImageHeight==16); + CHECK(r.buffer.imageExtent.width==10&&r.buffer.imageExtent.height==7); + box=RenderBox(4,4,10,7);list.copyTextureRegion(dst,src,8,8,0,&box);r=records.back(); + CHECK(r.buffer.bufferOffset==1136&&r.buffer.imageExtent.width==6&&r.buffer.imageExtent.height==3); + image.desc.format=RenderFormat::BC3_UNORM;dst.placedFootprint.format=RenderFormat::BC3_UNORM; + list.copyTextureRegion(dst,src,4,4);r=records.back(); + CHECK(r.buffer.bufferOffset==1136&&r.buffer.bufferRowLength==24&&r.buffer.bufferImageHeight==16); + destination.desc=image.desc; + auto bcUpload=dst;bcUpload.placedFootprint.width=10;bcUpload.placedFootprint.height=7; + list.copyTextureRegion(RenderTextureCopyLocation::Subresource(&destination),bcUpload);r=records.back(); + CHECK(r.kind==1&&r.buffer.bufferRowLength==24&&r.buffer.bufferImageHeight==8); + // Single depth aspect is representable; combined depth/stencil is refused. + image.desc=RenderTextureDesc::Texture2D(4,3,1,RenderFormat::D32_FLOAT,RenderTextureFlag::DEPTH_TARGET);src=RenderTextureCopyLocation::Subresource(&image); + dst=RenderTextureCopyLocation::PlacedFootprint(&buffer,RenderFormat::D32_FLOAT,4,3,1,8,256); + list.copyTextureRegion(dst,src);r=records.back();CHECK(r.buffer.imageSubresource.aspectMask==VK_IMAGE_ASPECT_DEPTH_BIT&&r.buffer.bufferRowLength==8); + auto reject=[&](const RenderTextureCopyLocation& d,const RenderTextureCopyLocation& s,uint32_t x=0,uint32_t y=0,uint32_t z=0,const RenderBox* b=nullptr){size_t count=records.size();int closures=list.closures;list.copyTextureRegion(d,s,x,y,z,b);CHECK(records.size()==count&&list.closures==closures+1);}; + image.desc.format=RenderFormat::D24_UNORM_S8_UINT;dst.placedFootprint.format=RenderFormat::D24_UNORM_S8_UINT;reject(dst,src); + image.desc=RenderTextureDesc::Texture2D(8,8,1,RenderFormat::R8G8B8A8_UNORM);src=RenderTextureCopyLocation::Subresource(&image); + dst=RenderTextureCopyLocation::PlacedFootprint(&buffer,RenderFormat::R8G8B8A8_UNORM,8,8,1,8,256); + auto small=dst;small.placedFootprint.width=7;reject(small,src); + small=dst;small.placedFootprint.format=RenderFormat::UNKNOWN;reject(small,src); + small=dst;small.placedFootprint.format=RenderFormat::MAX;reject(small,src); + small=dst;small.placedFootprint.format=RenderFormat(9999);reject(small,src); + image.desc.format=RenderFormat::UNKNOWN;reject(dst,src);image.desc.format=RenderFormat::R8G8B8A8_UNORM; + image.desc.multisampling.sampleCount=RenderSampleCount::COUNT_4;reject(dst,src);image.desc.multisampling.sampleCount=RenderSampleCount::COUNT_1; + list.queue=nullptr;reject(dst,src);list.queue=&list.ownedQueue; + list.queue->type=RenderCommandListType::UNKNOWN;reject(dst,src);list.queue->type=RenderCommandListType::DIRECT; + small=dst;small.placedFootprint.rowWidth=536870912;reject(small,src); // 2GiB row pitch VUID09108. + small=dst;small.placedFootprint.rowWidth=7;reject(small,src); + small=dst;small.placedFootprint.offset=UINT64_MAX-3;reject(small,src); + small=dst;small.placedFootprint.offset=258;reject(small,src); + small=dst;small.placedFootprint.rowWidth=UINT32_MAX;small.placedFootprint.height=UINT32_MAX;reject(small,src); + small=dst;small.buffer=nullptr;reject(small,src); + reject(dst,RenderTextureCopyLocation::Subresource(&image,1)); + reject(dst,RenderTextureCopyLocation::Subresource(&image,0,1)); + box=RenderBox(-1,0,4,4);reject(dst,src,0,0,0,&box); + box=RenderBox(0,0,9,4);reject(dst,src,0,0,0,&box); + reject(dst,src,1);reject(dst,src,0,0,1); + buffer.desc.size=511;reject(dst,src); // needs exactly512 bytes incl prefix. + buffer.desc.size=512;list.copyTextureRegion(dst,src);CHECK(records.back().buffer.bufferOffset==256); + // Large unused row/slice padding does not require allocation past the last + // copied byte (D3D12 footprints also omit the final row's trailing padding). + image.desc=RenderTextureDesc::Texture2D(1,1,1,RenderFormat::R8G8B8A8_UNORM); + dst=RenderTextureCopyLocation::PlacedFootprint(&buffer,RenderFormat::R8G8B8A8_UNORM,1,64,1,256,0); + buffer.desc.size=4;list.copyTextureRegion(dst,src);r=records.back(); + CHECK(r.buffer.bufferOffset==0&&r.buffer.bufferRowLength==256&&r.buffer.bufferImageHeight==64&&r.buffer.imageExtent.width==1); + buffer.desc.size=3;reject(dst,src); + // D16 has two-byte texels but its final copy offset must align to4. + buffer.desc.size=4096;image.desc=RenderTextureDesc::Texture2D(1,1,1,RenderFormat::D16_UNORM,RenderTextureFlag::DEPTH_TARGET); + dst=RenderTextureCopyLocation::PlacedFootprint(&buffer,RenderFormat::D16_UNORM,4,1,1,4,0); + reject(dst,src,1);list.copyTextureRegion(dst,src,2);CHECK(records.back().buffer.bufferOffset==4); + dst.placedFootprint.offset=2;list.copyTextureRegion(dst,src,1);CHECK(records.back().buffer.bufferOffset==4); + list.queue->type=RenderCommandListType::COMPUTE;reject(dst,src);list.queue->type=RenderCommandListType::DIRECT; + dst.placedFootprint.format=RenderFormat::R32_FLOAT;reject(dst,src); // Depth footprint texel size mismatch. + // COPY queue uses4-byte offsets even for byte-wide color formats. Check + // destination X, Y and Z, and permit a final offset aligned by dstX. + image.desc=RenderTextureDesc::Texture2D(1,1,1,RenderFormat::R8_UNORM); + dst=RenderTextureCopyLocation::PlacedFootprint(&buffer,RenderFormat::R8_UNORM,4,2,2,5,0); + list.queue->type=RenderCommandListType::COPY;reject(dst,src,1);reject(dst,src,0,1);reject(dst,src,0,0,1); + list.copyTextureRegion(dst,src);CHECK(records.back().buffer.bufferOffset==0); + dst.placedFootprint.offset=1;list.copyTextureRegion(dst,src,3);CHECK(records.back().buffer.bufferOffset==4); + list.queue->type=RenderCommandListType::DIRECT;list.copyTextureRegion(dst,src);CHECK(records.back().buffer.bufferOffset==1); + // No fallback may reinterpret a second buffer as an image. + reject(dst,dst);auto unknown=dst;unknown.type=RenderTextureCopyType::UNKNOWN;reject(unknown,src); + std::puts("PASS actual copy body: upload/image/readback, mip/layer/3D/BC1/depth/offset/bounds/release guards"); +} +'''.replace('const RenderBox*=nullptr', 'const RenderBox* = nullptr') + args.output.mkdir(parents=True, exist_ok=True) + def limit(): + resource.setrlimit(resource.RLIMIT_AS, (3 * 1024**3, 3 * 1024**3)) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + def run(name, body): + cpp = args.output / (name + '.cpp') + exe = args.output / name + cpp.write_text(harness.replace('@HELPERS@', helpers).replace('@BODY@', body)) + compiled = subprocess.run([args.compiler, '-std=c++17', '-O0', '-DNDEBUG', + '-I', str(plume), '-I', str(vulkan_include), + str(cpp), '-o', str(exe)], capture_output=True, text=True, timeout=45, preexec_fn=limit) + if compiled.returncode: + raise RuntimeError(compiled.stdout + compiled.stderr) + return subprocess.run([str(exe.resolve())], capture_output=True, text=True, timeout=20, preexec_fn=limit) + positive = run('copy', actual) + if positive.returncode: + raise RuntimeError(positive.stdout + positive.stderr) + negative = None + if args.verify_regression: + old = subprocess.check_output(['git', '-C', str(plume), 'show', 'e0c8871b930dc1a6544bf457f2348c456b9d7e13:plume_vulkan.cpp'], text=True, timeout=10) + failed = run('copy-stock-negative', function(old, signature)) + if failed.returncode != -signal.SIGSEGV or 'BEGIN stock readback null-image regression' not in failed.stdout: + raise RuntimeError('Stock negative must reach named readback and fail with SIGSEGV: ' + repr(failed)) + negative = {'returncode': failed.returncode, 'stdout': failed.stdout, 'stderr': failed.stderr} + report = {'status': 'passed', 'production_sha256': hashlib.sha256(path.read_bytes()).hexdigest(), + 'positive': positive.stdout, 'stock_negative': negative, + 'scope': 'Actual production copy/format/layout bodies with mocked Vulkan dispatch; no GPU execution'} + (args.output / 'verification.json').write_text(json.dumps(report, indent=2) + '\n') + print(positive.stdout, end='') + if negative: + print('PASS stock image-to-buffer negative, exit', negative['returncode']) + + +if __name__ == '__main__': + main() From a54ac14ef2711955422eda433248bf1c6185b8b4 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:43:50 +0200 Subject: [PATCH 35/40] Pin SDK with compact ordered function registration output --- rexglue-sdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rexglue-sdk b/rexglue-sdk index 5dc986f..47b59e0 160000 --- a/rexglue-sdk +++ b/rexglue-sdk @@ -1 +1 @@ -Subproject commit 5dc986f69c454fb474bc9472481625a04a500789 +Subproject commit 47b59e0baed49ae6e23df354f4bfebd80fc03dc0 From 88cf091a0ff0bdab662147923367edea69dcd6cd Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:55:14 +0200 Subject: [PATCH 36/40] Pin validated registration fixture deadline --- rexglue-sdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rexglue-sdk b/rexglue-sdk index 47b59e0..2897c31 160000 --- a/rexglue-sdk +++ b/rexglue-sdk @@ -1 +1 @@ -Subproject commit 47b59e0baed49ae6e23df354f4bfebd80fc03dc0 +Subproject commit 2897c31b1407dd96480dc25965f8ee7807f00e0f From 38ebe531d02b30bc1a4859bc23a54bada3a0b93e Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:31:34 +0200 Subject: [PATCH 37/40] Fix viewport clipping against promoted native tile targets --- rexlego/src/gpu_native/hooks_state.cpp | 62 +++++++++++++++++- rexlego/src/gpu_native/textures.cpp | 12 ++++ rexlego/src/gpu_native/textures.h | 3 + tools/gpu-aot/COLLABORATION.md | 31 +++++++-- tools/gpu-aot/test_viewport_shadow.py | 89 +++++++++++++++++++++++--- 5 files changed, 181 insertions(+), 16 deletions(-) diff --git a/rexlego/src/gpu_native/hooks_state.cpp b/rexlego/src/gpu_native/hooks_state.cpp index b4900c1..7448fb7 100644 --- a/rexlego/src/gpu_native/hooks_state.cpp +++ b/rexlego/src/gpu_native/hooks_state.cpp @@ -123,16 +123,43 @@ bool NativeViewportRange(u32 address, u32 length, bool write) { return true; } -bool NativeViewportArguments(u32 device_address, u32 viewport_address, u32 stack) { +bool NativeViewportDeviceArguments(u32 device_address, u32 stack, u32 scratch = 512) { if (!NativeViewportRange(device_address, kGuestDeviceSize, true) || - !NativeViewportRange(viewport_address, sizeof(D3DViewport9), false) || - stack < 512 || !NativeViewportRange(stack - 512, 512, true)) return false; + stack < scratch || !NativeViewportRange(stack - scratch, scratch, true)) return false; const auto* device = REX_KERNEL_MEMORY()->TranslateVirtual(device_address); const u32 surface = device->render_targets[0] ? u32(device->render_targets[0]) : u32(device->depth_stencil); return !surface || NativeViewportRange(surface, sizeof(D3DSurface), false); } +bool NativeViewportArguments(u32 device_address, u32 viewport_address, u32 stack) { + return NativeViewportDeviceArguments(device_address, stack) && + NativeViewportRange(viewport_address, sizeof(D3DViewport9), false); +} + +class ScopedNativeViewportExtent { + public: + ScopedNativeViewportExtent(D3DDevice* device, u32 width, u32 height) + : bytes_(reinterpret_cast(device)), flags_(bytes_[11068]), + width_(*reinterpret_cast(bytes_ + 13556)), + height_(*reinterpret_cast(bytes_ + 13560)) { + *reinterpret_cast(bytes_ + 13556) = width; + *reinterpret_cast(bytes_ + 13560) = height; + bytes_[11068] = flags_ | 0x10; + } + ~ScopedNativeViewportExtent() { + bytes_[11068] = flags_; + *reinterpret_cast(bytes_ + 13556) = width_; + *reinterpret_cast(bytes_ + 13560) = height_; + } + ScopedNativeViewportExtent(const ScopedNativeViewportExtent&) = delete; + ScopedNativeViewportExtent& operator=(const ScopedNativeViewportExtent&) = delete; + private: + u8* bytes_; + u8 flags_; + u32 width_, height_; +}; + void SetScissorRectHook(D3DDevice* device, const D3DRect* rect) { if (device && rect) { device->scissor = *rect; @@ -195,6 +222,35 @@ REX_HOOK_RAW(sub_83FBA978) { // shadows. Its scissor call takes the native hook, without Xbox packets. __imp__sub_83FBA978(ctx, base); } +REX_HOOK_RAW(sub_83FBA710) { + auto recording = legodimensions::gpu_native::HostDevice::LockRecording(); + // Shared TU23 writer reserves 160 bytes. Its save-FPR slots are within + // that range; API caller has already reserved its separate 112-byte frame. + if (!legodimensions::gpu_native::NativeViewportDeviceArguments( + ctx.r3.u32, ctx.r1.u32, 160)) return; + auto* device = reinterpret_cast(base + ctx.r3.u32); + const u32 surface = device->render_targets[0] ? u32(device->render_targets[0]) + : u32(device->depth_stencil); + u32 width = 0, height = 0; + if (legodimensions::gpu_native::NativePromotedSurfaceExtent(surface, width, height)) { + // Native draws cover the whole promoted target, rather than replaying a + // physical Xbox tile. Let the original CPU writer clip against that extent. + // Only this call sees the extent override; persistent tiling state and + // guest surface dimensions remain unchanged. + const double requested_width = ctx.f3.f64, requested_height = ctx.f4.f64; + { + legodimensions::gpu_native::ScopedNativeViewportExtent extent(device, width, height); + __imp__sub_83FBA710(ctx, base); + } + static u32 logs = 0; + if (logs++ < 16) + REXLOG_INFO("Native GPU: promoted viewport surface={:08X} requested={}x{} host={}x{} effective={}x{}", + surface, requested_width, requested_height, width, height, + float(device->viewport.width), float(device->viewport.height)); + } else { + __imp__sub_83FBA710(ctx, base); + } +} REX_HOOK(sub_83FBA068, legodimensions::gpu_native::SetScissorRectHook); REX_HOOK(sub_83FBA968, legodimensions::gpu_native::SetScissorEnableHook); REX_HOOK(sub_83FBCBB8, legodimensions::gpu_native::SetPredicationHook); diff --git a/rexlego/src/gpu_native/textures.cpp b/rexlego/src/gpu_native/textures.cpp index 7af2d26..ccd1789 100644 --- a/rexlego/src/gpu_native/textures.cpp +++ b/rexlego/src/gpu_native/textures.cpp @@ -1854,6 +1854,18 @@ bool ResolveTextureFromSurface(u32 destination_texture, u32 source_surface, return UpdateResolvedSampling(*resolved_destination); } +bool NativePromotedSurfaceExtent(u32 guest_address, u32& width, u32& height) { + const auto resource = FindTexture(guest_address); + if (!resource || !resource->surface || !resource->texture || + !resource->host_width || !resource->host_height || + resource->host_width > 16384 || resource->host_height > 32768 || + (resource->host_width == resource->width && + resource->host_height == resource->height)) return false; + width = resource->host_width; + height = resource->host_height; + return true; +} + bool PromoteTiledSurface(u32 guest_address, u32 width, u32 height) { auto recording = HostDevice::LockRecording(); if (!guest_address) return true; diff --git a/rexlego/src/gpu_native/textures.h b/rexlego/src/gpu_native/textures.h index e4840e6..4b2b585 100644 --- a/rexlego/src/gpu_native/textures.h +++ b/rexlego/src/gpu_native/textures.h @@ -72,6 +72,9 @@ plume::RenderFramebuffer* ResolveFramebuffer( // Synchronous BeginTiling allocation, before the pass's clear and geometry. // Guest surface dimensions and Xbox tiling/command-queue fields are unchanged. bool PromoteTiledSurface(u32 guest_address, u32 width, u32 height); +// Caller holds HostDevice::LockRecording. Metadata only: never adopts a guest +// texture or allocates host storage. +bool NativePromotedSurfaceExtent(u32 guest_address, u32& width, u32& height); float SurfaceColorOutputScale(u32 guest_address); bool PrepareSurfaceDepthAlias(u32 guest_address); void MarkSurfaceWritten(u32 guest_address); diff --git a/tools/gpu-aot/COLLABORATION.md b/tools/gpu-aot/COLLABORATION.md index 5a97a7e..3aa11d8 100644 --- a/tools/gpu-aot/COLLABORATION.md +++ b/tools/gpu-aot/COLLABORATION.md @@ -14,7 +14,16 @@ viewport audit found integer API bytes copied into a six-float private shadow; bulk reset uses a 65,535-pixel sentinel that the original setter clips to the active surface. The correction and scissor enable/empty-rectangle semantics passed the full 68-check renderer suite and independent review. Older frozen -builds below predate them. +builds below predate them. Live fingerprint `73ca236405793015` then exposed a +missing integration case: Xbox tile surfaces retain physical dimensions while +the native renderer promotes host storage to the complete image. Calling the +original viewport writer without that logical extent squashed the menu into +512 rows on a 720-row target. That run is a confirmed regression, not a visual +baseline. The shared writer now receives a scoped promoted extent through its +original CPU clipping branch; guest resource dimensions and persistent tiling +state remain unchanged. The actual integer, float and bulk setter bodies cover +this regression, including the unadapted 512-row negative control. Live menu +recovery and post-promotion setter ordering still require verification. An isolated native SDK build through local commit `5dc986f` passed its 225 selected unit and production regressions. Four existing BitStream cases remain @@ -33,11 +42,23 @@ produces byte-identical output. This establishes the tested ABI and readback path, not a complete Vulkan game renderer or a performance gain. Plume stays on its public pin with an additional verified `plume-vulkan-readback.patch`. +SDK commit `2897c31` includes compact codegen registration. The genuinely +rebuilt CLI SHA256 is +`69dd8b7aeb1715277fed6720dab5b4127234206384d9267b0f2137193d9e00d1`. +It regenerated all 165,261 game bindings with identical addresses, symbols and +order; other generated instructions and headers were byte-identical. The +production registration regression passed after increasing its total deadline +for five bounded compiler invocations. The complete native game then built +within a 6 GiB owned-process ceiling, peaking around 2.4 GiB. Its matching +Windows runtime and PM4 plugin also built; Linux PM4 configuration is being +prepared in a separate source tree. + The last accepted native checkpoint samples remain 15.84/16.59 FPS with unequal -draw workloads; no cache speedup is established and the cache stays off. The matching Windows runtime and PM4 plugin have built. The full game -build exposed a separate 165,000-call generated registration function that -exceeds the 3 GiB compiler cap; compact registration output is now the build -gate before the native/PM4 scene comparison. No level or full campaign completion has been verified. +draw workloads; no cache speedup is established and the cache stays off. A PM4 +correctness reference must use full uncapped resolve readback, coherent +memexport/page-state refresh and synchronous pipeline creation. Inherited fast +readback settings are unsuitable for that comparison. No level or full campaign +completion has been verified. ## Linux limitations diff --git a/tools/gpu-aot/test_viewport_shadow.py b/tools/gpu-aot/test_viewport_shadow.py index cfb72b1..1cededd 100644 --- a/tools/gpu-aot/test_viewport_shadow.py +++ b/tools/gpu-aot/test_viewport_shadow.py @@ -33,10 +33,13 @@ def body(source, signature): hooks = (root / 'rexlego/src/gpu_native/hooks_state.cpp').read_text() draw = (root / 'rexlego/src/gpu_native/draw.cpp').read_text() native = '\n'.join(body(hooks, signature) for signature in - ['bool NativeViewportRange(', 'bool NativeViewportArguments(', + ['bool NativeViewportRange(', 'bool NativeViewportDeviceArguments(', 'bool NativeViewportArguments(', 'void SetScissorRectHook(', 'void SetScissorEnableHook(']) -raw = body(hooks, 'REX_HOOK_RAW(sub_83FBA978)') -raw = raw.replace('REX_HOOK_RAW(sub_83FBA978)', 'void sub_83FBA978(PPCContext& ctx,u8* base)', 1) +native += '\n' + body(hooks, 'class ScopedNativeViewportExtent') + ';' +textures = (root / 'rexlego/src/gpu_native/textures.cpp').read_text() +raw = '\n'.join(body(hooks, f'REX_HOOK_RAW({name})').replace( + f'REX_HOOK_RAW({name})', f'void {name}(PPCContext& ctx,u8* base)', 1) + for name in ('sub_83FBA978', 'sub_83FBA710')) begin = draw.index(' const float viewport_width =') end = draw.index('\n std::array(1);}; +std::map surfaces; +SurfaceMetadata* FindTexture(u32 at){auto it=surfaces.find(at);return it==surfaces.end()?nullptr:&it->second;} +''' + body(textures, 'bool NativePromotedSurfaceExtent(') + r''' struct HostDevice {static auto LockRecording(){return std::unique_lock(recording);}static inline std::recursive_mutex recording;}; ''' + native + r''' } void sub_83FBA710(PPCContext&,u8*);void sub_83FBA978(PPCContext&,u8*); -void __imp__sub_83FBA978(PPCContext&,u8*); +void __imp__sub_83FBA978(PPCContext&,u8*);void __imp__sub_83FBA710(PPCContext&,u8*); void sub_83FBA068(PPCContext& c,u8*){ auto* d=memory.TranslateVirtual(c.r3.u32); SetScissorRectHook(d,memory.TranslateVirtual(c.r4.u32)); } ''' + raw + '\n' + '\n'.join(originals) if not has_oracle: - source += 'u32 original_calls=0;void __imp__sub_83FBA978(PPCContext&,u8*){++original_calls;}\n' + source += 'u32 original_calls=0;void __imp__sub_83FBA978(PPCContext&,u8*){++original_calls;}void __imp__sub_83FBA710(PPCContext&,u8*){}\n' source += r''' namespace plume { using RenderRect=NativeScissorRect; @@ -195,7 +203,7 @@ def body(source, signature): void Api(float x,float y,u32 w,u32 h){auto* a=memory.TranslateVirtual(argument); a->x=u32(x);a->y=u32(y);a->width=w;a->height=h;a->min_z=0.25f;a->max_z=0.75f; auto c=Context();sub_83FBA978(c,ram.data());} -void Reset(){ram.fill(0);heap.unreadable.clear();heap.readonly.clear(); +void Reset(){surfaces.clear();ram.fill(0);heap.unreadable.clear();heap.readonly.clear(); Device().render_targets[0]=color_surface;Device().depth_stencil=depth_surface; auto* color=memory.TranslateVirtual(color_surface);color->size_bits=((64-1)<<18)|((48-1)<<3); auto* depth=memory.TranslateVirtual(depth_surface);depth->size_bits=((32-1)<<18)|((24-1)<<3); @@ -213,6 +221,10 @@ def body(source, signature): assert(NativeViewportInteger(12.75f)==12);assert(NativeViewportInteger(-12.75f)==-12); assert(NativeViewportInteger(INFINITY)==INT_MAX);assert(NativeViewportInteger(-INFINITY)==INT_MIN); assert(NativeViewportInteger(NAN)==INT_MIN); + const auto saved_scope_ram=ram; + try {ScopedNativeViewportExtent scope(&Device(),1280,720);throw 17;} + catch(int value){assert(value==17);} + assert(ram==saved_scope_ram); // Exception also restores transient tile state. Device().viewport.x=3.0f;Device().viewport.y=4.0f;Device().viewport.width=20.0f;Device().viewport.height=16.0f; auto recorded=Record(&Device());assert(recorded.viewport.x==3&&recorded.viewport.w==20); assert((recorded.scissor==NativeScissorRect{3,4,23,20})); @@ -278,6 +290,65 @@ def body(source, signature): c=Context();c.r5.u32=0;sub_83FBB110(c,ram.data()); assert(float(Device().viewport.width)==64&&float(Device().viewport.height)==48); recorded=Record(&Device());assert(recorded.viewport.w==64&&recorded.viewport.h==48); + + // Reproduce the reported menu: physical tile512, promoted host720. + Reset();auto* tiled=memory.TranslateVirtual(color_surface); + tiled->size_bits=((1280-1)<<18)|((512-1)<<3); + Api(0,0,1280,720);assert(float(Device().viewport.height)==512); + surfaces[color_surface]={1280,512,1280,720}; + // Promotion alone cannot recover a previously clipped request. The real + // scene must set its viewport after promotion; live bounded logs verify it. + assert(float(Device().viewport.height)==512); + ram[device+11068]=0xA0;Store(device+13556,17);Store(device+13560,19); + const auto unchanged_size=u32(tiled->size_bits); + const auto check_extent_restore=[&]{ + assert(ram[device+11068]==0xA0&&Load(device+13556)==17&&Load(device+13560)==19); + assert(u32(tiled->size_bits)==unchanged_size); + }; + Api(0,0,1280,720);assert(float(Device().viewport.height)==720);check_extent_restore(); + assert(Load(device+10512)==std::bit_cast(-360.0f)); + // Unadapted original is the exact512px regression negative oracle. + c=Context();c.f1.f64=0;c.f2.f64=0;c.f3.f64=1280;c.f4.f64=720;c.f5.f64=0;c.f6.f64=1; + __imp__sub_83FBA710(c,ram.data());assert(float(Device().viewport.height)==512); + Api(0,0,1280,720); + std::memcpy(ram.data()+argument,reinterpret_cast(&Device().viewport),28); + Device().viewport={};c=Context();sub_83FBA9F8(c,ram.data()); + assert(float(Device().viewport.height)==720);check_extent_restore(); + attachments=memory.TranslateVirtual(argument); + attachments[0]=depth_surface;attachments[1]=color_surface;attachments[2]=attachments[3]=attachments[4]=0; + c=Context();c.r5.u32=0;sub_83FBB110(c,ram.data()); + assert(float(Device().viewport.height)==720);check_extent_restore(); + Api(12,700,2000,200);assert(float(Device().viewport.width)==1268&&float(Device().viewport.height)==20); + check_extent_restore(); + Api(0,0,1280,0);assert(float(Device().viewport.height)==0);check_extent_restore(); + Api(0,800,1280,100);assert(float(Device().viewport.width)==0&&float(Device().viewport.height)==0); + check_extent_restore(); + // Depth fallback uses its own promotion, never the preceding color target. + Device().render_targets[0]=0;surfaces[depth_surface]={32,24,32,80}; + Api(0,0,100,100);assert(float(Device().viewport.width)==32&&float(Device().viewport.height)==80); + check_extent_restore(); + // Unknown, ordinary, non-surface and unrepresentable metadata stay untouched. + u32 ew=123,eh=456;assert(!NativePromotedSurfaceExtent(0,ew,eh)&&ew==123&&eh==456); + surfaces[color_surface]={1280,512,1280,512};assert(!NativePromotedSurfaceExtent(color_surface,ew,eh)); + surfaces[color_surface]={1280,512,1280,720,false};assert(!NativePromotedSurfaceExtent(color_surface,ew,eh)); + surfaces[color_surface]={1280,512,1280,720,true,nullptr};assert(!NativePromotedSurfaceExtent(color_surface,ew,eh)); + surfaces[color_surface]={1280,512,0,720};assert(!NativePromotedSurfaceExtent(color_surface,ew,eh)); + surfaces[color_surface]={1280,512,16385,720};assert(!NativePromotedSurfaceExtent(color_surface,ew,eh)); + surfaces[color_surface]={1280,512,1280,32769};assert(!NativePromotedSurfaceExtent(color_surface,ew,eh)); + // Shared raw entry validates arguments before touching state. + Reset();c=Context();c.r1.u32=128;before=ram;sub_83FBA710(c,ram.data());assert(ram==before); + c=Context();heap.readonly.insert(3);before=ram;sub_83FBA710(c,ram.data());assert(ram==before); + Reset(); + // API's valid512-byte stack guard must not grow to624 through its112-byte + // frame. The shared writer uses its exact160-byte scratch, all on same page. + heap.readonly.insert(0x1D); + assert(NativeViewportArguments(device,argument,0x1E200)); + assert(NativeViewportDeviceArguments(device,0x1E190,160)); + api=memory.TranslateVirtual(argument); + api->x=0;api->y=0;api->width=64;api->height=48;api->min_z=0;api->max_z=1; + c=Context();c.r1.u32=0x1E200;sub_83FBA978(c,ram.data()); + assert(float(Device().viewport.width)==64&&float(Device().viewport.height)==48); + Reset(); // Repeated bulk binds/reset cannot resurrect the preceding small viewport. Api(3,4,5,6);attachments[0]=depth_surface;attachments[1]=color_surface; attachments[2]=attachments[3]=attachments[4]=0; @@ -300,6 +371,8 @@ def body(source, signature): report = {'passed': True, 'actual_native_raw_guard_and_recording': True, 'actual_d3d_layout': True, 'actual_original_tu23_setters_and_bulk': has_oracle, 'legacy_uint_memcpy_negative': has_oracle, + 'promoted_tile_extent_and_512px_regression_negative': has_oracle, + 'actual_metadata_only_extent_query': True, 'original_bodies': provenance, 'image_constants_provenance': '.local-testing/reports/TU23-viewport-reference-byte-proof.json', 'limits': 'Bounded CPU fixture; no game or GPU execution'} From 99e839de9042ee13e83fe9b0d34fb46723f6fc38 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:38:09 +0200 Subject: [PATCH 38/40] Prepare explicit coherent PM4 correctness references --- tools/gpu-aot/regression/README.md | 17 +++++ tools/gpu-aot/regression/prepare_run.py | 39 +++++++++++ tools/gpu-aot/regression/test_regression.py | 76 +++++++++++++++++++++ 3 files changed, 132 insertions(+) diff --git a/tools/gpu-aot/regression/README.md b/tools/gpu-aot/regression/README.md index 5b94303..7124bc0 100644 --- a/tools/gpu-aot/regression/README.md +++ b/tools/gpu-aot/regression/README.md @@ -93,6 +93,23 @@ closure alone does not establish internal ABI compatibility. PM4 cannot use native diagnostic flags. Actual runtime route logs and visible rendering are still required; a launch descriptor does not prove either. +Add `--pm4-correctness` when preparing a fresh PM4 reference to apply an +explicit synchronous D3D12 RTV policy. It sets `readback_resolve="full"` and +`readback_resolve_max_kb=0`, enables shared memexport readback and both D3D12 +readback aliases, disables fast/batched/on-demand memexport, and enables +`clear_memory_page_state`. It also disables asynchronous shader compilation, +sets `d3d12_pipeline_creation_threads=0`, and sets +`invalid_function_nonfatal=false`. Native routes and an explicit conflicting +`--render-target-path-d3d12 rov` are rejected. + +The profile overrides inherited values before the configuration is hashed. +The manifest records every requested setting; reporting verifies those values +against the actual TOML as well as its hash. The source configuration and any +previously prepared runs remain unchanged. Use a new run name and port with +the same pinned baseline, runtime, plugin and checkpoint inputs. Full readback +adds synchronization cost, so this reference is not a throughput baseline. +Actual route, logs and rendering still need validation. + `--state-cache off` (default) or `on` explicitly selects the experimental Plume command state cache. The native descriptor writes an exact `0` or `1`; inherited cache settings are removed even when an older descriptor omits its sanitizer. diff --git a/tools/gpu-aot/regression/prepare_run.py b/tools/gpu-aot/regression/prepare_run.py index 1ada634..3d01b29 100644 --- a/tools/gpu-aot/regression/prepare_run.py +++ b/tools/gpu-aot/regression/prepare_run.py @@ -31,6 +31,19 @@ def save_path(xuid): SUPPORT_NAMES = ('FiraSans-Regular.ttf', 'achievement_unlocked.wav', 'gamecontrollerdb.txt') SLOTS = {'batman': (2, 4), 'gandalf': (2, 0), 'wyldstyle': (3, 6), 'batmobile': (1, 1)} SANITIZED_PREFIXES = ('LEGO_NATIVE_', 'LEGO_GPU_', 'LEGO_DUMP_', 'REX_') +# Explicit synchronous PM4 reference policy. Shared readback settings and +# D3D12 aliases both participate in the SDK's effective readback mode. +PM4_CORRECTNESS_SETTINGS = { + 'gpu_plugin': 'xenos', 'gpu_backend': 'd3d12', 'gpu_native_pm4': True, + 'render_target_path_d3d12': 'rtv', + 'readback_resolve': 'full', 'readback_resolve_max_kb': 0, + 'readback_memexport': True, 'd3d12_readback_memexport': True, + 'd3d12_readback_resolve': True, + 'readback_memexport_fast': False, 'readback_memexport_batched': False, + 'readback_memexport_on_demand': False, 'clear_memory_page_state': True, + 'async_shader_compilation': False, 'd3d12_pipeline_creation_threads': 0, + 'invalid_function_nonfatal': False, +} RECIPE = { 'name': 'operator-confirmed-checkpoint', 'steps': ['Start the exact verified descriptor after every existing game closes.', @@ -289,6 +302,11 @@ def config_text(original, replacements): def prepare(args): + pm4_correctness = getattr(args, 'pm4_correctness', False) + require(not pm4_correctness or args.renderer == 'pm4', + 'PM4 correctness profile requires PM4 route') + require(not pm4_correctness or getattr(args, 'render_target_path_d3d12', None) in (None, 'rtv'), + 'PM4 correctness profile requires RTV render-target path') timer_wait = getattr(args, 'timer_wait', 'spin') require(timer_wait in ('spin', 'blocking'), 'Unknown timer wait mode') cadence = getattr(args, 'cadence_only', False) @@ -452,6 +470,8 @@ def prepare(args): require(args.renderer == 'pm4', 'Render-target backend experiment requires PM4') require(target_path in ('rov', 'rtv'), 'Unsupported render-target backend') replacements['render_target_path_d3d12'] = target_path + if pm4_correctness: + replacements.update(PM4_CORRECTNESS_SETTINGS) config = config_text(original_config, replacements) (stage / 'legodimensions.toml').write_text(config) overrides = { @@ -520,6 +540,8 @@ def prepare(args): 'run_root': str(final), 'workspace': str(workspace), 'baseline': str(baseline), 'frozen_build': frozen, 'dependency_proof': dependency_proof, 'renderer_requested': args.renderer, 'renderer_verified_by_launch': False, + 'pm4_correctness': pm4_correctness, + 'pm4_correctness_settings': dict(PM4_CORRECTNESS_SETTINGS) if pm4_correctness else {}, 'performance_probe': performance, 'cadence_only_probe': cadence, 'gpu_timestamp_probe': gpu_timestamps, @@ -599,6 +621,20 @@ def verify(run, workspace=None): 'This verifier only accepts a prepared, unlaunched setup') renderer = manifest['renderer_requested'] require(renderer in ('native', 'pm4'), 'Unknown requested renderer') + correctness = manifest.get('pm4_correctness', False) + requested_settings = manifest.get('pm4_correctness_settings', {}) + require(type(correctness) is bool, 'Invalid PM4 correctness profile request') + if correctness: + require(renderer == 'pm4', 'PM4 correctness profile requires PM4 route') + require(requested_settings == PM4_CORRECTNESS_SETTINGS and + all(type(requested_settings.get(key)) is type(value) + for key, value in PM4_CORRECTNESS_SETTINGS.items()), + 'PM4 correctness requested settings changed') + require(all(type(config.get(key)) is type(value) and config[key] == value + for key, value in PM4_CORRECTNESS_SETTINGS.items()), + 'Prepared TOML disagrees with PM4 correctness profile') + else: + require(requested_settings == {}, 'Unrequested PM4 correctness settings') env = manifest['launch_plan']['environment_overrides'] allowed = {'STEAM_COMPAT_CLIENT_INSTALL_PATH', 'STEAM_COMPAT_DATA_PATH', 'PROTON_LOG', 'PROTON_LOG_DIR', 'SteamAppId', 'SteamGameId', 'WINEDLLOVERRIDES', @@ -710,6 +746,7 @@ def verify(run, workspace=None): 'status': 'prepared-identity-verified-not-launched', 'run_root': str(run), 'build_fingerprint': manifest['frozen_build']['build_fingerprint'], 'renderer_requested': renderer, 'renderer_verified_by_launch': False, + 'pm4_correctness': correctness, 'pm4_correctness_settings': requested_settings, 'route_authority': 'A future actual GPU route log decides renderer identity; embedded native fingerprint alone does not.', 'executable_sha256': digest(run / 'legodimensions.exe')['sha256'], 'checkpoint_files_verified': len(manifest['checkpoint_files']), @@ -758,6 +795,8 @@ def main(): p.add_argument('--snapshots', action='store_true', help='Explicitly enable bounded native DDS readbacks; changes timing workload') p.add_argument('--renderer', choices=('native', 'pm4'), default='native') + p.add_argument('--pm4-correctness', action='store_true', + help='PM4-only synchronous D3D12 RTV reference policy; overrides inherited readback/compilation settings') p.add_argument('--render-target-path-d3d12', choices=('rov', 'rtv'), help='Explicit PM4 render-target backend experiment; restart required') p.add_argument('--pm4-plugin', type=Path) diff --git a/tools/gpu-aot/regression/test_regression.py b/tools/gpu-aot/regression/test_regression.py index 93fda05..72055fe 100644 --- a/tools/gpu-aot/regression/test_regression.py +++ b/tools/gpu-aot/regression/test_regression.py @@ -522,6 +522,82 @@ def test_rtv_is_explicit_and_preserved_only_for_pm4(self): m.prepare(self.args('native-rtv', render_target_path_d3d12='rtv')) self.assertFalse((self.root / 'runs/native-rtv').exists()) + def test_pm4_correctness_transforms_inherited_fast_policy_before_hashing(self): + source = self.install / 'legodimensions.toml' + source.write_text(m.config_text(source.read_text(), { + 'gpu_backend': 'vulkan', 'render_target_path_d3d12': 'rov', + 'readback_resolve': 'fast', 'readback_resolve_max_kb': 256, + 'readback_memexport': False, 'd3d12_readback_memexport': False, + 'd3d12_readback_resolve': False, 'readback_memexport_fast': True, + 'readback_memexport_batched': True, 'readback_memexport_on_demand': True, + 'clear_memory_page_state': False, 'async_shader_compilation': True, + 'd3d12_pipeline_creation_threads': 8, 'invalid_function_nonfatal': True, + })) + source_before = source.read_bytes() + source_identity = m.digest(source) + previous = Path(m.prepare(self.args('pm4-existing', renderer='pm4'))['run_root']) + previous_config = (previous / 'legodimensions.toml').read_bytes() + previous_manifest = (previous / 'regression-manifest.json').read_bytes() + report = m.prepare(self.args('pm4-correctness', renderer='pm4', pm4_correctness=True)) + run = Path(report['run_root']) + config = m.tomllib.loads((run / 'legodimensions.toml').read_text()) + expected = { + 'gpu_plugin': 'xenos', 'gpu_backend': 'd3d12', 'gpu_native_pm4': True, + 'render_target_path_d3d12': 'rtv', 'readback_resolve': 'full', + 'readback_resolve_max_kb': 0, 'readback_memexport': True, + 'd3d12_readback_memexport': True, 'd3d12_readback_resolve': True, + 'readback_memexport_fast': False, 'readback_memexport_batched': False, + 'readback_memexport_on_demand': False, 'clear_memory_page_state': True, + 'async_shader_compilation': False, 'd3d12_pipeline_creation_threads': 0, + 'invalid_function_nonfatal': False, + } + self.assertEqual({key: config[key] for key in expected}, expected) + self.assertTrue(report['pm4_correctness']) + self.assertEqual(report['pm4_correctness_settings'], expected) + manifest = json.loads((run / 'regression-manifest.json').read_text()) + self.assertEqual(manifest['pm4_correctness_settings'], expected) + self.assertEqual(manifest['config_sha256'], m.digest(run / 'legodimensions.toml')['sha256']) + self.assertEqual(manifest['source_config_sha256'], source_identity['sha256']) + self.assertEqual(source.read_bytes(), source_before) + self.assertEqual(m.digest(source), source_identity) + self.assertEqual((previous / 'legodimensions.toml').read_bytes(), previous_config) + self.assertEqual((previous / 'regression-manifest.json').read_bytes(), previous_manifest) + self.assertFalse(m.verify(previous)['pm4_correctness']) + + def test_pm4_correctness_rejects_native_route_and_conflicting_backend(self): + for name, options in [('native-correctness', {'renderer': 'native'}), + ('pm4-correctness-rov', {'renderer': 'pm4', + 'render_target_path_d3d12': 'rov'})]: + with self.assertRaisesRegex(ValueError, 'PM4 correctness profile requires'): + m.prepare(self.args(name, pm4_correctness=True, **options)) + self.assertFalse((self.root / 'runs' / name).exists()) + + def test_pm4_correctness_verifier_rejects_config_and_profile_tampering(self): + run = Path(m.prepare(self.args('pm4-correctness-tamper', renderer='pm4', + pm4_correctness=True))['run_root']) + path = run / 'legodimensions.toml' + path.write_text(m.config_text(path.read_text(), {'readback_resolve_max_kb': 256})) + with self.assertRaisesRegex(ValueError, 'Prepared configuration changed'): + m.verify(run) + # Even a rehashed TOML cannot conceal disagreement with the declared + # reference profile; the verifier checks actual values and their types. + manifest_path = run / 'regression-manifest.json' + manifest = json.loads(manifest_path.read_text()) + manifest['config_sha256'] = m.digest(path)['sha256'] + manifest_path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, 'Prepared TOML disagrees'): + m.verify(run) + path.write_text(m.config_text(path.read_text(), {'readback_resolve_max_kb': 0, + 'clear_memory_page_state': 1})) + manifest['config_sha256'] = m.digest(path)['sha256'] + manifest_path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, 'Prepared TOML disagrees'): + m.verify(run) + manifest['pm4_correctness_settings']['readback_resolve'] = 'fast' + manifest_path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, 'requested settings changed'): + m.verify(run) + def test_runner_refuses_invalid_environment_without_spawning(self): plan = {'remove_inherited_prefixes': [], 'remove_inherited_keys': [], 'environment_overrides': {'REXGLUE_TOYPAD_PORT': '19201\0extra'}} From 8f235b6d5998ccd9bd4b313e8ebe14e5f2e2d058 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:12:10 +0200 Subject: [PATCH 39/40] Record verified menu recovery and stopped testing handoff --- tools/gpu-aot/COLLABORATION.md | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tools/gpu-aot/COLLABORATION.md b/tools/gpu-aot/COLLABORATION.md index 3aa11d8..4d44f4b 100644 --- a/tools/gpu-aot/COLLABORATION.md +++ b/tools/gpu-aot/COLLABORATION.md @@ -22,8 +22,18 @@ original viewport writer without that logical extent squashed the menu into baseline. The shared writer now receives a scoped promoted extent through its original CPU clipping branch; guest resource dimensions and persistent tiling state remain unchanged. The actual integer, float and bulk setter bodies cover -this regression, including the unadapted 512-row negative control. Live menu -recovery and post-promotion setter ordering still require verification. +this regression, including the unadapted 512-row negative control. Live build +`54d86a1b958e726e` recovered the full-height menu: bounded logs confirmed +post-promotion requests and effective viewports of 1280 by 720. The targeted +scissor, viewport and draw diagnostics passed 3/3 checks. Save replay reached +Vorton gameplay with Gandalf, Wyldstyle and Batman present. A controlled PM4 +comparison using the same executable, runtime, save checkpoint and prepared +tag bytes exposed a remaining native visual gap: large ground shadows are +missing. Small pose and camera differences prevent pixel-perfect comparison. +The exact failing shadow pass has not been identified. Evidence is preserved +under `.local-testing/reports/native-54d8-live-visual-verification.json`, +`native-pm4-54d8-controlled-visual-comparison.json` and +`native-54d8-shadow-gap-audit/`. An isolated native SDK build through local commit `5dc986f` passed its 225 selected unit and production regressions. Four existing BitStream cases remain @@ -50,8 +60,12 @@ order; other generated instructions and headers were byte-identical. The production registration regression passed after increasing its total deadline for five bounded compiler invocations. The complete native game then built within a 6 GiB owned-process ceiling, peaking around 2.4 GiB. Its matching -Windows runtime and PM4 plugin also built; Linux PM4 configuration is being -prepared in a separate source tree. +Windows runtime and PM4 plugin also built. The separate Linux PM4 build +completed compilation but initially failed its final game link with performance +counters disabled. A private retry with counters enabled was stopped at the +user's request at 659/711 tasks; final links remain unverified. No Linux game +was launched. Partial artifacts and the stop report remain in +`.local-testing/linux-pm4-candidate/user-requested-stop.json`. The last accepted native checkpoint samples remain 15.84/16.59 FPS with unequal draw workloads; no cache speedup is established and the cache stays off. A PM4 @@ -60,6 +74,13 @@ memexport/page-state refresh and synchronous pipeline creation. Inherited fast readback settings are unsuitable for that comparison. No level or full campaign completion has been verified. +Work concluded at the user's request to return the laptop. The PM4 game closed +normally with exit code zero. The owned Linux build group, compiler/linker/codegen +workers and private input daemon were stopped; their absence was verified. +No further gameplay, build or profiling jobs are running on behalf of this work. +The committed renderer fixes and private evidence are preserved. A full campaign +run without crashes, visual glitches or performance issues has not been achieved. + ## Linux limitations - `rexlego/CMakeLists.txt` explicitly rejects `LEGODIMENSIONS_NATIVE_GPU=ON` From 674975c60901147b07bd0f542453c9552a4699d2 Mon Sep 17 00:00:00 2001 From: AlenHay <82121826+AlenHay@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:32:20 +0200 Subject: [PATCH 40/40] Publish fetchable SDK integration and focused review handoff --- .gitattributes | 4 ++-- .gitmodules | 4 ++-- tools/gpu-aot/COLLABORATION.md | 30 ++++++++++++++++++++++++++---- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.gitattributes b/.gitattributes index 0955a1a..69ec4d3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,5 +5,5 @@ *.ttf binary *.bin binary -# Empty unified-diff context lines contain their required leading space. -tools/gpu-aot/patches/*.patch whitespace=-blank-at-eol +# Empty unified-diff context lines require their leading space, including at EOF. +tools/gpu-aot/patches/*.patch whitespace=-blank-at-eol,-blank-at-eof diff --git a/.gitmodules b/.gitmodules index d983f12..85b0db3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "rexglue-sdk"] path = rexglue-sdk - url = https://github.com/NeverCookFirst/rexglue-sdk.git - branch = toypad-ui + url = https://github.com/AlenHay/rexglue-sdk.git + branch = codex/runtime-correctness-20261007 [submodule "DimensionsModManager"] path = DimensionsModManager url = https://github.com/NeverCookFirst/DimensionsModLoader.git diff --git a/tools/gpu-aot/COLLABORATION.md b/tools/gpu-aot/COLLABORATION.md index 4d44f4b..e39ee54 100644 --- a/tools/gpu-aot/COLLABORATION.md +++ b/tools/gpu-aot/COLLABORATION.md @@ -6,6 +6,24 @@ development handoff, not a completed renderer or a release. ## Local verification on October 7, 2026 +The follow-up is published from `AlenHay/DimensionsRecomp` on +`t3/gpu-renderer-correctness`, against the original experimental draft branch. +The authenticated account cannot push to either upstream repository, so the +SDK gitlink now uses the public `AlenHay/rexglue-sdk` fork. Its exact tested +integration revision remains `2897c31b1407dd96480dc25965f8ee7807f00e0f`. +Focused SDK drafts against the original dependency branch are: + +- [Runtime races and teardown](https://github.com/NeverCookFirst/rexglue-sdk/pull/2). +- [Host/cross-build portability](https://github.com/NeverCookFirst/rexglue-sdk/pull/3). +- [Interior entries and compact registration](https://github.com/NeverCookFirst/rexglue-sdk/pull/4). +- [Portable NT calendar conversion](https://github.com/NeverCookFirst/rexglue-sdk/pull/5). + +Their production and test files match the tested integration; each independent +draft registers only its own fixtures. The independent drafts were not rebuilt +separately. Local SIMDe and Plume edits are excluded from publication; apply +their tracked preparers when reconstructing the tested dependencies. Game +inputs, generated game code, shader banks, captures and saves remain private. + The current source approach preserves exact TU23 CPU setters wherever they do not emit GPU packets, then translates their private state at the draw boundary. Public API structures and private device structures must not share a type @@ -111,7 +129,7 @@ run without crashes, visual glitches or performance issues has not been achieved ## Clone and portable checks ```sh -git clone --branch codex/native-renderer --recurse-submodules https://github.com/NeverCookFirst/DimensionsRecomp.git +git clone --branch t3/gpu-renderer-correctness --recurse-submodules https://github.com/AlenHay/DimensionsRecomp.git cd DimensionsRecomp cmake -S tools/gpu-aot -B native-checks -DCMAKE_BUILD_TYPE=Release cmake --build native-checks --parallel 2 @@ -207,11 +225,15 @@ allocation failure propagation needed by this renderer. The pin-checked preparer refuses conflicting local edits. The patch lives at `tools/gpu-aot/patches/plume-lego.patch`. -The SDK gitlink is `17e3675767067339581018c7e722e1d8d65de201`, published on -the fork's `codex/native-renderer` branch. Its dependency draft is -https://github.com/NeverCookFirst/rexglue-sdk/pull/1. Use the gitlink with +The SDK gitlink is `2897c31b1407dd96480dc25965f8ee7807f00e0f`, published on +the public `AlenHay/rexglue-sdk` fork's `codex/runtime-correctness-20261007` +branch. The original dependency draft is +https://github.com/NeverCookFirst/rexglue-sdk/pull/1; the focused follow-up +drafts are linked above. Use the gitlink with `git submodule update --init --recursive`; `git submodule update --remote` selects branch tips instead of the recorded dependency revision. +For the tested SIMDe compatibility overlay, run +`python3 rexglue-sdk/scripts/prepare_simde.py`, then repeat with `--check`. ## Reconstruct current shader compiler inputs